Merge branch 'master' into newparser

compiler/ast.nim

@@ -275,6 +275,7 @@ const
 
   sfDirty* = sfPure
     # template is not hygienic (old styled template)
+    # module, compiled from a dirty-buffer
 
   sfAnon* = sfDiscardable
     # symbol name that was generated by the compiler
@@ -680,7 +681,6 @@ type
     size*: BiggestInt         # the size of the type in bytes
                               # -1 means that the size is unkwown
     align*: int               # the type's alignment requirements
-    containerID*: int         # used for type checking of generics
     loc*: TLoc
 
   TPair*{.final.} = object 
@@ -1020,7 +1020,6 @@ proc assignType(dest, src: PType) =
   dest.n = src.n
   dest.size = src.size
   dest.align = src.align
-  dest.containerID = src.containerID
   dest.destructor = src.destructor
   # this fixes 'type TLock = TSysLock':
   if src.sym != nil:

compiler/cgen.nim

@@ -1279,10 +1279,6 @@ proc updateCachedModule(m: BModule) =
 
   addFileToLink(cfilenoext)
 
-proc updateCachedModules* =
-  for m in cgenModules():
-    if m.fromCache: m.updateCachedModule
-
 proc myClose(b: PPassContext, n: PNode): PNode = 
   result = n
   if b == nil or passes.skipCodegen(n): return 
@@ -1297,6 +1293,8 @@ proc myClose(b: PPassContext, n: PNode): PNode =
     var disp = generateMethodDispatchers()
     for i in 0..sonsLen(disp)-1: genProcAux(m, disp.sons[i].sym)
     genMainProc(m)
+
+proc cgenWriteModules* =
   # we need to process the transitive closure because recursive module
   # deps are allowed (and the system module is processed in the wrong
   # order anyway)
@@ -1306,8 +1304,10 @@ proc myClose(b: PPassContext, n: PNode): PNode =
       if not m.fromCache:
         finishModule(m)
   for m in cgenModules():
-      if not m.fromCache:
+    if m.fromCache:
-        writeModule(m, pending=true)
+      m.updateCachedModule
+    else:
+      m.writeModule(pending=true)
   writeMapping(gMapping)
   if generatedHeader != nil: writeHeader(generatedHeader)
 

compiler/commands.nim

@@ -207,6 +207,22 @@ proc processPath(path: string, notRelativeToProj = false): string =
     "projectname", options.gProjectName,
     "projectpath", options.gProjectPath])
 
+proc trackDirty(arg: string, info: TLineInfo) =
+  var a = arg.split(',')
+  if a.len != 4: LocalError(info, errTokenExpected,
+                            "DIRTY_BUFFER,ORIGINAL_FILE,LINE,COLUMN")
+  var line, column: int
+  if parseUtils.parseInt(a[2], line) <= 0:
+    LocalError(info, errInvalidNumber, a[1])
+  if parseUtils.parseInt(a[3], column) <= 0:
+    LocalError(info, errInvalidNumber, a[2])
+  
+  gDirtyBufferIdx = a[0].fileInfoIdx
+  gDirtyOriginalIdx = a[1].fileInfoIdx
+ 
+  optTrackPos = newLineInfo(gDirtyBufferIdx, line, column)
+  msgs.addCheckpoint(optTrackPos)
+
 proc track(arg: string, info: TLineInfo) = 
   var a = arg.split(',')
   if a.len != 3: LocalError(info, errTokenExpected, "FILE,LINE,COLUMN")
@@ -215,7 +231,8 @@ proc track(arg: string, info: TLineInfo) =
     LocalError(info, errInvalidNumber, a[1])
   if parseUtils.parseInt(a[2], column) <= 0:
     LocalError(info, errInvalidNumber, a[2])
-  msgs.addCheckpoint(newLineInfo(a[0], line, column))
+  optTrackPos = newLineInfo(a[0], line, column)
+  msgs.addCheckpoint(optTrackPos)
 
 proc dynlibOverride(switch, arg: string, pass: TCmdlinePass, info: TLineInfo) =
   if pass in {passCmd2, passPP}:
@@ -469,6 +486,9 @@ proc processSwitch(switch, arg: string, pass: TCmdlinePass, info: TLineInfo) =
   of "track":
     expectArg(switch, arg, pass, info)
     track(arg, info)
+  of "trackdirty":
+    expectArg(switch, arg, pass, info)
+    trackDirty(arg, info)
   of "suggest": 
     expectNoArg(switch, arg, pass, info)
     incl(gGlobalOptions, optSuggest)

compiler/extccomp.nim

@@ -579,7 +579,7 @@ proc CallCCompiler*(projectfile: string) =
       else:
         rawMessage(errGenerated, " execution of an external program failed; " &
                    "rerun with --parallelBuild:1 to see the error message")
-  if optNoLinking notin gGlobalOptions:
+  if optNoLinking notin gGlobalOptions and cmds.len > 0:
     # call the linker:
     var it = PStrEntry(toLink.head)
     var objfiles = ""

compiler/main.nim

@@ -16,7 +16,7 @@ import
   wordrecg, sem, semdata, idents, passes, docgen, extccomp,
   cgen, jsgen, cgendata, json, nversion,
   platform, nimconf, importer, passaux, depends, evals, types, idgen,
-  tables, docgen2, service, magicsys, parser, crc, ccgutils
+  tables, docgen2, service, magicsys, parser, crc, ccgutils, sigmatch
 
 const
   has_LLVM_Backend = false
@@ -89,38 +89,38 @@ proc addDep(x: Psym, dep: int32) =
   growCache gMemCacheData, dep
   gMemCacheData[x.position].deps.safeAdd(dep)
 
-proc ResetModule(fileIdx: int32) =
+proc resetModule(fileIdx: int32) =
-  echo "HARD RESETTING ", fileIdx.toFilename
+  # echo "HARD RESETTING ", fileIdx.toFilename
   gMemCacheData[fileIdx].needsRecompile = Yes
   gCompiledModules[fileIdx] = nil
   cgendata.gModules[fileIdx] = nil
+  resetSourceMap(fileIdx)
 
-proc ResetAllModules =
+proc resetAllModules =
   for i in 0..gCompiledModules.high:
     if gCompiledModules[i] != nil:
-      ResetModule(i.int32)
+      resetModule(i.int32)
 
-  for m in cgenModules():
+  # for m in cgenModules(): echo "CGEN MODULE FOUND"
-    echo "CGEN MODULE FOUND"
 
 proc checkDepMem(fileIdx: int32): TNeedRecompile  =
   template markDirty =
-    ResetModule(fileIdx)
+    resetModule(fileIdx)
     return Yes
 
   if gMemCacheData[fileIdx].needsRecompile != Maybe:
     return gMemCacheData[fileIdx].needsRecompile
 
   if optForceFullMake in gGlobalOptions or
-     curCaasCmd != lastCaasCmd or
+     crcChanged(fileIdx):
-     crcChanged(fileIdx): markDirty
+       markDirty
   
   if gMemCacheData[fileIdx].deps != nil:
     gMemCacheData[fileIdx].needsRecompile = Probing
     for dep in gMemCacheData[fileIdx].deps:
       let d = checkDepMem(dep)
       if d in { Yes, Recompiled }:
-        echo fileIdx.toFilename, " depends on ", dep.toFilename, " ", d
+        # echo fileIdx.toFilename, " depends on ", dep.toFilename, " ", d
         markDirty
   
   gMemCacheData[fileIdx].needsRecompile = No
@@ -213,6 +213,9 @@ proc rodPass =
   if optSymbolFiles in gGlobalOptions:
     registerPass(rodwritePass)
 
+proc codegenPass =
+  registerPass cgenPass
+
 proc semanticPasses =
   registerPass verbosePass
   registerPass semPass
@@ -251,14 +254,11 @@ proc CommandCompileToC =
     # echo "CHECK DEP COMPLETE"
 
   compileProject()
-
+  cgenWriteModules()
-  if compilationCachePresent:
-    updateCachedModules()
-
   if gCmd != cmdRun:
     extccomp.CallCCompiler(changeFileExt(gProjectFull, ""))
 
-  if optCaasEnabled in gGlobalOptions:
+  if isServing:
     # caas will keep track only of the compilation commands
     lastCaasCmd = curCaasCmd
     resetCgenModules()
@@ -377,6 +377,19 @@ proc CommandScan =
     rawMessage(errCannotOpenFile, f)
   
 proc CommandSuggest =
+  if isServing:
+    # XXX: hacky work-around ahead
+    # Currently, it's possible to issue a idetools command, before
+    # issuing the first compile command. This will leave the compiler
+    # cache in a state where "no recompilation is necessary", but the
+    # cgen pass was never executed at all.
+    CommandCompileToC()
+    if gDirtyBufferIdx != 0:
+      discard compileModule(gDirtyBufferIdx, {sfDirty})
+      resetModule(gDirtyBufferIdx)
+    if optDef in gGlobalOptions:
+      defFromSourceMap(optTrackPos)
+  else:
     msgs.gErrorMax = high(int)  # do not stop after first error
     semanticPasses()
     rodPass()
@@ -404,7 +417,7 @@ proc requireMainModuleOption =
 proc resetMemory =
   resetCompilationLists()
   ccgutils.resetCaches()
-  ResetAllModules()
+  resetAllModules()
   resetRopeCache()
   resetSysTypes()
   gOwners = @[]

compiler/msgs.nim

@@ -528,9 +528,13 @@ proc SuggestWriteln*(s: string) =
       stdoutSocket.send(s & "\c\L")
 
 proc SuggestQuit*() =
-  if not isServing: quit(0)
+  if not isServing:
-  elif not isNil(stdoutSocket):
+    quit(0)
-    stdoutSocket.send("\c\L")
+  elif isWorkingWithDirtyBuffer:
+    # No need to compile the rest if we are working with a
+    # throw-away buffer. Incomplete dot expressions frequently
+    # found in dirty buffers will result in errors few steps
+    # from now anyway.
     raise newException(ESuggestDone, "suggest done")
 
 # this format is understood by many text editors: it is the same that
@@ -606,6 +610,7 @@ proc `??`* (info: TLineInfo, filename: string): bool =
   result = filename in info.toFilename
 
 var checkPoints*: seq[TLineInfo] = @[]
+var optTrackPos*: TLineInfo
 
 proc addCheckpoint*(info: TLineInfo) = 
   checkPoints.add(info)

compiler/options.nim

@@ -107,10 +107,16 @@ var
   gLastCmdTime*: float        # when caas is enabled, we measure each command
   gListFullPaths*: bool
   isServing*: bool = false
+  gDirtyBufferIdx* = 0'i32    # indicates the fileIdx of the dirty version of
+                              # the tracked source X, saved by the CAAS client.
+  gDirtyOriginalIdx* = 0'i32  # the original source file of the dirtified buffer.
 
 proc importantComments*(): bool {.inline.} = gCmd in {cmdDoc, cmdIdeTools}
 proc usesNativeGC*(): bool {.inline.} = gSelectedGC >= gcRefc
 
+template isWorkingWithDirtyBuffer*: expr =
+  gDirtyBufferIdx != 0
+
 template compilationCachePresent*: expr =
   {optCaasEnabled, optSymbolFiles} * gGlobalOptions != {}
 

compiler/rodread.nim

@@ -328,9 +328,6 @@ proc decodeType(r: PRodReader, info: TLineInfo): PType =
     result.align = decodeVInt(r.s, r.pos)
   else: 
     result.align = 2
-  if r.s[r.pos] == '@': 
-    inc(r.pos)
-    result.containerID = decodeVInt(r.s, r.pos)
   decodeLoc(r, result.loc, info)
   while r.s[r.pos] == '^': 
     inc(r.pos)
@@ -1012,9 +1009,6 @@ proc writeType(f: TFile; t: PType) =
   if t.align != 2:
     f.write('=')
     f.write($t.align)
-  if t.containerID != 0: 
-    f.write('@')
-    f.write($t.containerID)
   for i in countup(0, sonsLen(t) - 1): 
     if t.sons[i] == nil: 
       f.write("^()")

compiler/rodwrite.nim

@@ -233,9 +233,6 @@ proc encodeType(w: PRodWriter, t: PType, result: var string) =
   if t.align != 2: 
     add(result, '=')
     encodeVInt(t.align, result)
-  if t.containerID != 0: 
-    add(result, '@')
-    encodeVInt(t.containerID, result)
   encodeLoc(w, t.loc, result)
   for i in countup(0, sonsLen(t) - 1): 
     if t.sons[i] == nil: 

compiler/semstmts.nim

@@ -693,9 +693,6 @@ proc typeSectionRightSidePass(c: PContext, n: PNode) =
       openScope(c.tab)
       pushOwner(s)
       if s.magic == mNone: s.typ.kind = tyGenericBody
-      if s.typ.containerID != 0: 
-        InternalError(a.info, "semTypeSection: containerID")
-      s.typ.containerID = s.typ.id
       # XXX for generic type aliases this is not correct! We need the
       # underlying Id really: 
       #

compiler/semtypes.nim

@@ -770,9 +770,6 @@ proc semGeneric(c: PContext, n: PNode, s: PSym, prev: PType): PType =
     return newOrPrevType(tyError, prev, c)
   elif s.typ.kind != tyGenericBody:
     isConcrete = false
-  elif s.typ.containerID == 0: 
-    InternalError(n.info, "semtypes.semGeneric")
-    return newOrPrevType(tyError, prev, c)
   elif sonsLen(n) != sonsLen(s.typ): 
     LocalError(n.info, errWrongNumberOfArguments)
     return newOrPrevType(tyError, prev, c)

compiler/service.nim

@@ -65,6 +65,9 @@ proc serve*(action: proc (){.nimcall.}) =
     curCaasCmd = cmd
     processCmdLine(passCmd2, cmd)
     action()
+    gDirtyBufferIdx = 0
+    gDirtyOriginalIdx = 0
+    gErrorCounter = 0
 
   let typ = getConfigVar("server.type")
   case typ

compiler/sigmatch.nim

@@ -559,9 +559,8 @@ proc typeRel(c: var TCandidate, f, a: PType): TTypeRelation =
       # simply no match for now:
       nil
     elif x.kind == tyGenericInst and 
-          (f.sons[0].containerID == x.sons[0].containerID) and
+          (f.sons[0] == x.sons[0]) and
           (sonsLen(x) - 1 == sonsLen(f)):
-      assert(x.sons[0].kind == tyGenericBody)
       for i in countup(1, sonsLen(f) - 1):
         if x.sons[i].kind == tyGenericParam:
           InternalError("wrong instantiated type!")

compiler/suggest.nim

@@ -9,7 +9,9 @@
 
 ## This file implements features required for IDE support.
 
-# imported from sigmatch.nim
+# included from sigmatch.nim
+
+import algorithm
 
 const
   sep = '\t'
@@ -238,12 +240,69 @@ proc findDefinition(node: PNode, s: PSym) =
     SuggestWriteln(SymToStr(s, isLocal=false, sectionDef))
     SuggestQuit()
 
+type
+  TSourceMap = object
+    lines: seq[TLineMap]
+  
+  TEntry = object
+    pos: int
+    sym: PSym
+
+  TLineMap = object
+    entries: seq[TEntry]
+
+var
+  gSourceMaps: seq[TSourceMap] = @[]
+
+proc ensureIdx[T](x: var T, y: int) =
+  if x.len <= y: x.setLen(y+1)
+
+proc ensureSeq[T](x: var seq[T]) =
+  if x == nil: newSeq(x, 0)
+
+proc resetSourceMap*(fileIdx: int32) =
+  ensureIdx(gSourceMaps, fileIdx)
+  gSourceMaps[fileIdx].lines = @[]
+
+proc addToSourceMap(sym: Psym, info: TLineInfo) =
+  ensureIdx(gSourceMaps, info.fileIndex)
+  ensureSeq(gSourceMaps[info.fileIndex].lines)
+  ensureIdx(gSourceMaps[info.fileIndex].lines, info.line)
+  ensureSeq(gSourceMaps[info.fileIndex].lines[info.line].entries)
+  gSourceMaps[info.fileIndex].lines[info.line].entries.add(TEntry(pos: info.col, sym: sym))
+
+proc defFromLine(entries: var seq[TEntry], col: int32) =
+  if entries == nil: return
+  # The sorting is done lazily here on purpose.
+  # No need to pay the price for it unless the user requests
+  # "goto definition" on a particular line
+  sort(entries) do (a,b: TEntry) -> int:
+    return cmp(a.pos, b.pos)
+  
+  for e in entries:
+    # currently, the line-infos for most expressions point to
+    # one position past the end of the expression. This means
+    # that the first expr that ends after the cursor column is
+    # the one we are looking for.
+    if e.pos >= col:
+      SuggestWriteln(SymToStr(e.sym, isLocal=false, sectionDef))
+      return
+
+proc defFromSourceMap*(i: TLineInfo) =
+  if not ((i.fileIndex < gSourceMaps.len) and
+          (gSourceMaps[i.fileIndex].lines != nil) and
+          (i.line < gSourceMaps[i.fileIndex].lines.len)): return
+  
+  defFromLine(gSourceMaps[i.fileIndex].lines[i.line].entries, i.col)
+
 proc suggestSym*(n: PNode, s: PSym) {.inline.} =
   ## misnamed: should be 'symDeclared'
   if optUsages in gGlobalOptions:
     findUsages(n, s)
   if optDef in gGlobalOptions:
     findDefinition(n, s)
+  if isServing:
+    addToSourceMap(s, n.info)
 
 proc markUsed(n: PNode, s: PSym) =
   incl(s.flags, sfUsed)

tests/caas/compile-suggest.txt

@@ -0,0 +1,7 @@
+main.nim
+> c
+SuccessX
+> idetools --trackDirty:main_dirty.nim,main.nim,12,7 --suggest main.nim
+skField\tx
+skField\ty
+

tests/caas/def-def-compile.txt

@@ -0,0 +1,10 @@
+main.nim
+> idetools --track:main.nim,5,18 --def main.nim
+strutils.toUpper
+SuccessX
+> idetools --track:main.nim,5,18 --def main.nim
+strutils.toUpper
+SuccessX
+> c
+SuccessX
+

tests/caas/main_dirty.nim

@@ -0,0 +1,14 @@
+import imported, strutils
+
+type
+  TFoo = object
+    x: int
+    y: string
+
+proc main =
+  var t1 = "text"
+  var t2 = t1.toUpper
+  var foo = TFoo(x: 10, y: "test")
+  foo.
+  echo(t1 +++ t2)
+

tests/caas/suggest-compile.txt

@@ -0,0 +1,7 @@
+main.nim
+> idetools --trackDirty:main_dirty.nim,main.nim,12,7 --suggest main.nim
+skField\tx
+skField\ty
+> c
+SuccessX
+