nimsuggest is now part of Nim

compiler/suggest.nim

@@ -233,7 +233,7 @@ proc suggestFieldAccess(c: PContext, n: PNode, outputs: var int) =
         # error: no known module name:
         typ = nil
       else:
-        let m = gImportModule(c.module, fullpath.fileInfoIdx)
+        let m = gImportModule(c.module, fullpath.fileInfoIdx, c.cache)
         if m == nil: typ = nil
         else:
           for it in items(n.sym.tab):

koch.nim

@@ -194,14 +194,10 @@ proc buildNimble() =
   copyExe(installDir / "src/nimble".exe, "bin/nimble".exe)
 
 proc bundleNimsuggest(buildExe: bool) =
-  if dirExists("dist/nimsuggest/.git"):
-    exec("git --git-dir dist/nimsuggest/.git pull")
-  else:
-    exec("git clone https://github.com/nim-lang/nimsuggest.git dist/nimsuggest")
   if buildExe:
-    nimexec("c --noNimblePath -d:release -p:compiler dist/nimsuggest/nimsuggest.nim")
+    nimexec("c --noNimblePath -d:release -p:compiler tools/nimsuggest/nimsuggest.nim")
-    copyExe("dist/nimsuggest/nimsuggest".exe, "bin/nimsuggest".exe)
+    copyExe("tools/nimsuggest/nimsuggest".exe, "bin/nimsuggest".exe)
-    removeFile("dist/nimsuggest/nimsuggest".exe)
+    removeFile("tools/nimsuggest/nimsuggest".exe)
 
 proc bundleWinTools() =
   nimexec("c tools/finish.nim")

tools/nimsuggest/nimsuggest.nim

@@ -0,0 +1,477 @@
+#
+#
+#           The Nim Compiler
+#        (c) Copyright 2016 Andreas Rumpf
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+## Nimsuggest is a tool that helps to give editors IDE like capabilities.
+
+import strutils, os, parseopt, parseutils, sequtils, net, rdstdin, sexp
+# Do NOT import suggest. It will lead to wierd bugs with
+# suggestionResultHook, because suggest.nim is included by sigmatch.
+# So we import that one instead.
+import compiler/options, compiler/commands, compiler/modules, compiler/sem,
+  compiler/passes, compiler/passaux, compiler/msgs, compiler/nimconf,
+  compiler/extccomp, compiler/condsyms, compiler/lists,
+  compiler/sigmatch, compiler/ast, compiler/scriptconfig,
+  compiler/idents
+
+when defined(windows):
+  import winlean
+else:
+  import posix
+
+const DummyEof = "!EOF!"
+const Usage = """
+Nimsuggest - Tool to give every editor IDE like capabilities for Nim
+Usage:
+  nimsuggest [options] projectfile.nim
+
+Options:
+  --port:PORT             port, by default 6000
+  --address:HOST          binds to that address, by default ""
+  --stdin                 read commands from stdin and write results to
+                          stdout instead of using sockets
+  --epc                   use emacs epc mode
+  --debug                 enable debug output
+  --log                   enable verbose logging to nimsuggest.log file
+  --v2                    use version 2 of the protocol; more features and
+                          much faster
+  --tester                implies --v2 and --stdin and outputs a line
+                          '""" & DummyEof & """' for the tester
+
+The server then listens to the connection and takes line-based commands.
+
+In addition, all command line options of Nim that do not affect code generation
+are supported.
+"""
+type
+  Mode = enum mstdin, mtcp, mepc
+
+var
+  gPort = 6000.Port
+  gAddress = ""
+  gMode: Mode
+  gEmitEof: bool # whether we write '!EOF!' dummy lines
+  gLogging = false
+
+const
+  seps = {':', ';', ' ', '\t'}
+  Help = "usage: sug|con|def|use|dus|chk|highlight|outline file.nim[;dirtyfile.nim]:line:col\n" &
+         "type 'quit' to quit\n" &
+         "type 'debug' to toggle debug mode on/off\n" &
+         "type 'terse' to toggle terse mode on/off"
+
+type
+  EUnexpectedCommand = object of Exception
+
+proc logStr(line: string) =
+  var f: File
+  if open(f, getHomeDir() / "nimsuggest.log", fmAppend):
+    f.writeLine(line)
+    f.close()
+
+proc parseQuoted(cmd: string; outp: var string; start: int): int =
+  var i = start
+  i += skipWhitespace(cmd, i)
+  if cmd[i] == '"':
+    i += parseUntil(cmd, outp, '"', i+1)+2
+  else:
+    i += parseUntil(cmd, outp, seps, i)
+  result = i
+
+proc sexp(s: IdeCmd|TSymKind): SexpNode = sexp($s)
+
+proc sexp(s: Suggest): SexpNode =
+  # If you change the order here, make sure to change it over in
+  # nim-mode.el too.
+  result = convertSexp([
+    s.section,
+    s.symkind,
+    s.qualifiedPath.map(newSString),
+    s.filePath,
+    s.forth,
+    s.line,
+    s.column,
+    s.doc
+  ])
+
+proc sexp(s: seq[Suggest]): SexpNode =
+  result = newSList()
+  for sug in s:
+    result.add(sexp(sug))
+
+proc listEPC(): SexpNode =
+  # This function is called from Emacs to show available options.
+  let
+    argspecs = sexp("file line column dirtyfile".split(" ").map(newSSymbol))
+    docstring = sexp("line starts at 1, column at 0, dirtyfile is optional")
+  result = newSList()
+  for command in ["sug", "con", "def", "use", "dus", "chk"]:
+    let
+      cmd = sexp(command)
+      methodDesc = newSList()
+    methodDesc.add(cmd)
+    methodDesc.add(argspecs)
+    methodDesc.add(docstring)
+    result.add(methodDesc)
+
+proc findNode(n: PNode): PSym =
+  #echo "checking node ", n.info
+  if n.kind == nkSym:
+    if isTracked(n.info, n.sym.name.s.len): return n.sym
+  else:
+    for i in 0 ..< safeLen(n):
+      let res = n.sons[i].findNode
+      if res != nil: return res
+
+proc symFromInfo(gTrackPos: TLineInfo): PSym =
+  let m = getModule(gTrackPos.fileIndex)
+  #echo m.isNil, " I knew it ", gTrackPos.fileIndex
+  if m != nil and m.ast != nil:
+    result = m.ast.findNode
+
+proc execute(cmd: IdeCmd, file, dirtyfile: string, line, col: int;
+             cache: IdentCache) =
+  if gLogging:
+    logStr("cmd: " & $cmd & ", file: " & file & ", dirtyFile: " & dirtyfile & "[" & $line & ":" & $col & "]")
+  gIdeCmd = cmd
+  if cmd == ideUse and suggestVersion != 2:
+    modules.resetAllModules()
+  var isKnownFile = true
+  let dirtyIdx = file.fileInfoIdx(isKnownFile)
+
+  if dirtyfile.len != 0: msgs.setDirtyFile(dirtyIdx, dirtyfile)
+  else: msgs.setDirtyFile(dirtyIdx, nil)
+
+  gTrackPos = newLineInfo(dirtyIdx, line, col)
+  gErrorCounter = 0
+  if suggestVersion < 2:
+    usageSym = nil
+  if not isKnownFile:
+    compileProject(cache)
+  if suggestVersion == 2 and gIdeCmd in {ideUse, ideDus} and
+      dirtyfile.len == 0:
+    discard "no need to recompile anything"
+  else:
+    resetModule dirtyIdx
+    if dirtyIdx != gProjectMainIdx:
+      resetModule gProjectMainIdx
+    compileProject(cache, dirtyIdx)
+  if gIdeCmd in {ideUse, ideDus}:
+    let u = if suggestVersion >= 2: symFromInfo(gTrackPos) else: usageSym
+    if u != nil:
+      listUsages(u)
+    else:
+      localError(gTrackPos, "found no symbol at this position " & $gTrackPos)
+
+proc executeEpc(cmd: IdeCmd, args: SexpNode; cache: IdentCache) =
+  let
+    file = args[0].getStr
+    line = args[1].getNum
+    column = args[2].getNum
+  var dirtyfile = ""
+  if len(args) > 3:
+    dirtyfile = args[3].getStr(nil)
+  execute(cmd, file, dirtyfile, int(line), int(column), cache)
+
+proc returnEpc(socket: var Socket, uid: BiggestInt, s: SexpNode|string,
+               return_symbol = "return") =
+  let response = $convertSexp([newSSymbol(return_symbol), uid, s])
+  socket.send(toHex(len(response), 6))
+  socket.send(response)
+
+template sendEpc(results: typed, tdef, hook: untyped) =
+  hook = proc (s: tdef) =
+    results.add(
+      # Put newlines to parse output by flycheck-nim.el
+      when results is string: s & "\n"
+      else: s
+    )
+
+  executeEpc(gIdeCmd, args, cache)
+  let res = sexp(results)
+  if gLogging:
+    logStr($res)
+  returnEPC(client, uid, res)
+
+template checkSanity(client, sizeHex, size, messageBuffer: typed) =
+  if client.recv(sizeHex, 6) != 6:
+    raise newException(ValueError, "didn't get all the hexbytes")
+  if parseHex(sizeHex, size) == 0:
+    raise newException(ValueError, "invalid size hex: " & $sizeHex)
+  if client.recv(messageBuffer, size) != size:
+    raise newException(ValueError, "didn't get all the bytes")
+
+template setVerbosity(level: typed) =
+  gVerbosity = level
+  gNotes = NotesVerbosity[gVerbosity]
+
+proc connectToNextFreePort(server: Socket, host: string): Port =
+  server.bindaddr(Port(0), host)
+  let (_, port) = server.getLocalAddr
+  result = port
+
+proc parseCmdLine(cmd: string; cache: IdentCache) =
+  template toggle(sw) =
+    if sw in gGlobalOptions:
+      excl(gGlobalOptions, sw)
+    else:
+      incl(gGlobalOptions, sw)
+    return
+
+  template err() =
+    echo Help
+    return
+
+  var opc = ""
+  var i = parseIdent(cmd, opc, 0)
+  case opc.normalize
+  of "sug": gIdeCmd = ideSug
+  of "con": gIdeCmd = ideCon
+  of "def": gIdeCmd = ideDef
+  of "use": gIdeCmd = ideUse
+  of "dus": gIdeCmd = ideDus
+  of "chk":
+    gIdeCmd = ideChk
+    incl(gGlobalOptions, optIdeDebug)
+  of "highlight": gIdeCmd = ideHighlight
+  of "outline": gIdeCmd = ideOutline
+  of "quit": quit()
+  of "debug": toggle optIdeDebug
+  of "terse": toggle optIdeTerse
+  else: err()
+  var dirtyfile = ""
+  var orig = ""
+  i = parseQuoted(cmd, orig, i)
+  if cmd[i] == ';':
+    i = parseQuoted(cmd, dirtyfile, i+1)
+  i += skipWhile(cmd, seps, i)
+  var line = -1
+  var col = 0
+  i += parseInt(cmd, line, i)
+  i += skipWhile(cmd, seps, i)
+  i += parseInt(cmd, col, i)
+
+  execute(gIdeCmd, orig, dirtyfile, line, col-1, cache)
+
+proc serveStdin(cache: IdentCache) =
+  if gEmitEof:
+    echo DummyEof
+    while true:
+      let line = readLine(stdin)
+      parseCmdLine line, cache
+      echo DummyEof
+      flushFile(stdout)
+  else:
+    echo Help
+    var line = ""
+    while readLineFromStdin("> ", line):
+      parseCmdLine line, cache
+      echo ""
+      flushFile(stdout)
+
+proc serveTcp(cache: IdentCache) =
+  var server = newSocket()
+  server.bindAddr(gPort, gAddress)
+  var inp = "".TaintedString
+  server.listen()
+
+  while true:
+    var stdoutSocket = newSocket()
+    msgs.writelnHook = proc (line: string) =
+      stdoutSocket.send(line & "\c\L")
+
+    accept(server, stdoutSocket)
+
+    stdoutSocket.readLine(inp)
+    parseCmdLine inp.string, cache
+
+    stdoutSocket.send("\c\L")
+    stdoutSocket.close()
+
+proc serveEpc(server: Socket; cache: IdentCache) =
+  var client = newSocket()
+  # Wait for connection
+  accept(server, client)
+  if gLogging:
+    var it = searchPaths.head
+    while it != nil:
+      logStr(PStrEntry(it).data)
+      it = it.next
+    msgs.writelnHook = proc (line: string) = logStr(line)
+
+  while true:
+    var
+      sizeHex = ""
+      size = 0
+      messageBuffer = ""
+    checkSanity(client, sizeHex, size, messageBuffer)
+    let
+      message = parseSexp($messageBuffer)
+      epcAPI = message[0].getSymbol
+    case epcAPI:
+    of "call":
+      let
+        uid = message[1].getNum
+        args = message[3]
+
+      gIdeCmd = parseIdeCmd(message[2].getSymbol)
+      case gIdeCmd
+      of ideChk:
+        setVerbosity(1)
+        # Use full path because other emacs plugins depends it
+        gListFullPaths = true
+        incl(gGlobalOptions, optIdeDebug)
+        var hints_or_errors = ""
+        sendEpc(hints_or_errors, string, msgs.writelnHook)
+      of ideSug, ideCon, ideDef, ideUse, ideDus, ideOutline, ideHighlight:
+        setVerbosity(0)
+        var suggests: seq[Suggest] = @[]
+        sendEpc(suggests, Suggest, suggestionResultHook)
+      else: discard
+    of "methods":
+      returnEpc(client, message[1].getNum, listEPC())
+    of "epc-error":
+      stderr.writeline("recieved epc error: " & $messageBuffer)
+      raise newException(IOError, "epc error")
+    else:
+      let errMessage = case epcAPI
+                       of "return", "return-error":
+                         "no return expected"
+                       else:
+                         "unexpected call: " & epcAPI
+      raise newException(EUnexpectedCommand, errMessage)
+
+template beCompatible() =
+  when compiles(modules.gFuzzyGraphChecking):
+    modules.gFuzzyGraphChecking = true
+
+proc mainCommand(cache: IdentCache) =
+  clearPasses()
+  registerPass verbosePass
+  registerPass semPass
+  gCmd = cmdIdeTools
+  incl gGlobalOptions, optCaasEnabled
+  isServing = true
+  wantMainModule()
+  appendStr(searchPaths, options.libpath)
+  #if gProjectFull.len != 0:
+    # current path is always looked first for modules
+  #  prependStr(searchPaths, gProjectPath)
+
+  # do not stop after the first error:
+  msgs.gErrorMax = high(int)
+
+  case gMode
+  of mstdin:
+    beCompatible()
+    compileProject(cache)
+    #modules.gFuzzyGraphChecking = false
+    serveStdin(cache)
+  of mtcp:
+    # until somebody accepted the connection, produce no output (logging is too
+    # slow for big projects):
+    msgs.writelnHook = proc (msg: string) = discard
+    beCompatible()
+    compileProject(cache)
+    #modules.gFuzzyGraphChecking = false
+    serveTcp(cache)
+  of mepc:
+    beCompatible()
+    var server = newSocket()
+    let port = connectToNextFreePort(server, "localhost")
+    server.listen()
+    echo port
+    compileProject(cache)
+    serveEpc(server, cache)
+
+proc processCmdLine*(pass: TCmdLinePass, cmd: string) =
+  var p = parseopt.initOptParser(cmd)
+  while true:
+    parseopt.next(p)
+    case p.kind
+    of cmdEnd: break
+    of cmdLongoption, cmdShortOption:
+      case p.key.normalize
+      of "port":
+        gPort = parseInt(p.val).Port
+        gMode = mtcp
+      of "address":
+        gAddress = p.val
+        gMode = mtcp
+      of "stdin": gMode = mstdin
+      of "epc":
+        gMode = mepc
+        gVerbosity = 0          # Port number gotta be first.
+      of "debug":
+        incl(gGlobalOptions, optIdeDebug)
+      of "v2":
+        suggestVersion = 2
+      of "tester":
+        suggestVersion = 2
+        gMode = mstdin
+        gEmitEof = true
+      of "log":
+        gLogging = true
+      else: processSwitch(pass, p)
+    of cmdArgument:
+      options.gProjectName = unixToNativePath(p.key)
+      # if processArgument(pass, p, argsCount): break
+
+proc handleCmdLine(cache: IdentCache) =
+  if paramCount() == 0:
+    stdout.writeline(Usage)
+  else:
+    processCmdLine(passCmd1, "")
+    if gMode != mstdin:
+      msgs.writelnHook = proc (msg: string) = discard
+    if gProjectName != "":
+      try:
+        gProjectFull = canonicalizePath(gProjectName)
+      except OSError:
+        gProjectFull = gProjectName
+      var p = splitFile(gProjectFull)
+      gProjectPath = p.dir
+      gProjectName = p.name
+    else:
+      gProjectPath = getCurrentDir()
+
+    # Find Nim's prefix dir.
+    let binaryPath = findExe("nim")
+    if binaryPath == "":
+      raise newException(IOError,
+          "Cannot find Nim standard library: Nim compiler not in PATH")
+    gPrefixDir = binaryPath.splitPath().head.parentDir()
+    #msgs.writelnHook = proc (line: string) = logStr(line)
+
+    loadConfigs(DefaultConfig, cache) # load all config files
+    # now process command line arguments again, because some options in the
+    # command line can overwite the config file's settings
+    options.command = "nimsuggest"
+    let scriptFile = gProjectFull.changeFileExt("nims")
+    if fileExists(scriptFile):
+      runNimScript(cache, scriptFile, freshDefines=false)
+      # 'nim foo.nims' means to just run the NimScript file and do nothing more:
+      if scriptFile == gProjectFull: return
+    elif fileExists(gProjectPath / "config.nims"):
+      # directory wide NimScript file
+      runNimScript(cache, gProjectPath / "config.nims", freshDefines=false)
+
+    extccomp.initVars()
+    processCmdLine(passCmd2, "")
+
+    mainCommand(cache)
+
+when false:
+  proc quitCalled() {.noconv.} =
+    writeStackTrace()
+
+  addQuitProc(quitCalled)
+
+condsyms.initDefines()
+defineSymbol "nimsuggest"
+handleCmdline(newIdentCache())

tools/nimsuggest/nimsuggest.nim.cfg

@@ -0,0 +1,16 @@
+# Special configuration file for the Nim project
+
+gc:markAndSweep
+
+hint[XDeclaredButNotUsed]:off
+
+path:"$lib/packages/docutils"
+
+define:useStdoutAsStdmsg
+define:nimsuggest
+
+#cs:partial
+#define:useNodeIds
+#define:booting
+#define:noDocgen
+--path:"$nim"

tools/nimsuggest/nimsuggest.nimble

@@ -0,0 +1,11 @@
+[Package]
+name          = "nimsuggest"
+version       = "0.1.0"
+author        = "Andreas Rumpf"
+description   = "Tool for providing auto completion data for Nim source code."
+license       = "MIT"
+
+bin = "nimsuggest"
+
+[Deps]
+Requires: "nim >= 0.11.2, compiler#head"

tools/nimsuggest/sexp.nim

@@ -0,0 +1,697 @@
+#
+#
+#            Nim's Runtime Library
+#        (c) Copyright 2015 Andreas Rumpf, Dominik Picheta
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+import
+  hashes, strutils, lexbase, streams, unicode, macros
+
+type
+  SexpEventKind* = enum  ## enumeration of all events that may occur when parsing
+    sexpError,           ## an error occurred during parsing
+    sexpEof,             ## end of file reached
+    sexpString,          ## a string literal
+    sexpSymbol,          ## a symbol
+    sexpInt,             ## an integer literal
+    sexpFloat,           ## a float literal
+    sexpNil,             ## the value ``nil``
+    sexpDot,             ## the dot to separate car/cdr
+    sexpListStart,       ## start of a list: the ``(`` token
+    sexpListEnd,         ## end of a list: the ``)`` token
+
+  TTokKind = enum        # must be synchronized with SexpEventKind!
+    tkError,
+    tkEof,
+    tkString,
+    tkSymbol,
+    tkInt,
+    tkFloat,
+    tkNil,
+    tkDot,
+    tkParensLe,
+    tkParensRi
+    tkSpace
+
+  SexpError* = enum        ## enumeration that lists all errors that can occur
+    errNone,               ## no error
+    errInvalidToken,       ## invalid token
+    errParensRiExpected,    ## ``)`` expected
+    errQuoteExpected,      ## ``"`` expected
+    errEofExpected,        ## EOF expected
+
+  SexpParser* = object of BaseLexer ## the parser object.
+    a: string
+    tok: TTokKind
+    kind: SexpEventKind
+    err: SexpError
+
+const
+  errorMessages: array[SexpError, string] = [
+    "no error",
+    "invalid token",
+    "')' expected",
+    "'\"' or \"'\" expected",
+    "EOF expected",
+  ]
+  tokToStr: array[TTokKind, string] = [
+    "invalid token",
+    "EOF",
+    "string literal",
+    "symbol",
+    "int literal",
+    "float literal",
+    "nil",
+    ".",
+    "(", ")", "space"
+  ]
+
+proc close*(my: var SexpParser) {.inline.} =
+  ## closes the parser `my` and its associated input stream.
+  lexbase.close(my)
+
+proc str*(my: SexpParser): string {.inline.} =
+  ## returns the character data for the events: ``sexpInt``, ``sexpFloat``,
+  ## ``sexpString``
+  assert(my.kind in {sexpInt, sexpFloat, sexpString})
+  result = my.a
+
+proc getInt*(my: SexpParser): BiggestInt {.inline.} =
+  ## returns the number for the event: ``sexpInt``
+  assert(my.kind == sexpInt)
+  result = parseBiggestInt(my.a)
+
+proc getFloat*(my: SexpParser): float {.inline.} =
+  ## returns the number for the event: ``sexpFloat``
+  assert(my.kind == sexpFloat)
+  result = parseFloat(my.a)
+
+proc kind*(my: SexpParser): SexpEventKind {.inline.} =
+  ## returns the current event type for the SEXP parser
+  result = my.kind
+
+proc getColumn*(my: SexpParser): int {.inline.} =
+  ## get the current column the parser has arrived at.
+  result = getColNumber(my, my.bufpos)
+
+proc getLine*(my: SexpParser): int {.inline.} =
+  ## get the current line the parser has arrived at.
+  result = my.lineNumber
+
+proc errorMsg*(my: SexpParser): string =
+  ## returns a helpful error message for the event ``sexpError``
+  assert(my.kind == sexpError)
+  result = "($1, $2) Error: $3" % [$getLine(my), $getColumn(my), errorMessages[my.err]]
+
+proc errorMsgExpected*(my: SexpParser, e: string): string =
+  ## returns an error message "`e` expected" in the same format as the
+  ## other error messages
+  result = "($1, $2) Error: $3" % [$getLine(my), $getColumn(my), e & " expected"]
+
+proc handleHexChar(c: char, x: var int): bool =
+  result = true # Success
+  case c
+  of '0'..'9': x = (x shl 4) or (ord(c) - ord('0'))
+  of 'a'..'f': x = (x shl 4) or (ord(c) - ord('a') + 10)
+  of 'A'..'F': x = (x shl 4) or (ord(c) - ord('A') + 10)
+  else: result = false # error
+
+proc parseString(my: var SexpParser): TTokKind =
+  result = tkString
+  var pos = my.bufpos + 1
+  var buf = my.buf
+  while true:
+    case buf[pos]
+    of '\0':
+      my.err = errQuoteExpected
+      result = tkError
+      break
+    of '"':
+      inc(pos)
+      break
+    of '\\':
+      case buf[pos+1]
+      of '\\', '"', '\'', '/':
+        add(my.a, buf[pos+1])
+        inc(pos, 2)
+      of 'b':
+        add(my.a, '\b')
+        inc(pos, 2)
+      of 'f':
+        add(my.a, '\f')
+        inc(pos, 2)
+      of 'n':
+        add(my.a, '\L')
+        inc(pos, 2)
+      of 'r':
+        add(my.a, '\C')
+        inc(pos, 2)
+      of 't':
+        add(my.a, '\t')
+        inc(pos, 2)
+      of 'u':
+        inc(pos, 2)
+        var r: int
+        if handleHexChar(buf[pos], r): inc(pos)
+        if handleHexChar(buf[pos], r): inc(pos)
+        if handleHexChar(buf[pos], r): inc(pos)
+        if handleHexChar(buf[pos], r): inc(pos)
+        add(my.a, toUTF8(Rune(r)))
+      else:
+        # don't bother with the error
+        add(my.a, buf[pos])
+        inc(pos)
+    of '\c':
+      pos = lexbase.handleCR(my, pos)
+      buf = my.buf
+      add(my.a, '\c')
+    of '\L':
+      pos = lexbase.handleLF(my, pos)
+      buf = my.buf
+      add(my.a, '\L')
+    else:
+      add(my.a, buf[pos])
+      inc(pos)
+  my.bufpos = pos # store back
+
+proc parseNumber(my: var SexpParser) =
+  var pos = my.bufpos
+  var buf = my.buf
+  if buf[pos] == '-':
+    add(my.a, '-')
+    inc(pos)
+  if buf[pos] == '.':
+    add(my.a, "0.")
+    inc(pos)
+  else:
+    while buf[pos] in Digits:
+      add(my.a, buf[pos])
+      inc(pos)
+    if buf[pos] == '.':
+      add(my.a, '.')
+      inc(pos)
+  # digits after the dot:
+  while buf[pos] in Digits:
+    add(my.a, buf[pos])
+    inc(pos)
+  if buf[pos] in {'E', 'e'}:
+    add(my.a, buf[pos])
+    inc(pos)
+    if buf[pos] in {'+', '-'}:
+      add(my.a, buf[pos])
+      inc(pos)
+    while buf[pos] in Digits:
+      add(my.a, buf[pos])
+      inc(pos)
+  my.bufpos = pos
+
+proc parseSymbol(my: var SexpParser) =
+  var pos = my.bufpos
+  var buf = my.buf
+  if buf[pos] in IdentStartChars:
+    while buf[pos] in IdentChars:
+      add(my.a, buf[pos])
+      inc(pos)
+  my.bufpos = pos
+
+proc getTok(my: var SexpParser): TTokKind =
+  setLen(my.a, 0)
+  case my.buf[my.bufpos]
+  of '-', '0'..'9': # numbers that start with a . are not parsed
+                    # correctly.
+    parseNumber(my)
+    if {'.', 'e', 'E'} in my.a:
+      result = tkFloat
+    else:
+      result = tkInt
+  of '"': #" # gotta fix nim-mode
+    result = parseString(my)
+  of '(':
+    inc(my.bufpos)
+    result = tkParensLe
+  of ')':
+    inc(my.bufpos)
+    result = tkParensRi
+  of '\0':
+    result = tkEof
+  of 'a'..'z', 'A'..'Z', '_':
+    parseSymbol(my)
+    if my.a == "nil":
+      result = tkNil
+    else:
+      result = tkSymbol
+  of ' ':
+    result = tkSpace
+    inc(my.bufpos)
+  of '.':
+    result = tkDot
+    inc(my.bufpos)
+  else:
+    inc(my.bufpos)
+    result = tkError
+  my.tok = result
+
+# ------------- higher level interface ---------------------------------------
+
+type
+  SexpNodeKind* = enum ## possible SEXP node types
+    SNil,
+    SInt,
+    SFloat,
+    SString,
+    SSymbol,
+    SList,
+    SCons
+
+  SexpNode* = ref SexpNodeObj ## SEXP node
+  SexpNodeObj* {.acyclic.} = object
+    case kind*: SexpNodeKind
+    of SString:
+      str*: string
+    of SSymbol:
+      symbol*: string
+    of SInt:
+      num*: BiggestInt
+    of SFloat:
+      fnum*: float
+    of SList:
+      elems*: seq[SexpNode]
+    of SCons:
+      car: SexpNode
+      cdr: SexpNode
+    of SNil:
+      discard
+
+  Cons = tuple[car: SexpNode, cdr: SexpNode]
+
+  SexpParsingError* = object of ValueError ## is raised for a SEXP error
+
+proc raiseParseErr*(p: SexpParser, msg: string) {.noinline, noreturn.} =
+  ## raises an `ESexpParsingError` exception.
+  raise newException(SexpParsingError, errorMsgExpected(p, msg))
+
+proc newSString*(s: string): SexpNode {.procvar.}=
+  ## Creates a new `SString SexpNode`.
+  new(result)
+  result.kind = SString
+  result.str = s
+
+proc newSStringMove(s: string): SexpNode =
+  new(result)
+  result.kind = SString
+  shallowCopy(result.str, s)
+
+proc newSInt*(n: BiggestInt): SexpNode {.procvar.} =
+  ## Creates a new `SInt SexpNode`.
+  new(result)
+  result.kind = SInt
+  result.num  = n
+
+proc newSFloat*(n: float): SexpNode {.procvar.} =
+  ## Creates a new `SFloat SexpNode`.
+  new(result)
+  result.kind = SFloat
+  result.fnum  = n
+
+proc newSNil*(): SexpNode {.procvar.} =
+  ## Creates a new `SNil SexpNode`.
+  new(result)
+
+proc newSCons*(car, cdr: SexpNode): SexpNode {.procvar.} =
+  ## Creates a new `SCons SexpNode`
+  new(result)
+  result.kind = SCons
+  result.car = car
+  result.cdr = cdr
+
+proc newSList*(): SexpNode {.procvar.} =
+  ## Creates a new `SList SexpNode`
+  new(result)
+  result.kind = SList
+  result.elems = @[]
+
+proc newSSymbol*(s: string): SexpNode {.procvar.} =
+  new(result)
+  result.kind = SSymbol
+  result.symbol = s
+
+proc newSSymbolMove(s: string): SexpNode =
+  new(result)
+  result.kind = SSymbol
+  shallowCopy(result.symbol, s)
+
+proc getStr*(n: SexpNode, default: string = ""): string =
+  ## Retrieves the string value of a `SString SexpNode`.
+  ##
+  ## Returns ``default`` if ``n`` is not a ``SString``.
+  if n.kind != SString: return default
+  else: return n.str
+
+proc getNum*(n: SexpNode, default: BiggestInt = 0): BiggestInt =
+  ## Retrieves the int value of a `SInt SexpNode`.
+  ##
+  ## Returns ``default`` if ``n`` is not a ``SInt``.
+  if n.kind != SInt: return default
+  else: return n.num
+
+proc getFNum*(n: SexpNode, default: float = 0.0): float =
+  ## Retrieves the float value of a `SFloat SexpNode`.
+  ##
+  ## Returns ``default`` if ``n`` is not a ``SFloat``.
+  if n.kind != SFloat: return default
+  else: return n.fnum
+
+proc getSymbol*(n: SexpNode, default: string = ""): string =
+  ## Retrieves the int value of a `SList SexpNode`.
+  ##
+  ## Returns ``default`` if ``n`` is not a ``SList``.
+  if n.kind != SSymbol: return default
+  else: return n.symbol
+
+proc getElems*(n: SexpNode, default: seq[SexpNode] = @[]): seq[SexpNode] =
+  ## Retrieves the int value of a `SList SexpNode`.
+  ##
+  ## Returns ``default`` if ``n`` is not a ``SList``.
+  if n.kind == SNil: return @[]
+  elif n.kind != SList: return default
+  else: return n.elems
+
+proc getCons*(n: SexpNode, defaults: Cons = (newSNil(), newSNil())): Cons =
+  ## Retrieves the cons value of a `SList SexpNode`.
+  ##
+  ## Returns ``default`` if ``n`` is not a ``SList``.
+  if n.kind == SCons: return (n.car, n.cdr)
+  elif n.kind == SList: return (n.elems[0], n.elems[1])
+  else: return defaults
+
+proc sexp*(s: string): SexpNode =
+  ## Generic constructor for SEXP data. Creates a new `SString SexpNode`.
+  new(result)
+  result.kind = SString
+  result.str = s
+
+proc sexp*(n: BiggestInt): SexpNode =
+  ## Generic constructor for SEXP data. Creates a new `SInt SexpNode`.
+  new(result)
+  result.kind = SInt
+  result.num  = n
+
+proc sexp*(n: float): SexpNode =
+  ## Generic constructor for SEXP data. Creates a new `SFloat SexpNode`.
+  new(result)
+  result.kind = SFloat
+  result.fnum  = n
+
+proc sexp*(b: bool): SexpNode =
+  ## Generic constructor for SEXP data. Creates a new `SSymbol
+  ## SexpNode` with value t or `SNil SexpNode`.
+  new(result)
+  if b:
+    result.kind = SSymbol
+    result.symbol = "t"
+  else:
+    result.kind = SNil
+
+proc sexp*(elements: openArray[SexpNode]): SexpNode =
+  ## Generic constructor for SEXP data. Creates a new `SList SexpNode`
+  new(result)
+  result.kind = SList
+  newSeq(result.elems, elements.len)
+  for i, p in pairs(elements): result.elems[i] = p
+
+proc sexp*(s: SexpNode): SexpNode =
+  result = s
+
+proc toSexp(x: NimNode): NimNode {.compiletime.} =
+  case x.kind
+  of nnkBracket:
+    result = newNimNode(nnkBracket)
+    for i in 0 .. <x.len:
+      result.add(toSexp(x[i]))
+
+  else:
+    result = x
+
+  result = prefix(result, "sexp")
+
+macro convertSexp*(x: untyped): untyped =
+  ## Convert an expression to a SexpNode directly, without having to specify
+  ## `%` for every element.
+  result = toSexp(x)
+
+proc `==`* (a,b: SexpNode): bool =
+  ## Check two nodes for equality
+  if a.isNil:
+    if b.isNil: return true
+    return false
+  elif b.isNil or a.kind != b.kind:
+    return false
+  else:
+    return case a.kind
+    of SString:
+      a.str == b.str
+    of SInt:
+      a.num == b.num
+    of SFloat:
+      a.fnum == b.fnum
+    of SNil:
+      true
+    of SList:
+      a.elems == b.elems
+    of SSymbol:
+      a.symbol == b.symbol
+    of SCons:
+      a.car == b.car and a.cdr == b.cdr
+
+proc hash* (n:SexpNode): Hash =
+  ## Compute the hash for a SEXP node
+  case n.kind
+  of SList:
+    result = hash(n.elems)
+  of SInt:
+    result = hash(n.num)
+  of SFloat:
+    result = hash(n.fnum)
+  of SString:
+    result = hash(n.str)
+  of SNil:
+    result = hash(0)
+  of SSymbol:
+    result = hash(n.symbol)
+  of SCons:
+    result = hash(n.car) !& hash(n.cdr)
+
+proc len*(n: SexpNode): int =
+  ## If `n` is a `SList`, it returns the number of elements.
+  ## If `n` is a `JObject`, it returns the number of pairs.
+  ## Else it returns 0.
+  case n.kind
+  of SList: result = n.elems.len
+  else: discard
+
+proc `[]`*(node: SexpNode, index: int): SexpNode =
+  ## Gets the node at `index` in a List. Result is undefined if `index`
+  ## is out of bounds
+  assert(not isNil(node))
+  assert(node.kind == SList)
+  return node.elems[index]
+
+proc add*(father, child: SexpNode) =
+  ## Adds `child` to a SList node `father`.
+  assert father.kind == SList
+  father.elems.add(child)
+
+# ------------- pretty printing ----------------------------------------------
+
+proc indent(s: var string, i: int) =
+  s.add(spaces(i))
+
+proc newIndent(curr, indent: int, ml: bool): int =
+  if ml: return curr + indent
+  else: return indent
+
+proc nl(s: var string, ml: bool) =
+  if ml: s.add("\n")
+
+proc escapeJson*(s: string): string =
+  ## Converts a string `s` to its JSON representation.
+  result = newStringOfCap(s.len + s.len shr 3)
+  result.add("\"")
+  for x in runes(s):
+    var r = int(x)
+    if r >= 32 and r <= 127:
+      var c = chr(r)
+      case c
+      of '"': result.add("\\\"") #" # gotta fix nim-mode
+      of '\\': result.add("\\\\")
+      else: result.add(c)
+    else:
+      result.add("\\u")
+      result.add(toHex(r, 4))
+  result.add("\"")
+
+proc copy*(p: SexpNode): SexpNode =
+  ## Performs a deep copy of `a`.
+  case p.kind
+  of SString:
+    result = newSString(p.str)
+  of SInt:
+    result = newSInt(p.num)
+  of SFloat:
+    result = newSFloat(p.fnum)
+  of SNil:
+    result = newSNil()
+  of SSymbol:
+    result = newSSymbol(p.symbol)
+  of SList:
+    result = newSList()
+    for i in items(p.elems):
+      result.elems.add(copy(i))
+  of SCons:
+    result = newSCons(copy(p.car), copy(p.cdr))
+
+proc toPretty(result: var string, node: SexpNode, indent = 2, ml = true,
+              lstArr = false, currIndent = 0) =
+  case node.kind
+  of SString:
+    if lstArr: result.indent(currIndent)
+    result.add(escapeJson(node.str))
+  of SInt:
+    if lstArr: result.indent(currIndent)
+    result.add($node.num)
+  of SFloat:
+    if lstArr: result.indent(currIndent)
+    result.add($node.fnum)
+  of SNil:
+    if lstArr: result.indent(currIndent)
+    result.add("nil")
+  of SSymbol:
+    if lstArr: result.indent(currIndent)
+    result.add($node.symbol)
+  of SList:
+    if lstArr: result.indent(currIndent)
+    if len(node.elems) != 0:
+      result.add("(")
+      result.nl(ml)
+      for i in 0..len(node.elems)-1:
+        if i > 0:
+          result.add(" ")
+          result.nl(ml) # New Line
+        toPretty(result, node.elems[i], indent, ml,
+            true, newIndent(currIndent, indent, ml))
+      result.nl(ml)
+      result.indent(currIndent)
+      result.add(")")
+    else: result.add("nil")
+  of SCons:
+    if lstArr: result.indent(currIndent)
+    result.add("(")
+    toPretty(result, node.car, indent, ml,
+        true, newIndent(currIndent, indent, ml))
+    result.add(" . ")
+    toPretty(result, node.cdr, indent, ml,
+        true, newIndent(currIndent, indent, ml))
+    result.add(")")
+
+proc pretty*(node: SexpNode, indent = 2): string =
+  ## Converts `node` to its Sexp Representation, with indentation and
+  ## on multiple lines.
+  result = ""
+  toPretty(result, node, indent)
+
+proc `$`*(node: SexpNode): string =
+  ## Converts `node` to its SEXP Representation on one line.
+  result = ""
+  toPretty(result, node, 0, false)
+
+iterator items*(node: SexpNode): SexpNode =
+  ## Iterator for the items of `node`. `node` has to be a SList.
+  assert node.kind == SList
+  for i in items(node.elems):
+    yield i
+
+iterator mitems*(node: var SexpNode): var SexpNode =
+  ## Iterator for the items of `node`. `node` has to be a SList. Items can be
+  ## modified.
+  assert node.kind == SList
+  for i in mitems(node.elems):
+    yield i
+
+proc eat(p: var SexpParser, tok: TTokKind) =
+  if p.tok == tok: discard getTok(p)
+  else: raiseParseErr(p, tokToStr[tok])
+
+proc parseSexp(p: var SexpParser): SexpNode =
+  ## Parses SEXP from a SEXP Parser `p`.
+  case p.tok
+  of tkString:
+    # we capture 'p.a' here, so we need to give it a fresh buffer afterwards:
+    result = newSStringMove(p.a)
+    p.a = ""
+    discard getTok(p)
+  of tkInt:
+    result = newSInt(parseBiggestInt(p.a))
+    discard getTok(p)
+  of tkFloat:
+    result = newSFloat(parseFloat(p.a))
+    discard getTok(p)
+  of tkNil:
+    result = newSNil()
+    discard getTok(p)
+  of tkSymbol:
+    result = newSSymbolMove(p.a)
+    p.a = ""
+    discard getTok(p)
+  of tkParensLe:
+    result = newSList()
+    discard getTok(p)
+    while p.tok notin {tkParensRi, tkDot}:
+      result.add(parseSexp(p))
+      if p.tok != tkSpace: break
+      discard getTok(p)
+    if p.tok == tkDot:
+      eat(p, tkDot)
+      eat(p, tkSpace)
+      result.add(parseSexp(p))
+      result = newSCons(result[0], result[1])
+    eat(p, tkParensRi)
+  of tkSpace, tkDot, tkError, tkParensRi, tkEof:
+    raiseParseErr(p, "(")
+
+proc open*(my: var SexpParser, input: Stream) =
+  ## initializes the parser with an input stream.
+  lexbase.open(my, input)
+  my.kind = sexpError
+  my.a = ""
+
+proc parseSexp*(s: Stream): SexpNode =
+  ## Parses from a buffer `s` into a `SexpNode`.
+  var p: SexpParser
+  p.open(s)
+  discard getTok(p) # read first token
+  result = p.parseSexp()
+  p.close()
+
+proc parseSexp*(buffer: string): SexpNode =
+  ## Parses Sexp from `buffer`.
+  result = parseSexp(newStringStream(buffer))
+
+when isMainModule:
+  let testSexp = parseSexp("""(1 (98 2) nil (2) foobar "foo" 9.234)""")
+  assert(testSexp[0].getNum == 1)
+  assert(testSexp[1][0].getNum == 98)
+  assert(testSexp[2].getElems == @[])
+  assert(testSexp[4].getSymbol == "foobar")
+  assert(testSexp[5].getStr == "foo")
+
+  let alist = parseSexp("""((1 . 2) (2 . "foo"))""")
+  assert(alist[0].getCons.car.getNum == 1)
+  assert(alist[0].getCons.cdr.getNum == 2)
+  assert(alist[1].getCons.cdr.getStr == "foo")
+
+  # Generator:
+  var j = convertSexp([true, false, "foobar", [1, 2, "baz"]])
+  assert($j == """(t nil "foobar" (1 2 "baz"))""")

tools/nimsuggest/tester.nim

@@ -0,0 +1,99 @@
+# Tester for nimsuggest.
+# Every test file can have a #[!]# comment that is deleted from the input
+# before 'nimsuggest' is invoked to ensure this token doesn't make a
+# crucial difference for Nim's parser.
+
+import os, osproc, strutils, streams, re
+
+type
+  Test = object
+    cmd: string
+    script: seq[(string, string)]
+
+const
+  curDir = when defined(windows): "" else: "./"
+  DummyEof = "!EOF!"
+
+proc parseTest(filename: string): Test =
+  const cursorMarker = "#[!]#"
+  let nimsug = curDir & addFileExt("nimsuggest", ExeExt)
+  let dest = getTempDir() / extractFilename(filename)
+  result.cmd = nimsug & " --tester " & dest
+  result.script = @[]
+  var tmp = open(dest, fmWrite)
+  var specSection = 0
+  var markers = newSeq[string]()
+  var i = 1
+  for x in lines(filename):
+    let marker = x.find(cursorMarker)+1
+    if marker > 0:
+      markers.add filename & ";" & dest & ":" & $i & ":" & $marker
+      tmp.writeLine x.replace(cursorMarker, "")
+    else:
+      tmp.writeLine x
+    if x.contains("""""""""):
+      inc specSection
+    elif specSection == 1:
+      if x.startsWith("$nimsuggest"):
+        result.cmd = x % ["nimsuggest", nimsug, "file", filename]
+      elif x.startsWith(">"):
+        # since 'markers' here are not complete yet, we do the $substitutions
+        # afterwards
+        result.script.add((x.substr(1), ""))
+      else:
+        # expected output line:
+        let x = x % ["file", filename]
+        result.script[^1][1].add x.replace(";;", "\t") & '\L'
+    inc i
+  tmp.close()
+  # now that we know the markers, substitute them:
+  for a in mitems(result.script):
+    a[0] = a[0] % markers
+
+proc smartCompare(pattern, x: string): bool =
+  if pattern.contains('*'):
+    result = match(x, re(escapeRe(pattern).replace("\\x2A","(.*)"), {}))
+
+proc runTest(filename: string): int =
+  let s = parseTest filename
+  let cl = parseCmdLine(s.cmd)
+  var p = startProcess(command=cl[0], args=cl[1 .. ^1],
+                       options={poStdErrToStdOut, poUsePath,
+                       poInteractive, poDemon})
+  let outp = p.outputStream
+  let inp = p.inputStream
+  var report = ""
+  var a = newStringOfCap(120)
+  try:
+    # read and ignore anything nimsuggest says at startup:
+    while outp.readLine(a):
+      if a == DummyEof: break
+    for req, resp in items(s.script):
+      inp.writeLine(req)
+      inp.flush()
+      var answer = ""
+      while outp.readLine(a):
+        if a == DummyEof: break
+        answer.add a
+        answer.add '\L'
+      if resp != answer and not smartCompare(resp, answer):
+        report.add "\nTest failed: " & filename
+        report.add "\n  Expected:  " & resp
+        report.add "\n  But got:   " & answer
+  finally:
+    inp.writeLine("quit")
+    inp.flush()
+    close(p)
+  if report.len > 0:
+    echo report
+  result = report.len
+
+proc main() =
+  var failures = 0
+  for x in walkFiles("tests/t*.nim"):
+    echo "Test ", x
+    failures += runTest(expandFilename(x))
+  if failures > 0:
+    quit 1
+
+main()

tools/nimsuggest/tests/tdef1.nim

@@ -0,0 +1,16 @@
+discard """
+$nimsuggest --tester $file
+>def $1
+def;;skProc;;tdef1.hello;;proc ();;$file;;9;;5;;"";;100
+>def $1
+def;;skProc;;tdef1.hello;;proc ();;$file;;9;;5;;"";;100
+"""
+
+proc hello() string =
+  ## Return hello
+  "Hello"
+
+hel#[!]#lo()
+
+# v uncompleted id for sug (13,2)
+he

tools/nimsuggest/tests/tdot1.nim

@@ -0,0 +1,14 @@
+discard """
+$nimsuggest --tester $file
+>sug $1
+sug;;skField;;x;;int;;$file;;11;;4;;"";;100
+sug;;skField;;y;;int;;$file;;11;;7;;"";;100
+sug;;skProc;;tdot1.main;;proc (f: Foo);;$file;;13;;5;;"";;100
+"""
+
+type
+  Foo = object
+    x, y: int
+
+proc main(f: Foo) =
+  f.#[!]#

tools/nimsuggest/tests/tstrutils.nim

@@ -0,0 +1,9 @@
+discard """
+$nimsuggest --tester ../nim/lib/pure/strutils.nim
+>def ../nim/lib/pure/strutils.nim:2300:6
+def;;skTemplate;;system.doAssert;;proc (cond: bool, msg: string): typed;;*/lib/system.nim;;*;;9;;"";;100
+"""
+
+# Line 2300 in strutils.nim is doAssert and this is unlikely to change
+# soon since there are a whole lot of doAsserts there.
+