Merge branch 'devel' into feature/async-streams

This commit is contained in:
Araq 2017-02-25 11:18:48 +01:00
commit 1961e444c3
115 changed files with 2208 additions and 877 deletions

View file

@ -209,6 +209,9 @@ proc main() =
of Manual:
echo "After download, move it to: ", dest
if askBool("Download successful? (y/n) "):
while not fileExists("dist" / mingw):
echo "could not find: ", "dist" / mingw
if not askBool("Try again? (y/n) "): break
if unzip(): retry = true
of Failure: discard
of Success:

View file

@ -13,11 +13,11 @@ import strutils, os, parseopt, parseutils, sequtils, net, rdstdin, sexp
# Do NOT import suggest. It will lead to wierd bugs with
# suggestionResultHook, because suggest.nim is included by sigmatch.
# So we import that one instead.
import compiler/options, compiler/commands, compiler/modules, compiler/sem,
compiler/passes, compiler/passaux, compiler/msgs, compiler/nimconf,
compiler/extccomp, compiler/condsyms, compiler/lists,
compiler/sigmatch, compiler/ast, compiler/scriptconfig,
compiler/idents, compiler/modulegraphs
import compiler / [options, commands, modules, sem,
passes, passaux, msgs, nimconf,
extccomp, condsyms,
sigmatch, ast, scriptconfig,
idents, modulegraphs, compilerlog, vm]
when defined(windows):
import winlean
@ -40,6 +40,7 @@ Options:
--log enable verbose logging to nimsuggest.log file
--v2 use version 2 of the protocol; more features and
much faster
--refresh perform automatic refreshes to keep the analysis precise
--tester implies --v2 and --stdin and outputs a line
'""" & DummyEof & """' for the tester
@ -57,10 +58,11 @@ var
gMode: Mode
gEmitEof: bool # whether we write '!EOF!' dummy lines
gLogging = false
gRefresh: bool
const
seps = {':', ';', ' ', '\t'}
Help = "usage: sug|con|def|use|dus|chk|mod|highlight|outline file.nim[;dirtyfile.nim]:line:col\n" &
Help = "usage: sug|con|def|use|dus|chk|mod|highlight|outline|known file.nim[;dirtyfile.nim]:line:col\n" &
"type 'quit' to quit\n" &
"type 'debug' to toggle debug mode on/off\n" &
"type 'terse' to toggle terse mode on/off"
@ -68,12 +70,6 @@ const
type
EUnexpectedCommand = object of Exception
proc logStr(line: string) =
var f: File
if open(f, getHomeDir() / "nimsuggest.log", fmAppend):
f.writeLine(line)
f.close()
proc parseQuoted(cmd: string; outp: var string; start: int): int =
var i = start
i += skipWhitespace(cmd, i)
@ -150,7 +146,7 @@ proc execute(cmd: IdeCmd, file, dirtyfile: string, line, col: int;
gTrackPos = newLineInfo(dirtyIdx, line, col)
gErrorCounter = 0
if suggestVersion < 2:
usageSym = nil
graph.usageSym = nil
if not isKnownFile:
graph.compileProject(cache)
if suggestVersion == 2 and gIdeCmd in {ideUse, ideDus} and
@ -163,7 +159,7 @@ proc execute(cmd: IdeCmd, file, dirtyfile: string, line, col: int;
if gIdeCmd != ideMod:
graph.compileProject(cache, modIdx)
if gIdeCmd in {ideUse, ideDus}:
let u = if suggestVersion >= 2: graph.symFromInfo(gTrackPos) else: usageSym
let u = if suggestVersion >= 2: graph.symFromInfo(gTrackPos) else: graph.usageSym
if u != nil:
listUsages(u)
else:
@ -180,7 +176,7 @@ proc executeEpc(cmd: IdeCmd, args: SexpNode;
dirtyfile = args[3].getStr(nil)
execute(cmd, file, dirtyfile, int(line), int(column), graph, cache)
proc returnEpc(socket: var Socket, uid: BiggestInt, s: SexpNode|string,
proc returnEpc(socket: Socket, uid: BiggestInt, s: SexpNode|string,
return_symbol = "return") =
let response = $convertSexp([newSSymbol(return_symbol), uid, s])
socket.send(toHex(len(response), 6))
@ -198,7 +194,7 @@ template sendEpc(results: typed, tdef, hook: untyped) =
let res = sexp(results)
if gLogging:
logStr($res)
returnEPC(client, uid, res)
returnEpc(client, uid, res)
template checkSanity(client, sizeHex, size, messageBuffer: typed) =
if client.recv(sizeHex, 6) != 6:
@ -208,6 +204,48 @@ template checkSanity(client, sizeHex, size, messageBuffer: typed) =
if client.recv(messageBuffer, size) != size:
raise newException(ValueError, "didn't get all the bytes")
var
requests: Channel[string]
results: Channel[Suggest]
proc toStdout() {.gcsafe.} =
while true:
let res = results.recv()
case res.section
of ideNone: break
of ideChk: echo res.doc
of ideKnown: echo res.quality == 1
else: echo res
proc toSocket(stdoutSocket: Socket) {.gcsafe.} =
while true:
let res = results.recv()
case res.section
of ideNone: break
of ideChk: stdoutSocket.send(res.doc & "\c\L")
of ideKnown: stdoutSocket.send($(res.quality == 1) & "\c\L")
else: stdoutSocket.send($res & "\c\L")
proc toEpc(client: Socket; uid: BiggestInt) {.gcsafe.} =
var list = newSList()
while true:
let res = results.recv()
case res.section
of ideNone: break
of ideChk:
list.add sexp(res.doc)
of ideKnown:
list.add sexp(res.quality == 1)
else:
list.add sexp(res)
returnEpc(client, uid, list)
proc writelnToChannel(line: string) =
results.send(Suggest(section: ideChk, doc: line))
proc sugResultHook(s: Suggest) =
results.send(s)
template setVerbosity(level: typed) =
gVerbosity = level
gNotes = NotesVerbosity[gVerbosity]
@ -217,16 +255,128 @@ proc connectToNextFreePort(server: Socket, host: string): Port =
let (_, port) = server.getLocalAddr
result = port
proc parseCmdLine(cmd: string; graph: ModuleGraph; cache: IdentCache) =
type
ThreadParams = tuple[port: Port; address: string]
proc replStdin(x: ThreadParams) {.thread.} =
if gEmitEof:
echo DummyEof
while true:
let line = readLine(stdin)
requests.send line
toStdout()
echo DummyEof
flushFile(stdout)
else:
echo Help
var line = ""
while readLineFromStdin("> ", line):
requests.send line
toStdout()
echo ""
flushFile(stdout)
proc replTcp(x: ThreadParams) {.thread.} =
var server = newSocket()
server.bindAddr(x.port, x.address)
var inp = "".TaintedString
server.listen()
while true:
var stdoutSocket = newSocket()
accept(server, stdoutSocket)
stdoutSocket.readLine(inp)
requests.send inp
toSocket(stdoutSocket)
stdoutSocket.send("\c\L")
stdoutSocket.close()
proc argsToStr(x: SexpNode): string =
if x.kind != SList: return x.getStr
doAssert x.kind == SList
doAssert x.len >= 4
let file = x[0].getStr
let line = x[1].getNum
let col = x[2].getNum
let dirty = x[3].getStr
result = x[0].getStr.escape
if dirty.len > 0:
result.add ';'
result.add dirty.escape
result.add ':'
result.add line
result.add ':'
result.add col
proc replEpc(x: ThreadParams) {.thread.} =
var server = newSocket()
let port = connectToNextFreePort(server, "localhost")
server.listen()
echo port
var client = newSocket()
# Wait for connection
accept(server, client)
while true:
var
sizeHex = ""
size = 0
messageBuffer = ""
checkSanity(client, sizeHex, size, messageBuffer)
let
message = parseSexp($messageBuffer)
epcApi = message[0].getSymbol
case epcApi
of "call":
let
uid = message[1].getNum
args = message[3]
gIdeCmd = parseIdeCmd(message[2].getSymbol)
case gIdeCmd
of ideChk:
setVerbosity(1)
# Use full path because other emacs plugins depends it
gListFullPaths = true
incl(gGlobalOptions, optIdeDebug)
of ideSug, ideCon, ideDef, ideUse, ideDus, ideOutline, ideHighlight:
setVerbosity(0)
else: discard
let cmd = $gIdeCmd & " " & args.argsToStr
if gLogging:
logStr "MSG CMD: " & cmd
requests.send(cmd)
toEpc(client, uid)
of "methods":
returnEpc(client, message[1].getNum, listEpc())
of "epc-error":
# an unhandled exception forces down the whole process anyway, so we
# use 'quit' here instead of 'raise'
quit("recieved epc error: " & $messageBuffer)
else:
let errMessage = case epcApi
of "return", "return-error":
"no return expected"
else:
"unexpected call: " & epcAPI
quit errMessage
proc execCmd(cmd: string; graph: ModuleGraph; cache: IdentCache) =
template sentinel() =
# send sentinel for the input reading thread:
results.send(Suggest(section: ideNone))
template toggle(sw) =
if sw in gGlobalOptions:
excl(gGlobalOptions, sw)
else:
incl(gGlobalOptions, sw)
sentinel()
return
template err() =
echo Help
sentinel()
return
var opc = ""
@ -246,6 +396,7 @@ proc parseCmdLine(cmd: string; graph: ModuleGraph; cache: IdentCache) =
of "quit": quit()
of "debug": toggle optIdeDebug
of "terse": toggle optIdeTerse
of "known": gIdeCmd = ideKnown
else: err()
var dirtyfile = ""
var orig = ""
@ -259,25 +410,74 @@ proc parseCmdLine(cmd: string; graph: ModuleGraph; cache: IdentCache) =
i += skipWhile(cmd, seps, i)
i += parseInt(cmd, col, i)
execute(gIdeCmd, orig, dirtyfile, line, col-1, graph, cache)
if gIdeCmd == ideKnown:
results.send(Suggest(section: ideKnown, quality: ord(fileInfoKnown(orig))))
else:
execute(gIdeCmd, orig, dirtyfile, line, col-1, graph, cache)
sentinel()
proc serveStdin(graph: ModuleGraph; cache: IdentCache) =
proc recompileFullProject(graph: ModuleGraph; cache: IdentCache) =
echo "recompiling full project"
resetSystemArtifacts()
vm.globalCtx = nil
graph.resetAllModules()
GC_fullcollect()
compileProject(graph, cache)
echo GC_getStatistics()
proc mainThread(graph: ModuleGraph; cache: IdentCache) =
if gLogging:
for it in searchPaths:
logStr(it)
proc wrHook(line: string) {.closure.} =
if gMode == mepc:
if gLogging: logStr(line)
else:
writelnToChannel(line)
msgs.writelnHook = wrHook
suggestionResultHook = sugResultHook
graph.doStopCompile = proc (): bool = requests.peek() > 0
var idle = 0
while true:
let (hasData, req) = requests.tryRecv()
if hasData:
msgs.writelnHook = wrHook
suggestionResultHook = sugResultHook
execCmd(req, graph, cache)
idle = 0
else:
os.sleep 250
idle += 1
if idle == 20 and gRefresh:
# we use some nimsuggest activity to enable a lazy recompile:
gIdeCmd = ideChk
msgs.writelnHook = proc (s: string) = discard
suggestionResultHook = proc (s: Suggest) = discard
recompileFullProject(graph, cache)
var
inputThread: Thread[ThreadParams]
proc serveStdin(graph: ModuleGraph; cache: IdentCache) {.deprecated.} =
if gEmitEof:
echo DummyEof
while true:
let line = readLine(stdin)
parseCmdLine line, graph, cache
execCmd line, graph, cache
echo DummyEof
flushFile(stdout)
else:
echo Help
var line = ""
while readLineFromStdin("> ", line):
parseCmdLine line, graph, cache
execCmd line, graph, cache
echo ""
flushFile(stdout)
proc serveTcp(graph: ModuleGraph; cache: IdentCache) =
proc serveTcp(graph: ModuleGraph; cache: IdentCache) {.deprecated.} =
var server = newSocket()
server.bindAddr(gPort, gAddress)
var inp = "".TaintedString
@ -291,20 +491,18 @@ proc serveTcp(graph: ModuleGraph; cache: IdentCache) =
accept(server, stdoutSocket)
stdoutSocket.readLine(inp)
parseCmdLine inp.string, graph, cache
execCmd inp.string, graph, cache
stdoutSocket.send("\c\L")
stdoutSocket.close()
proc serveEpc(server: Socket; graph: ModuleGraph; cache: IdentCache) =
proc serveEpc(server: Socket; graph: ModuleGraph; cache: IdentCache) {.deprecated.} =
var client = newSocket()
# Wait for connection
accept(server, client)
if gLogging:
var it = searchPaths.head
while it != nil:
logStr(PStrEntry(it).data)
it = it.next
for it in searchPaths:
logStr(it)
msgs.writelnHook = proc (line: string) = logStr(line)
while true:
@ -357,33 +555,49 @@ proc mainCommand(graph: ModuleGraph; cache: IdentCache) =
incl gGlobalOptions, optCaasEnabled
isServing = true
wantMainModule()
appendStr(searchPaths, options.libpath)
add(searchPaths, options.libpath)
#if gProjectFull.len != 0:
# current path is always looked first for modules
# prependStr(searchPaths, gProjectPath)
# do not stop after the first error:
msgs.gErrorMax = high(int)
# compile the project before showing any input so that we already
# can answer questions right away:
compileProject(graph, cache)
open(requests)
open(results)
case gMode
of mstdin:
compileProject(graph, cache)
#modules.gFuzzyGraphChecking = false
serveStdin(graph, cache)
of mtcp:
# until somebody accepted the connection, produce no output (logging is too
# slow for big projects):
msgs.writelnHook = proc (msg: string) = discard
compileProject(graph, cache)
#modules.gFuzzyGraphChecking = false
serveTcp(graph, cache)
of mepc:
var server = newSocket()
let port = connectToNextFreePort(server, "localhost")
server.listen()
echo port
compileProject(graph, cache)
serveEpc(server, graph, cache)
of mstdin: createThread(inputThread, replStdin, (gPort, gAddress))
of mtcp: createThread(inputThread, replTcp, (gPort, gAddress))
of mepc: createThread(inputThread, replEpc, (gPort, gAddress))
mainThread(graph, cache)
joinThread(inputThread)
close(requests)
close(results)
when false:
case gMode
of mstdin:
compileProject(graph, cache)
#modules.gFuzzyGraphChecking = false
serveStdin(graph, cache)
of mtcp:
# until somebody accepted the connection, produce no output (logging is too
# slow for big projects):
msgs.writelnHook = proc (msg: string) = discard
compileProject(graph, cache)
#modules.gFuzzyGraphChecking = false
serveTcp(graph, cache)
of mepc:
var server = newSocket()
let port = connectToNextFreePort(server, "localhost")
server.listen()
echo port
compileProject(graph, cache)
serveEpc(server, graph, cache)
proc processCmdLine*(pass: TCmdLinePass, cmd: string) =
var p = parseopt.initOptParser(cmd)
@ -403,16 +617,14 @@ proc processCmdLine*(pass: TCmdLinePass, cmd: string) =
of "epc":
gMode = mepc
gVerbosity = 0 # Port number gotta be first.
of "debug":
incl(gGlobalOptions, optIdeDebug)
of "v2":
suggestVersion = 2
of "debug": incl(gGlobalOptions, optIdeDebug)
of "v2": suggestVersion = 2
of "tester":
suggestVersion = 2
gMode = mstdin
gEmitEof = true
of "log":
gLogging = true
of "log": gLogging = true
of "refresh": gRefresh = true
else: processSwitch(pass, p)
of cmdArgument:
options.gProjectName = unixToNativePath(p.key)

View file

@ -8,9 +8,18 @@ path:"$lib/packages/docutils"
define:useStdoutAsStdmsg
define:nimsuggest
# die when nimsuggest uses more than 4GB:
@if cpu32:
define:"nimMaxHeap=2000"
@else:
define:"nimMaxHeap=4000"
@end
#cs:partial
#define:useNodeIds
#define:booting
#define:noDocgen
--path:"$nim"
--threads:on
--noNimblePath
--path:"../../compiler"

View file

@ -561,16 +561,16 @@ proc toPretty(result: var string, node: SexpNode, indent = 2, ml = true,
result.add(escapeJson(node.str))
of SInt:
if lstArr: result.indent(currIndent)
result.add($node.num)
result.add(node.num)
of SFloat:
if lstArr: result.indent(currIndent)
result.add($node.fnum)
result.add(node.fnum)
of SNil:
if lstArr: result.indent(currIndent)
result.add("nil")
of SSymbol:
if lstArr: result.indent(currIndent)
result.add($node.symbol)
result.add(node.symbol)
of SList:
if lstArr: result.indent(currIndent)
if len(node.elems) != 0:

View file

@ -3,7 +3,7 @@
# before 'nimsuggest' is invoked to ensure this token doesn't make a
# crucial difference for Nim's parser.
import os, osproc, strutils, streams, re
import os, osproc, strutils, streams, re, sexp, net
type
Test = object
@ -17,7 +17,7 @@ const
template tpath(): untyped = getAppDir() / "tests"
proc parseTest(filename: string): Test =
proc parseTest(filename: string; epcMode=false): Test =
const cursorMarker = "#[!]#"
let nimsug = curDir & addFileExt("nimsuggest", ExeExt)
result.dest = getTempDir() / extractFilename(filename)
@ -31,7 +31,10 @@ proc parseTest(filename: string): Test =
for x in lines(filename):
let marker = x.find(cursorMarker)+1
if marker > 0:
markers.add "\"" & filename & "\";\"" & result.dest & "\":" & $i & ":" & $marker
if epcMode:
markers.add "(\"" & filename & "\" " & $i & " " & $marker & " \"" & result.dest & "\")"
else:
markers.add "\"" & filename & "\";\"" & result.dest & "\":" & $i & ":" & $marker
tmp.writeLine x.replace(cursorMarker, "")
else:
tmp.writeLine x
@ -133,6 +136,126 @@ proc smartCompare(pattern, x: string): bool =
if pattern.contains('*'):
result = match(x, re(escapeRe(pattern).replace("\\x2A","(.*)"), {}))
proc sendEpcStr(socket: Socket; cmd: string) =
let s = cmd.find(' ')
doAssert s > 0
var args = cmd.substr(s+1)
if not args.startsWith("("): args = escapeJson(args)
let c = "(call 567 " & cmd.substr(0, s) & args & ")"
socket.send toHex(c.len, 6)
socket.send c
proc recvEpc(socket: Socket): string =
var L = newStringOfCap(6)
if socket.recv(L, 6) != 6:
raise newException(ValueError, "recv A failed")
let x = parseHexInt(L)
result = newString(x)
if socket.recv(result, x) != x:
raise newException(ValueError, "recv B failed")
proc sexpToAnswer(s: SexpNode): string =
result = ""
doAssert s.kind == SList
doAssert s.len >= 3
let m = s[2]
if m.kind != SList:
echo s
doAssert m.kind == SList
for a in m:
doAssert a.kind == SList
var first = true
#s.section,
#s.symkind,
#s.qualifiedPath.map(newSString),
#s.filePath,
#s.forth,
#s.line,
#s.column,
#s.doc
if a.len >= 8:
let section = a[0].getStr
let symk = a[1].getStr
let qp = a[2]
let file = a[3].getStr
let typ = a[4].getStr
let line = a[5].getNum
let col = a[6].getNum
let doc = a[7].getStr.escape
result.add section
result.add '\t'
result.add symk
result.add '\t'
var i = 0
for aa in qp:
if i > 0: result.add '.'
result.add aa.getStr
inc i
result.add '\t'
result.add typ
result.add '\t'
result.add file
result.add '\t'
result.add line
result.add '\t'
result.add col
result.add '\t'
result.add doc
result.add '\t'
# for now Nim EPC does not return the quality
result.add "100"
result.add '\L'
proc doReport(filename, answer, resp: string; report: var string) =
if resp != answer and not smartCompare(resp, answer):
report.add "\nTest failed: " & filename
var hasDiff = false
for i in 0..min(resp.len-1, answer.len-1):
if resp[i] != answer[i]:
report.add "\n Expected: " & resp.substr(i)
report.add "\n But got: " & answer.substr(i)
hasDiff = true
break
if not hasDiff:
report.add "\n Expected: " & resp
report.add "\n But got: " & answer
proc runEpcTest(filename: string): int =
let s = parseTest(filename, true)
for cmd in s.startup:
if not runCmd(cmd, s.dest):
quit "invalid command: " & cmd
let epccmd = s.cmd.replace("--tester", "--epc --v2")
let cl = parseCmdLine(epccmd)
var p = startProcess(command=cl[0], args=cl[1 .. ^1],
options={poStdErrToStdOut, poUsePath,
poInteractive, poDemon})
let outp = p.outputStream
let inp = p.inputStream
var report = ""
var a = newStringOfCap(120)
try:
# read the port number:
if outp.readLine(a):
let port = parseInt(a)
var socket = newSocket()
socket.connect("localhost", Port(port))
for req, resp in items(s.script):
if not runCmd(req, s.dest):
socket.sendEpcStr(req)
let sx = parseSexp(socket.recvEpc())
if not req.startsWith("mod "):
let answer = sexpToAnswer(sx)
doReport(filename, answer, resp, report)
else:
raise newException(ValueError, "cannot read port number")
finally:
close(p)
if report.len > 0:
echo "==== EPC ========================================"
echo report
result = report.len
proc runTest(filename: string): int =
let s = parseTest filename
for cmd in s.startup:
@ -159,31 +282,28 @@ proc runTest(filename: string): int =
if a == DummyEof: break
answer.add a
answer.add '\L'
if resp != answer and not smartCompare(resp, answer):
report.add "\nTest failed: " & filename
var hasDiff = false
for i in 0..min(resp.len-1, answer.len-1):
if resp[i] != answer[i]:
report.add "\n Expected: " & resp.substr(i)
report.add "\n But got: " & answer.substr(i)
hasDiff = true
break
if not hasDiff:
report.add "\n Expected: " & resp
report.add "\n But got: " & answer
doReport(filename, answer, resp, report)
finally:
inp.writeLine("quit")
inp.flush()
close(p)
if report.len > 0:
echo "==== STDIN ======================================"
echo report
result = report.len
proc main() =
var failures = 0
for x in walkFiles(getAppDir() / "tests/t*.nim"):
echo "Test ", x
failures += runTest(expandFilename(x))
when false:
let x = getAppDir() / "tests/twithin_macro.nim"
let xx = expandFilename x
failures += runEpcTest(xx)
else:
for x in walkFiles(getAppDir() / "tests/t*.nim"):
echo "Test ", x
let xx = expandFilename x
failures += runTest(xx)
failures += runEpcTest(xx)
if failures > 0:
quit 1

View file

@ -206,8 +206,8 @@ $nimsuggest --tester $file
>sug $1
sug;;skField;;name;;string;;$file;;166;;6;;"";;100
sug;;skField;;age;;int;;$file;;167;;6;;"";;100
sug;;skMethod;;twithin_macro.age_human_yrs;;proc (self: Animal): int{.noSideEffect, gcsafe, locks: 0.};;$file;;169;;9;;"";;100
sug;;skMethod;;twithin_macro.age_human_yrs;;proc (self: Animal): int;;$file;;169;;9;;"";;100
sug;;skMacro;;twithin_macro.class;;proc (head: untyped, body: untyped): untyped{.gcsafe, locks: <unknown>.};;$file;;4;;6;;"Iterates over the children of the NimNode ``n``.";;100
sug;;skMethod;;twithin_macro.vocalize;;proc (self: Animal): string{.noSideEffect, gcsafe, locks: 0.};;$file;;168;;9;;"";;100
sug;;skMethod;;twithin_macro.vocalize;;proc (self: Rabbit): string{.noSideEffect, gcsafe, locks: 0.};;$file;;184;;9;;"";;100*
sug;;skMethod;;twithin_macro.vocalize;;proc (self: Animal): string;;$file;;168;;9;;"";;100
sug;;skMethod;;twithin_macro.vocalize;;proc (self: Rabbit): string;;$file;;184;;9;;"";;100*
"""