From 15c347af693250e60cdf564606fa1f90722b757a Mon Sep 17 00:00:00 2001 From: Simon Hafner Date: Tue, 14 Apr 2015 01:58:34 -0500 Subject: [PATCH 01/29] sexp parsing & generation --- lib/pure/sexpressions.nim | 716 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 716 insertions(+) create mode 100644 lib/pure/sexpressions.nim diff --git a/lib/pure/sexpressions.nim b/lib/pure/sexpressions.nim new file mode 100644 index 000000000..c30f35c25 --- /dev/null +++ b/lib/pure/sexpressions.nim @@ -0,0 +1,716 @@ +# +# +# Nim's Runtime Library +# (c) Copyright 2015 Andreas Rumpf, Dominik Picheta +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +import + hashes, strutils, lexbase, streams, unicode, macros + +type + SexpEventKind* = enum ## enumeration of all events that may occur when parsing + sexpError, ## an error occurred during parsing + sexpEof, ## end of file reached + sexpString, ## a string literal + sexpSymbol, ## a symbol + sexpInt, ## an integer literal + sexpFloat, ## a float literal + sexpNil, ## the value ``nil`` + sexpListStart, ## start of a list: the ``(`` token + sexpListEnd, ## end of a list: the ``)`` token + + TTokKind = enum # must be synchronized with SexpEventKind! + tkError, + tkEof, + tkString, + tkSymbol, + tkInt, + tkFloat, + tkNil, + tkParensLe, + tkParensRi + tkSpace + + SexpError* = enum ## enumeration that lists all errors that can occur + errNone, ## no error + errInvalidToken, ## invalid token + errParensRiExpected, ## ``)`` expected + errQuoteExpected, ## ``"`` expected + errEofExpected, ## EOF expected + + ParserState = enum + stateEof, stateStart, stateList, stateListExpectsSpace + + SexpParser* = object of BaseLexer ## the parser object. + a: string + tok: TTokKind + kind: SexpEventKind + err: SexpError + state: seq[ParserState] + filename: string + +const + errorMessages: array [SexpError, string] = [ + "no error", + "invalid token", + "')' expected", + "'\"' or \"'\" expected", + "EOF expected", + ] + tokToStr: array [TTokKind, string] = [ + "invalid token", + "EOF", + "string literal", + "symbol", + "int literal", + "float literal", + "nil", + "(", ")", " " + ] + +proc open*(my: var SexpParser, input: Stream, filename: string) = + ## initializes the parser with an input stream. `Filename` is only used + ## for nice error messages. + lexbase.open(my, input) + my.filename = filename + my.state = @[stateStart] + my.kind = sexpError + my.a = "" + +proc close*(my: var SexpParser) {.inline.} = + ## closes the parser `my` and its associated input stream. + lexbase.close(my) + +proc str*(my: SexpParser): string {.inline.} = + ## returns the character data for the events: ``sexpInt``, ``sexpFloat``, + ## ``sexpString`` + assert(my.kind in {sexpInt, sexpFloat, sexpString}) + result = my.a + +proc getInt*(my: SexpParser): BiggestInt {.inline.} = + ## returns the number for the event: ``sexpInt`` + assert(my.kind == sexpInt) + result = parseBiggestInt(my.a) + +proc getFloat*(my: SexpParser): float {.inline.} = + ## returns the number for the event: ``sexpFloat`` + assert(my.kind == sexpFloat) + result = parseFloat(my.a) + +proc kind*(my: SexpParser): SexpEventKind {.inline.} = + ## returns the current event type for the SEXP parser + result = my.kind + +proc getColumn*(my: SexpParser): int {.inline.} = + ## get the current column the parser has arrived at. + result = getColNumber(my, my.bufpos) + +proc getLine*(my: SexpParser): int {.inline.} = + ## get the current line the parser has arrived at. + result = my.lineNumber + +proc getFilename*(my: SexpParser): string {.inline.} = + ## get the filename of the file that the parser processes. + result = my.filename + +proc errorMsg*(my: SexpParser): string = + ## returns a helpful error message for the event ``sexpError`` + assert(my.kind == sexpError) + result = "$1($2, $3) Error: $4" % [ + my.filename, $getLine(my), $getColumn(my), errorMessages[my.err]] + +proc errorMsgExpected*(my: SexpParser, e: string): string = + ## returns an error message "`e` expected" in the same format as the + ## other error messages + result = "$1($2, $3) Error: $4" % [ + my.filename, $getLine(my), $getColumn(my), e & " expected"] + +proc handleHexChar(c: char, x: var int): bool = + result = true # Success + case c + of '0'..'9': x = (x shl 4) or (ord(c) - ord('0')) + of 'a'..'f': x = (x shl 4) or (ord(c) - ord('a') + 10) + of 'A'..'F': x = (x shl 4) or (ord(c) - ord('A') + 10) + else: result = false # error + +proc parseString(my: var SexpParser): TTokKind = + result = tkString + var pos = my.bufpos + 1 + var buf = my.buf + while true: + case buf[pos] + of '\0': + my.err = errQuoteExpected + result = tkError + break + of '"': + inc(pos) + break + of '\\': + case buf[pos+1] + of '\\', '"', '\'', '/': + add(my.a, buf[pos+1]) + inc(pos, 2) + of 'b': + add(my.a, '\b') + inc(pos, 2) + of 'f': + add(my.a, '\f') + inc(pos, 2) + of 'n': + add(my.a, '\L') + inc(pos, 2) + of 'r': + add(my.a, '\C') + inc(pos, 2) + of 't': + add(my.a, '\t') + inc(pos, 2) + of 'u': + inc(pos, 2) + var r: int + if handleHexChar(buf[pos], r): inc(pos) + if handleHexChar(buf[pos], r): inc(pos) + if handleHexChar(buf[pos], r): inc(pos) + if handleHexChar(buf[pos], r): inc(pos) + add(my.a, toUTF8(Rune(r))) + else: + # don't bother with the error + add(my.a, buf[pos]) + inc(pos) + of '\c': + pos = lexbase.handleCR(my, pos) + buf = my.buf + add(my.a, '\c') + of '\L': + pos = lexbase.handleLF(my, pos) + buf = my.buf + add(my.a, '\L') + else: + add(my.a, buf[pos]) + inc(pos) + my.bufpos = pos # store back + +proc parseNumber(my: var SexpParser) = + var pos = my.bufpos + var buf = my.buf + if buf[pos] == '-': + add(my.a, '-') + inc(pos) + if buf[pos] == '.': + add(my.a, "0.") + inc(pos) + else: + while buf[pos] in Digits: + add(my.a, buf[pos]) + inc(pos) + if buf[pos] == '.': + add(my.a, '.') + inc(pos) + # digits after the dot: + while buf[pos] in Digits: + add(my.a, buf[pos]) + inc(pos) + if buf[pos] in {'E', 'e'}: + add(my.a, buf[pos]) + inc(pos) + if buf[pos] in {'+', '-'}: + add(my.a, buf[pos]) + inc(pos) + while buf[pos] in Digits: + add(my.a, buf[pos]) + inc(pos) + my.bufpos = pos + +proc parseSymbol(my: var SexpParser) = + var pos = my.bufpos + var buf = my.buf + if buf[pos] in IdentStartChars: + while buf[pos] in IdentChars: + add(my.a, buf[pos]) + inc(pos) + my.bufpos = pos + +proc getTok(my: var SexpParser): TTokKind = + setLen(my.a, 0) + case my.buf[my.bufpos] + of '-', '.', '0'..'9': + parseNumber(my) + if {'.', 'e', 'E'} in my.a: + result = tkFloat + else: + result = tkInt + of '"': #" # gotta fix nim-mode + result = parseString(my) + of '(': + inc(my.bufpos) + result = tkParensLe + of ')': + inc(my.bufpos) + result = tkParensRi + of '\0': + result = tkEof + of 'a'..'z', 'A'..'Z', '_': + parseSymbol(my) + if my.a == "nil": + result = tkNil + else: + result = tkSymbol + of ' ': + result = tkSpace + inc(my.bufpos) + else: + inc(my.bufpos) + result = tkError + my.tok = result + +proc next*(my: var SexpParser) = + ## retrieves the first/next event. This controls the parser. + var tk = getTok(my) + var i = my.state.len-1 + # the following code is a state machine. If we had proper coroutines, + # the code could be much simpler. + case my.state[i] + of stateEof: + if tk == tkEof: + my.kind = sexpEof + else: + my.kind = sexpError + my.err = errEofExpected + of stateStart: + # tokens allowed? + case tk + of tkString, tkInt, tkFloat, tkSymbol, tkNil: + my.state[i] = stateEof # expect EOF next! + my.kind = SexpEventKind(ord(tk)) + of tkParensLe: + my.state.add(stateList) + my.kind = sexpListStart + of tkEof: + my.kind = sexpEof + else: + my.kind = sexpError + my.err = errEofExpected + of stateList: + case tk + of tkString, tkInt, tkFloat, tkSymbol, tkNil: + my.kind = SexpEventKind(ord(tk)) + my.state.add(stateListExpectsSpace) + of tkParensLe: + my.state.add(stateList) + my.kind = sexpListStart + of tkParensRi: + my.kind = sexpListEnd + discard my.state.pop() + else: + my.kind = sexpError + my.err = errParensRiExpected + of stateListExpectsSpace: + case tk + of tkSpace: + discard my.state.pop() + next(my) + of tkParensLe: + my.kind = sexpListEnd + discard my.state.pop() + discard my.state.pop() + else: + my.kind = sexpError + my.err = errParensRiExpected + +# ------------- higher level interface --------------------------------------- + +type + SexpNodeKind* = enum ## possible SEXP node types + SNil, + SInt, + SFloat, + SString, + SSymbol + SList, + + SexpNode* = ref SexpNodeObj ## SEXP node + SexpNodeObj* {.acyclic.} = object + case kind*: SexpNodeKind + of SString: + str*: string + of SSymbol: + symbol*: string + of SInt: + num*: BiggestInt + of SFloat: + fnum*: float + of SList: + elems*: seq[SexpNode] + of SNil: + discard + + SexpParsingError* = object of ValueError ## is raised for a SEXP error + +proc raiseParseErr*(p: SexpParser, msg: string) {.noinline, noreturn.} = + ## raises an `ESexpParsingError` exception. + raise newException(SexpParsingError, errorMsgExpected(p, msg)) + +proc newSString*(s: string): SexpNode = + ## Creates a new `SString SexpNode`. + new(result) + result.kind = SString + result.str = s + +proc newSStringMove(s: string): SexpNode = + new(result) + result.kind = SString + shallowCopy(result.str, s) + +proc newSInt*(n: BiggestInt): SexpNode = + ## Creates a new `SInt SexpNode`. + new(result) + result.kind = SInt + result.num = n + +proc newSFloat*(n: float): SexpNode = + ## Creates a new `SFloat SexpNode`. + new(result) + result.kind = SFloat + result.fnum = n + +proc newSNil*(): SexpNode = + ## Creates a new `SNil SexpNode`. + new(result) + +proc newSList*(): SexpNode = + ## Creates a new `SList SexpNode` + new(result) + result.kind = SList + result.elems = @[] + +proc newSSymbol*(s: string): SexpNode = + new(result) + result.kind = SSymbol + result.symbol = s + +proc newSSymbolMove(s: string): SexpNode = + new(result) + result.kind = SSymbol + shallowCopy(result.symbol, s) + +proc getStr*(n: SexpNode, default: string = ""): string = + ## Retrieves the string value of a `SString SexpNode`. + ## + ## Returns ``default`` if ``n`` is not a ``SString``. + if n.kind != SString: return default + else: return n.str + +proc getNum*(n: SexpNode, default: BiggestInt = 0): BiggestInt = + ## Retrieves the int value of a `SInt SexpNode`. + ## + ## Returns ``default`` if ``n`` is not a ``SInt``. + if n.kind != SInt: return default + else: return n.num + +proc getFNum*(n: SexpNode, default: float = 0.0): float = + ## Retrieves the float value of a `SFloat SexpNode`. + ## + ## Returns ``default`` if ``n`` is not a ``SFloat``. + if n.kind != SFloat: return default + else: return n.fnum + +proc getSymbol*(n: SexpNode, default: string = ""): string = + ## Retrieves the int value of a `SList SexpNode`. + ## + ## Returns ``default`` if ``n`` is not a ``SList``. + if n.kind != SSymbol: return default + else: return n.symbol + +proc getElems*(n: SexpNode, default: seq[SexpNode] = @[]): seq[SexpNode] = + ## Retrieves the int value of a `SList SexpNode`. + ## + ## Returns ``default`` if ``n`` is not a ``SList``. + if n.kind == SNil: return @[] + elif n.kind != SList: return default + else: return n.elems + +proc `sexp`*(s: string): SexpNode = + ## Generic constructor for SEXP data. Creates a new `SString SexpNode`. + new(result) + result.kind = SString + result.str = s + +proc `sexp`*(n: BiggestInt): SexpNode = + ## Generic constructor for SEXP data. Creates a new `SInt SexpNode`. + new(result) + result.kind = SInt + result.num = n + +proc `sexp`*(n: float): SexpNode = + ## Generic constructor for SEXP data. Creates a new `SFloat SexpNode`. + new(result) + result.kind = SFloat + result.fnum = n + +proc `sexp`*(b: bool): SexpNode = + ## Generic constructor for SEXP data. Creates a new `SSymbol + ## SexpNode` with value t or `SNil SexpNode`. + new(result) + if b: + result.kind = SSymbol + result.symbol = "t" + else: + result.kind = SNil + +proc `sexp`*(elements: openArray[SexpNode]): SexpNode = + ## Generic constructor for SEXP data. Creates a new `SList SexpNode` + new(result) + result.kind = SList + newSeq(result.elems, elements.len) + for i, p in pairs(elements): result.elems[i] = p + +proc toSexp(x: NimNode): NimNode {.compiletime.} = + case x.kind + of nnkBracket: + result = newNimNode(nnkBracket) + for i in 0 .. = 32 and r <= 127: + var c = chr(r) + case c + of '"': result.add("\\\"") #" # gotta fix nim-mode + of '\\': result.add("\\\\") + else: result.add(c) + else: + result.add("\\u") + result.add(toHex(r, 4)) + result.add("\"") +proc copy*(p: SexpNode): SexpNode = + ## Performs a deep copy of `a`. + case p.kind + of SString: + result = newSString(p.str) + of SInt: + result = newSInt(p.num) + of SFloat: + result = newSFloat(p.fnum) + of SNil: + result = newSNil() + of SSymbol: + result = newSSymbol(p.symbol) + of SList: + result = newSList() + for i in items(p.elems): + result.elems.add(copy(i)) + +proc toPretty(result: var string, node: SexpNode, indent = 2, ml = true, + lstArr = false, currIndent = 0) = + case node.kind + of SString: + if lstArr: result.indent(currIndent) + result.add(escapeJson(node.str)) + of SInt: + if lstArr: result.indent(currIndent) + result.add($node.num) + of SFloat: + if lstArr: result.indent(currIndent) + result.add($node.fnum) + of SNil: + if lstArr: result.indent(currIndent) + result.add("nil") + of SSymbol: + if lstArr: result.indent(currIndent) + result.add($node.symbol) + of SList: + if lstArr: result.indent(currIndent) + if len(node.elems) != 0: + result.add("(") + result.nl(ml) + for i in 0..len(node.elems)-1: + if i > 0: + result.add(" ") + result.nl(ml) # New Line + toPretty(result, node.elems[i], indent, ml, + true, newIndent(currIndent, indent, ml)) + result.nl(ml) + result.indent(currIndent) + result.add(")") + else: result.add("nil") + +proc pretty*(node: SexpNode, indent = 2): string = + ## Converts `node` to its JSON Representation, with indentation and + ## on multiple lines. + result = "" + toPretty(result, node, indent) + +proc `$`*(node: SexpNode): string = + ## Converts `node` to its SEXP Representation on one line. + result = "" + toPretty(result, node, 0, false) + +iterator items*(node: SexpNode): SexpNode = + ## Iterator for the items of `node`. `node` has to be a SList. + assert node.kind == SList + for i in items(node.elems): + yield i + +iterator mitems*(node: var SexpNode): var SexpNode = + ## Iterator for the items of `node`. `node` has to be a SList. Items can be + ## modified. + assert node.kind == SList + for i in mitems(node.elems): + yield i + +proc eat(p: var SexpParser, tok: TTokKind) = + if p.tok == tok: discard getTok(p) + else: raiseParseErr(p, tokToStr[tok]) + +proc parseSexp(p: var SexpParser): SexpNode = + ## Parses SEXP from a SEXP Parser `p`. + case p.tok + of tkString: + # we capture 'p.a' here, so we need to give it a fresh buffer afterwards: + result = newSStringMove(p.a) + p.a = "" + discard getTok(p) + of tkInt: + result = newSInt(parseBiggestInt(p.a)) + discard getTok(p) + of tkFloat: + result = newSFloat(parseFloat(p.a)) + discard getTok(p) + of tkNil: + result = newSNil() + discard getTok(p) + of tkSymbol: + result = newSSymbolMove(p.a) + p.a = "" + discard getTok(p) + of tkParensLe: + result = newSList() + discard getTok(p) + while p.tok != tkParensRi: + result.add(parseSexp(p)) + if p.tok != tkSpace: break + discard getTok(p) + eat(p, tkParensRi) + of tkSpace, tkError, tkParensRi, tkEof: + raiseParseErr(p, "(") + +proc parseSexp*(s: Stream, filename: string): SexpNode = + ## Parses from a stream `s` into a `SexpNode`. `filename` is only needed + ## for nice error messages. + var p: SexpParser + p.open(s, filename) + discard getTok(p) # read first token + result = p.parseSexp() + p.close() + +proc parseSexp*(buffer: string): SexpNode = + ## Parses SEXP from `buffer`. + result = parseSexp(newStringStream(buffer), "input") + +proc parseFile*(filename: string): SexpNode = + ## Parses `file` into a `SexpNode`. + var stream = newFileStream(filename, fmRead) + if stream == nil: + raise newException(IOError, "cannot read from file: " & filename) + result = parseSexp(stream, filename) + +when isMainModule: + let testSexp = parseSexp("""(1 (98 2) nil (2) foobar "foo" 9.234)""") + assert(testSexp[0].getNum == 1) + assert(testSexp[1][0].getNum == 98) + assert(testSexp[2].getElems == @[]) + assert(testSexp[4].getSymbol == "foobar") + assert(testSexp[5].getStr == "foo") + + # Generator: + var j = convertSexp([true, false, "foobar", [1, 2, "baz"]]) + assert($j == """(t nil "foobar" (1 2 "baz"))""") From 8fa8f137412f535784a5222f60d45b3e0439ae0a Mon Sep 17 00:00:00 2001 From: Simon Hafner Date: Tue, 14 Apr 2015 02:00:23 -0500 Subject: [PATCH 02/29] renamed sexp --- lib/pure/{sexpressions.nim => sexp.nim} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename lib/pure/{sexpressions.nim => sexp.nim} (100%) diff --git a/lib/pure/sexpressions.nim b/lib/pure/sexp.nim similarity index 100% rename from lib/pure/sexpressions.nim rename to lib/pure/sexp.nim From 49b953f318b8183056bc67d50571910026d05c43 Mon Sep 17 00:00:00 2001 From: Simon Hafner Date: Tue, 14 Apr 2015 02:47:51 -0500 Subject: [PATCH 03/29] sexp also accepts cons --- compiler/nimsuggest/nimsuggest.nim | 3 +- lib/pure/sexp.nim | 180 +++++++++++++---------------- 2 files changed, 81 insertions(+), 102 deletions(-) diff --git a/compiler/nimsuggest/nimsuggest.nim b/compiler/nimsuggest/nimsuggest.nim index b45ca475c..ca856ed57 100644 --- a/compiler/nimsuggest/nimsuggest.nim +++ b/compiler/nimsuggest/nimsuggest.nim @@ -11,7 +11,7 @@ import strutils, os, parseopt, parseUtils import options, commands, modules, sem, passes, passaux, msgs, nimconf, - extccomp, condsyms, lists, net, rdstdin + extccomp, condsyms, lists, net, rdstdin, sexp const Usage = """ Nimsuggest - Tool to give every editor IDE like capabilities for Nim @@ -23,6 +23,7 @@ Options: --address:HOST binds to that address, by default "" --stdin read commands from stdin and write results to stdout instead of using sockets + --sexp talk in sexp, mainly for emacs epc. The server then listens to the connection and takes line-based commands. diff --git a/lib/pure/sexp.nim b/lib/pure/sexp.nim index c30f35c25..97edbf52d 100644 --- a/lib/pure/sexp.nim +++ b/lib/pure/sexp.nim @@ -19,6 +19,7 @@ type sexpInt, ## an integer literal sexpFloat, ## a float literal sexpNil, ## the value ``nil`` + sexpDot, ## the dot to separate car/cdr sexpListStart, ## start of a list: the ``(`` token sexpListEnd, ## end of a list: the ``)`` token @@ -30,6 +31,7 @@ type tkInt, tkFloat, tkNil, + tkDot, tkParensLe, tkParensRi tkSpace @@ -41,16 +43,11 @@ type errQuoteExpected, ## ``"`` expected errEofExpected, ## EOF expected - ParserState = enum - stateEof, stateStart, stateList, stateListExpectsSpace - SexpParser* = object of BaseLexer ## the parser object. a: string tok: TTokKind kind: SexpEventKind err: SexpError - state: seq[ParserState] - filename: string const errorMessages: array [SexpError, string] = [ @@ -68,18 +65,10 @@ const "int literal", "float literal", "nil", - "(", ")", " " + ".", + "(", ")", "space" ] -proc open*(my: var SexpParser, input: Stream, filename: string) = - ## initializes the parser with an input stream. `Filename` is only used - ## for nice error messages. - lexbase.open(my, input) - my.filename = filename - my.state = @[stateStart] - my.kind = sexpError - my.a = "" - proc close*(my: var SexpParser) {.inline.} = ## closes the parser `my` and its associated input stream. lexbase.close(my) @@ -112,21 +101,15 @@ proc getLine*(my: SexpParser): int {.inline.} = ## get the current line the parser has arrived at. result = my.lineNumber -proc getFilename*(my: SexpParser): string {.inline.} = - ## get the filename of the file that the parser processes. - result = my.filename - proc errorMsg*(my: SexpParser): string = ## returns a helpful error message for the event ``sexpError`` assert(my.kind == sexpError) - result = "$1($2, $3) Error: $4" % [ - my.filename, $getLine(my), $getColumn(my), errorMessages[my.err]] + result = "($1, $2) Error: $3" % [$getLine(my), $getColumn(my), errorMessages[my.err]] proc errorMsgExpected*(my: SexpParser, e: string): string = ## returns an error message "`e` expected" in the same format as the ## other error messages - result = "$1($2, $3) Error: $4" % [ - my.filename, $getLine(my), $getColumn(my), e & " expected"] + result = "($1, $2) Error: $3" % [$getLine(my), $getColumn(my), e & " expected"] proc handleHexChar(c: char, x: var int): bool = result = true # Success @@ -237,7 +220,8 @@ proc parseSymbol(my: var SexpParser) = proc getTok(my: var SexpParser): TTokKind = setLen(my.a, 0) case my.buf[my.bufpos] - of '-', '.', '0'..'9': + of '-', '0'..'9': # numbers that start with a . are not parsed + # correctly. parseNumber(my) if {'.', 'e', 'E'} in my.a: result = tkFloat @@ -262,65 +246,14 @@ proc getTok(my: var SexpParser): TTokKind = of ' ': result = tkSpace inc(my.bufpos) + of '.': + result = tkDot + inc(my.bufpos) else: inc(my.bufpos) result = tkError my.tok = result -proc next*(my: var SexpParser) = - ## retrieves the first/next event. This controls the parser. - var tk = getTok(my) - var i = my.state.len-1 - # the following code is a state machine. If we had proper coroutines, - # the code could be much simpler. - case my.state[i] - of stateEof: - if tk == tkEof: - my.kind = sexpEof - else: - my.kind = sexpError - my.err = errEofExpected - of stateStart: - # tokens allowed? - case tk - of tkString, tkInt, tkFloat, tkSymbol, tkNil: - my.state[i] = stateEof # expect EOF next! - my.kind = SexpEventKind(ord(tk)) - of tkParensLe: - my.state.add(stateList) - my.kind = sexpListStart - of tkEof: - my.kind = sexpEof - else: - my.kind = sexpError - my.err = errEofExpected - of stateList: - case tk - of tkString, tkInt, tkFloat, tkSymbol, tkNil: - my.kind = SexpEventKind(ord(tk)) - my.state.add(stateListExpectsSpace) - of tkParensLe: - my.state.add(stateList) - my.kind = sexpListStart - of tkParensRi: - my.kind = sexpListEnd - discard my.state.pop() - else: - my.kind = sexpError - my.err = errParensRiExpected - of stateListExpectsSpace: - case tk - of tkSpace: - discard my.state.pop() - next(my) - of tkParensLe: - my.kind = sexpListEnd - discard my.state.pop() - discard my.state.pop() - else: - my.kind = sexpError - my.err = errParensRiExpected - # ------------- higher level interface --------------------------------------- type @@ -329,8 +262,9 @@ type SInt, SFloat, SString, - SSymbol + SSymbol, SList, + SCons SexpNode* = ref SexpNodeObj ## SEXP node SexpNodeObj* {.acyclic.} = object @@ -345,9 +279,14 @@ type fnum*: float of SList: elems*: seq[SexpNode] + of SCons: + car: SexpNode + cdr: SexpNode of SNil: discard + Cons = tuple[car: SexpNode, cdr: SexpNode] + SexpParsingError* = object of ValueError ## is raised for a SEXP error proc raiseParseErr*(p: SexpParser, msg: string) {.noinline, noreturn.} = @@ -381,6 +320,13 @@ proc newSNil*(): SexpNode = ## Creates a new `SNil SexpNode`. new(result) +proc newSCons*(car, cdr: SexpNode): SexpNode = + ## Creates a new `SCons SexpNode` + new(result) + result.kind = SCons + result.car = car + result.cdr = cdr + proc newSList*(): SexpNode = ## Creates a new `SList SexpNode` new(result) @@ -433,6 +379,14 @@ proc getElems*(n: SexpNode, default: seq[SexpNode] = @[]): seq[SexpNode] = elif n.kind != SList: return default else: return n.elems +proc getCons*(n: SexpNode, defaults: Cons = (newSNil(), newSNil())): Cons = + ## Retrieves the cons value of a `SList SexpNode`. + ## + ## Returns ``default`` if ``n`` is not a ``SList``. + if n.kind == SCons: return (n.car, n.cdr) + elif n.kind == SList: return (n.elems[0], n.elems[1]) + else: return defaults + proc `sexp`*(s: string): SexpNode = ## Generic constructor for SEXP data. Creates a new `SString SexpNode`. new(result) @@ -506,6 +460,8 @@ proc `==`* (a,b: SexpNode): bool = a.elems == b.elems of SSymbol: a.symbol == b.symbol + of SCons: + a.car == b.car and a.cdr == b.cdr proc hash* (n:SexpNode): THash = ## Compute the hash for a SEXP node @@ -522,6 +478,8 @@ proc hash* (n:SexpNode): THash = result = hash(0) of SSymbol: result = hash(n.symbol) + of SCons: + result = hash(n.car) !& hash(n.cdr) proc len*(n: SexpNode): int = ## If `n` is a `SList`, it returns the number of elements. @@ -571,6 +529,7 @@ proc escapeJson*(s: string): string = result.add("\\u") result.add(toHex(r, 4)) result.add("\"") + proc copy*(p: SexpNode): SexpNode = ## Performs a deep copy of `a`. case p.kind @@ -588,6 +547,8 @@ proc copy*(p: SexpNode): SexpNode = result = newSList() for i in items(p.elems): result.elems.add(copy(i)) + of SCons: + result = newSCons(copy(p.car), copy(p.cdr)) proc toPretty(result: var string, node: SexpNode, indent = 2, ml = true, lstArr = false, currIndent = 0) = @@ -622,9 +583,18 @@ proc toPretty(result: var string, node: SexpNode, indent = 2, ml = true, result.indent(currIndent) result.add(")") else: result.add("nil") + of SCons: + if lstArr: result.indent(currIndent) + result.add("(") + toPretty(result, node.car, indent, ml, + true, newIndent(currIndent, indent, ml)) + result.add(" . ") + toPretty(result, node.cdr, indent, ml, + true, newIndent(currIndent, indent, ml)) + result.add(")") proc pretty*(node: SexpNode, indent = 2): string = - ## Converts `node` to its JSON Representation, with indentation and + ## Converts `node` to its Sexp Representation, with indentation and ## on multiple lines. result = "" toPretty(result, node, indent) @@ -675,41 +645,49 @@ proc parseSexp(p: var SexpParser): SexpNode = of tkParensLe: result = newSList() discard getTok(p) - while p.tok != tkParensRi: + while p.tok notin {tkParensRi, tkDot}: result.add(parseSexp(p)) if p.tok != tkSpace: break discard getTok(p) + if p.tok == tkDot: + eat(p, tkDot) + eat(p, tkSpace) + result.add(parseSexp(p)) + result = newSCons(result[0], result[1]) eat(p, tkParensRi) - of tkSpace, tkError, tkParensRi, tkEof: + of tkSpace, tkDot, tkError, tkParensRi, tkEof: raiseParseErr(p, "(") -proc parseSexp*(s: Stream, filename: string): SexpNode = - ## Parses from a stream `s` into a `SexpNode`. `filename` is only needed - ## for nice error messages. +proc open*(my: var SexpParser, input: Stream) = + ## initializes the parser with an input stream. + lexbase.open(my, input) + my.kind = sexpError + my.a = "" + +proc parseSexp*(s: Stream): SexpNode = + ## Parses from a buffer `s` into a `SexpNode`. var p: SexpParser - p.open(s, filename) + p.open(s) discard getTok(p) # read first token result = p.parseSexp() p.close() proc parseSexp*(buffer: string): SexpNode = - ## Parses SEXP from `buffer`. - result = parseSexp(newStringStream(buffer), "input") - -proc parseFile*(filename: string): SexpNode = - ## Parses `file` into a `SexpNode`. - var stream = newFileStream(filename, fmRead) - if stream == nil: - raise newException(IOError, "cannot read from file: " & filename) - result = parseSexp(stream, filename) + ## Parses Sexp from `buffer`. + result = parseSexp(newStringStream(buffer)) when isMainModule: - let testSexp = parseSexp("""(1 (98 2) nil (2) foobar "foo" 9.234)""") - assert(testSexp[0].getNum == 1) - assert(testSexp[1][0].getNum == 98) - assert(testSexp[2].getElems == @[]) - assert(testSexp[4].getSymbol == "foobar") - assert(testSexp[5].getStr == "foo") + # let testSexp = parseSexp("""(1 (98 2) nil (2) foobar "foo" 9.234)""") + # assert(testSexp[0].getNum == 1) + # assert(testSexp[1][0].getNum == 98) + # assert(testSexp[2].getElems == @[]) + # assert(testSexp[4].getSymbol == "foobar") + # assert(testSexp[5].getStr == "foo") + + let alist = parseSexp("""((1 . 2) (2 . "foo"))""") + assert(alist[0].getCons.car.getNum == 1) + assert(alist[0].getCons.cdr.getNum == 2) + assert(alist[1].getCons.cdr.getStr == "foo") # Generator: var j = convertSexp([true, false, "foobar", [1, 2, "baz"]]) From 518db9eb7ee538f3c33732e5a04cbacd024ee8b7 Mon Sep 17 00:00:00 2001 From: Simon Hafner Date: Wed, 22 Apr 2015 06:10:47 +0500 Subject: [PATCH 04/29] rewrite suggest.nim for suggestionResultHook --- compiler/suggest.nim | 149 +++++++++++++++++++++++++++++++------------ 1 file changed, 108 insertions(+), 41 deletions(-) diff --git a/compiler/suggest.nim b/compiler/suggest.nim index 6b168670c..072145a70 100644 --- a/compiler/suggest.nim +++ b/compiler/suggest.nim @@ -20,52 +20,119 @@ const sectionContext = "con" sectionUsage = "use" +type + Section = enum sug, def, con, use + Suggest = object + section: Section + qualifiedPath: seq[string] + filePath: string + line: int # Starts at 1 + column: int # Starts at 0 + doc: string # Not escaped (yet) + symkind: TSymKind + forth: string # XXX TODO object on symkind + +var + suggestionResultHook*: proc (result: Suggest) {.closure.} + #template sectionSuggest(): expr = "##begin\n" & getStackTrace() & "##end\n" template origModuleName(m: PSym): string = m.name.s -proc symToStr(s: PSym, isLocal: bool, section: string, li: TLineInfo): string = - result = section - result.add(sep) +proc parseSection(s: string): Section = + case s: + of "sug": result = sug + of "con": result = con + of "use": result = use + of "def": result = def + else: raise newException(ERecoverableError, "Answer type incorrect") + +proc parseSymKind(s: string): TSymKind = + case s: + of "skDynLib": result = skDynLib + of "skParam": result = skParam + of "skGenericParam": result = skGenericParam + of "skTemp": result = skTemp + of "skModule": result = skModule + of "skType": result = skType + of "skVar": result = skVar + of "skLet": result = skLet + of "skConst": result = skConst + of "skResult": result = skResult + of "skProc": result = skProc + of "skMethod": result = skMethod + of "skIterator": result = skIterator + of "skClosureIterator": result = skClosureIterator + of "skConverter": result = skConverter + of "skMacro": result = skMacro + of "skTemplate": result = skTemplate + of "skField": result = skField + of "skEnumField": result = skEnumField + of "skForVar": result = skForVar + of "skLabel": result = skLabel + of "skStub": result = skStub + of "skPackage": result = skPackage + of "skAlias": result = skAlias + else: raise newException(ERecoverableError, "TSymKind not found") + +proc symToSuggest(s: PSym, isLocal: bool, section: string, li: TLineInfo): Suggest = + result.section = parseSection(section) if optIdeTerse in gGlobalOptions: if s.kind in routineKinds: - result.add renderTree(s.ast, {renderNoBody, renderNoComments, - renderDocComments, renderNoPragmas}) + result.symkind = parseSymKind(renderTree(s.ast, {renderNoBody, renderNoComments, + renderDocComments, renderNoPragmas})) else: - result.add s.name.s - result.add(sep) - result.add(toFullPath(li)) - result.add(sep) - result.add($toLinenumber(li)) - result.add(sep) - result.add($toColumn(li)) + result.symkind = parseSymKind(s.name.s) + result.filePath = toFullPath(li) + result.line = toLinenumber(li) + result.column = toColumn(li) else: - result.add($s.kind) - result.add(sep) + result.symkind = s.kind + result.qualifiedPath = @[] if not isLocal and s.kind != skModule: let ow = s.owner if ow.kind != skModule and ow.owner != nil: let ow2 = ow.owner - result.add(ow2.origModuleName) - result.add('.') - result.add(ow.origModuleName) - result.add('.') - result.add(s.name.s) - result.add(sep) - if s.typ != nil: - result.add(typeToString(s.typ)) - result.add(sep) - result.add(toFullPath(li)) - result.add(sep) - result.add($toLinenumber(li)) - result.add(sep) - result.add($toColumn(li)) - result.add(sep) - when not defined(noDocgen): - result.add(s.extractDocComment.escape) + result.qualifiedPath.add(ow2.origModuleName) + result.qualifiedPath.add(ow.origModuleName) + result.qualifiedPath.add(s.name.s) -proc symToStr(s: PSym, isLocal: bool, section: string): string = - result = symToStr(s, isLocal, section, s.info) + if s.typ != nil: + result.forth = typeToString(s.typ) + else: + result.forth = "" + result.filePath = toFullPath(li) + result.line = toLinenumber(li) + result.column = toColumn(li) + when not defined(noDocgen): + result.doc = s.extractDocComment + +proc `$`(suggest: Suggest): string = + result = $suggest.section + result.add(sep) + result.add($suggest.symkind) + result.add(sep) + result.add(suggest.qualifiedPath.join(".")) + result.add(sep) + result.add(suggest.forth) + result.add(sep) + result.add(suggest.filePath) + result.add(sep) + result.add($suggest.line) + result.add(sep) + result.add($suggest.column) + result.add(sep) + when not defined(noDocgen): + result.add(suggest.doc.escape) + +proc symToSuggest(s: PSym, isLocal: bool, section: string): Suggest = + result = symToSuggest(s, isLocal, section, s.info) + +proc suggestResult(s: Suggest) = + if not isNil(suggestionResultHook): + suggestionResultHook(s) + else: + suggestWriteln($(s)) proc filterSym(s: PSym): bool {.inline.} = result = s.kind != skModule @@ -84,7 +151,7 @@ proc fieldVisible*(c: PContext, f: PSym): bool {.inline.} = proc suggestField(c: PContext, s: PSym, outputs: var int) = if filterSym(s) and fieldVisible(c, s): - suggestWriteln(symToStr(s, isLocal=true, sectionSuggest)) + suggestResult(symToSuggest(s, isLocal=true, sectionSuggest)) inc outputs template wholeSymTab(cond, section: expr) {.immediate.} = @@ -97,7 +164,7 @@ template wholeSymTab(cond, section: expr) {.immediate.} = for item in entries: let it {.inject.} = item if cond: - suggestWriteln(symToStr(it, isLocal = isLocal, section)) + suggestResult(symToSuggest(it, isLocal = isLocal, section)) inc outputs proc suggestSymList(c: PContext, list: PNode, outputs: var int) = @@ -166,7 +233,7 @@ proc suggestEverything(c: PContext, n: PNode, outputs: var int) = if scope == c.topLevelScope: isLocal = false for it in items(scope.symbols): if filterSym(it): - suggestWriteln(symToStr(it, isLocal = isLocal, sectionSuggest)) + suggestResult(symToSuggest(it, isLocal = isLocal, sectionSuggest)) inc outputs if scope == c.topLevelScope: break @@ -181,12 +248,12 @@ proc suggestFieldAccess(c: PContext, n: PNode, outputs: var int) = # all symbols accessible, because we are in the current module: for it in items(c.topLevelScope.symbols): if filterSym(it): - suggestWriteln(symToStr(it, isLocal=false, sectionSuggest)) + suggestResult(symToSuggest(it, isLocal=false, sectionSuggest)) inc outputs else: for it in items(n.sym.tab): if filterSym(it): - suggestWriteln(symToStr(it, isLocal=false, sectionSuggest)) + suggestResult(symToSuggest(it, isLocal=false, sectionSuggest)) inc outputs else: # fallback: @@ -263,16 +330,16 @@ var proc findUsages(info: TLineInfo; s: PSym) = if usageSym == nil and isTracked(info, s.name.s.len): usageSym = s - suggestWriteln(symToStr(s, isLocal=false, sectionUsage)) + suggestResult(symToSuggest(s, isLocal=false, sectionUsage)) elif s == usageSym: if lastLineInfo != info: - suggestWriteln(symToStr(s, isLocal=false, sectionUsage, info)) + suggestResult(symToSuggest(s, isLocal=false, sectionUsage, info)) lastLineInfo = info proc findDefinition(info: TLineInfo; s: PSym) = if s.isNil: return if isTracked(info, s.name.s.len): - suggestWriteln(symToStr(s, isLocal=false, sectionDef)) + suggestResult(symToSuggest(s, isLocal=false, sectionDef)) suggestQuit() proc ensureIdx[T](x: var T, y: int) = From 766b7ea61d74532ede74e0aea59b2cda3400cfed Mon Sep 17 00:00:00 2001 From: Simon Hafner Date: Wed, 22 Apr 2015 06:11:30 +0500 Subject: [PATCH 05/29] enabled sexp tests --- lib/pure/sexp.nim | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/pure/sexp.nim b/lib/pure/sexp.nim index 97edbf52d..2a9678432 100644 --- a/lib/pure/sexp.nim +++ b/lib/pure/sexp.nim @@ -677,12 +677,12 @@ proc parseSexp*(buffer: string): SexpNode = result = parseSexp(newStringStream(buffer)) when isMainModule: - # let testSexp = parseSexp("""(1 (98 2) nil (2) foobar "foo" 9.234)""") - # assert(testSexp[0].getNum == 1) - # assert(testSexp[1][0].getNum == 98) - # assert(testSexp[2].getElems == @[]) - # assert(testSexp[4].getSymbol == "foobar") - # assert(testSexp[5].getStr == "foo") + let testSexp = parseSexp("""(1 (98 2) nil (2) foobar "foo" 9.234)""") + assert(testSexp[0].getNum == 1) + assert(testSexp[1][0].getNum == 98) + assert(testSexp[2].getElems == @[]) + assert(testSexp[4].getSymbol == "foobar") + assert(testSexp[5].getStr == "foo") let alist = parseSexp("""((1 . 2) (2 . "foo"))""") assert(alist[0].getCons.car.getNum == 1) From b01b7675abdbf9814b8c1b05a7df4c05b113e6e6 Mon Sep 17 00:00:00 2001 From: Simon Hafner Date: Wed, 22 Apr 2015 06:15:52 +0500 Subject: [PATCH 06/29] parseEnum ftw --- compiler/suggest.nim | 44 ++++---------------------------------------- 1 file changed, 4 insertions(+), 40 deletions(-) diff --git a/compiler/suggest.nim b/compiler/suggest.nim index 072145a70..97d0b51c2 100644 --- a/compiler/suggest.nim +++ b/compiler/suggest.nim @@ -11,7 +11,7 @@ # included from sigmatch.nim -import algorithm, sequtils +import algorithm, sequtils, strutils const sep = '\t' @@ -39,50 +39,14 @@ var template origModuleName(m: PSym): string = m.name.s -proc parseSection(s: string): Section = - case s: - of "sug": result = sug - of "con": result = con - of "use": result = use - of "def": result = def - else: raise newException(ERecoverableError, "Answer type incorrect") - -proc parseSymKind(s: string): TSymKind = - case s: - of "skDynLib": result = skDynLib - of "skParam": result = skParam - of "skGenericParam": result = skGenericParam - of "skTemp": result = skTemp - of "skModule": result = skModule - of "skType": result = skType - of "skVar": result = skVar - of "skLet": result = skLet - of "skConst": result = skConst - of "skResult": result = skResult - of "skProc": result = skProc - of "skMethod": result = skMethod - of "skIterator": result = skIterator - of "skClosureIterator": result = skClosureIterator - of "skConverter": result = skConverter - of "skMacro": result = skMacro - of "skTemplate": result = skTemplate - of "skField": result = skField - of "skEnumField": result = skEnumField - of "skForVar": result = skForVar - of "skLabel": result = skLabel - of "skStub": result = skStub - of "skPackage": result = skPackage - of "skAlias": result = skAlias - else: raise newException(ERecoverableError, "TSymKind not found") - proc symToSuggest(s: PSym, isLocal: bool, section: string, li: TLineInfo): Suggest = - result.section = parseSection(section) + result.section = parseEnum[Section](section) if optIdeTerse in gGlobalOptions: if s.kind in routineKinds: - result.symkind = parseSymKind(renderTree(s.ast, {renderNoBody, renderNoComments, + result.symkind = parseEnum[TSymKind](renderTree(s.ast, {renderNoBody, renderNoComments, renderDocComments, renderNoPragmas})) else: - result.symkind = parseSymKind(s.name.s) + result.symkind = parseEnum[TSymKind](s.name.s) result.filePath = toFullPath(li) result.line = toLinenumber(li) result.column = toColumn(li) From f0f72613d87f5bd742eb64f61b75567c205e62e8 Mon Sep 17 00:00:00 2001 From: Simon Hafner Date: Wed, 22 Apr 2015 06:32:02 +0500 Subject: [PATCH 07/29] epc skeleton --- compiler/nimsuggest/nimsuggest.nim | 24 +++++++++++++++++------- 1 file changed, 17 insertions(+), 7 deletions(-) diff --git a/compiler/nimsuggest/nimsuggest.nim b/compiler/nimsuggest/nimsuggest.nim index ca856ed57..caf21fbd0 100644 --- a/compiler/nimsuggest/nimsuggest.nim +++ b/compiler/nimsuggest/nimsuggest.nim @@ -23,18 +23,20 @@ Options: --address:HOST binds to that address, by default "" --stdin read commands from stdin and write results to stdout instead of using sockets - --sexp talk in sexp, mainly for emacs epc. + --epc use emacs epc mode The server then listens to the connection and takes line-based commands. In addition, all command line options of Nim that do not affect code generation are supported. """ +type + Mode = enum mstdin, mtcp, mepc var gPort = 6000.Port gAddress = "" - gUseStdin: bool + gMode: Mode const seps = {':', ';', ' ', '\t'} @@ -109,14 +111,15 @@ proc action(cmd: string) = proc serve() = # do not stop after the first error: msgs.gErrorMax = high(int) - if gUseStdin: + case gMode: + of mstdin: echo Help var line = "" while readLineFromStdin("> ", line): action line echo "" flushFile(stdout) - else: + of mtcp: var server = newSocket() server.bindAddr(gPort, gAddress) var inp = "".TaintedString @@ -134,6 +137,8 @@ proc serve() = stdoutSocket.send("\c\L") stdoutSocket.close() + of mepc: + discard proc mainCommand = registerPass verbosePass @@ -157,9 +162,14 @@ proc processCmdLine*(pass: TCmdLinePass, cmd: string) = of cmdEnd: break of cmdLongoption, cmdShortOption: case p.key.normalize - of "port": gPort = parseInt(p.val).Port - of "address": gAddress = p.val - of "stdin": gUseStdin = true + of "port": + gPort = parseInt(p.val).Port + gMode = mtcp + of "address": + gAddress = p.val + gMode = mtcp + of "stdin": gMode = mstdin + of "epc": gMode = mepc else: processSwitch(pass, p) of cmdArgument: options.gProjectName = unixToNativePath(p.key) From 7d42eb2de2ef85925c2a933e71f7fad3ab411bce Mon Sep 17 00:00:00 2001 From: Simon Hafner Date: Wed, 22 Apr 2015 21:53:04 +0500 Subject: [PATCH 08/29] barebones in nimsuggest for EPC --- compiler/nimsuggest/nimsuggest.nim | 84 ++++++++++++++++++++++++++++-- compiler/suggest.nim | 23 ++++---- lib/pure/sexp.nim | 24 ++++----- 3 files changed, 104 insertions(+), 27 deletions(-) diff --git a/compiler/nimsuggest/nimsuggest.nim b/compiler/nimsuggest/nimsuggest.nim index caf21fbd0..d6bd01ce7 100644 --- a/compiler/nimsuggest/nimsuggest.nim +++ b/compiler/nimsuggest/nimsuggest.nim @@ -9,9 +9,9 @@ ## Nimsuggest is a tool that helps to give editors IDE like capabilities. -import strutils, os, parseopt, parseUtils +import strutils, os, parseopt, parseutils, sequtils import options, commands, modules, sem, passes, passaux, msgs, nimconf, - extccomp, condsyms, lists, net, rdstdin, sexp + extccomp, condsyms, lists, net, rdstdin, sexp, suggest, ast const Usage = """ Nimsuggest - Tool to give every editor IDE like capabilities for Nim @@ -54,6 +54,35 @@ proc parseQuoted(cmd: string; outp: var string; start: int): int = i += parseUntil(cmd, outp, seps, i) result = i +# make sure it's in the same order as the proc below +let order: SexpNode = + sexp(@["section", "symkind", "qualifiedPath", "filePath", "forth", "line", "column", "doc"].map(newSSymbol)) + +proc sexp(s: Section): SexpNode = sexp($s) + +proc sexp(s: TSymKind): SexpNode = sexp($s) + +proc sexp(s: Suggest): SexpNode = + result = convertSexp([ + s.section, + s.symkind, + s.qualifiedPath.map(newSString), + s.filePath, + s.forth, + s.line, + s.column, + s.doc + ]) + +proc sexp(s: seq[Suggest]): SexpNode = + result = sexp(s) + +proc listEPC(): SexpNode = + discard + +proc executeEPC(body: SexpNode) = + discard + proc action(cmd: string) = template toggle(sw) = if sw in gGlobalOptions: @@ -138,7 +167,52 @@ proc serve() = stdoutSocket.send("\c\L") stdoutSocket.close() of mepc: - discard + let port = 98294 # guaranteed to be random + var server = newSocket() + server.bindaddr(port.Port, "localhost") + var inp = "".TaintedString + server.listen() + echo(port) + while true: + var results: seq[Suggest] = @[] + var client = newSocket() + suggest.suggestionResultHook = proc (s: Suggest) = + results.add(s) + + accept(server, client) + var sizeHex = "" + if client.recv(sizeHex, 6, 1000) != 6: + raise newException(ValueError, "didn't get all the hexbytes") + var size = 0 + if parseHex(sizeHex, size) == 0: + raiseRecoverableError("invalid size hex: " & $sizeHex) + var messageBuffer = "" + if client.recv(messageBuffer, size, 3000) != size: + raise newException(ValueError, "didn't get all the bytes") + let message = parseSexp($messageBuffer) + let messageType = message[0].getSymbol + let body = message[1] + case messageType: + of "call": + executeEPC(body) + let response = $sexp(results) + client.send(toHex(len(response), 6)) + client.send(response) + client.close() + of "return": + raise newException(ValueError, "no return expected") + of "return-error": + raise newException(ValueError, "no return expected") + of "epc-error": + stderr.writeln("recieved epc error: " & $messageBuffer) + raise newException(ValueError, "epc error") + of "methods": + let response = $listEPC() + client.send(toHex(len(response), 6)) + client.send(response) + client.close() + else: + raise newException(ValueError, "unexpected call: " & messageType) proc mainCommand = registerPass verbosePass @@ -169,7 +243,9 @@ proc processCmdLine*(pass: TCmdLinePass, cmd: string) = gAddress = p.val gMode = mtcp of "stdin": gMode = mstdin - of "epc": gMode = mepc + of "epc": + gMode = mepc + gVerbosity = 0 # Port number gotta be first. else: processSwitch(pass, p) of cmdArgument: options.gProjectName = unixToNativePath(p.key) diff --git a/compiler/suggest.nim b/compiler/suggest.nim index 97d0b51c2..539d4bc5d 100644 --- a/compiler/suggest.nim +++ b/compiler/suggest.nim @@ -11,7 +11,8 @@ # included from sigmatch.nim -import algorithm, sequtils, strutils +import algorithm, sequtils, strutils, ast, msgs, options, renderer, + types, docgen, lexer, semdata, astalgo, idents, sigmatch, lookups const sep = '\t' @@ -21,16 +22,16 @@ const sectionUsage = "use" type - Section = enum sug, def, con, use - Suggest = object - section: Section - qualifiedPath: seq[string] - filePath: string - line: int # Starts at 1 - column: int # Starts at 0 - doc: string # Not escaped (yet) - symkind: TSymKind - forth: string # XXX TODO object on symkind + Section* = enum sug, def, con, use + Suggest* = object + section*: Section + qualifiedPath*: seq[string] + filePath*: string + line*: int # Starts at 1 + column*: int # Starts at 0 + doc*: string # Not escaped (yet) + symkind*: TSymKind + forth*: string # XXX TODO object on symkind var suggestionResultHook*: proc (result: Suggest) {.closure.} diff --git a/lib/pure/sexp.nim b/lib/pure/sexp.nim index 2a9678432..b5e67b90f 100644 --- a/lib/pure/sexp.nim +++ b/lib/pure/sexp.nim @@ -293,7 +293,7 @@ proc raiseParseErr*(p: SexpParser, msg: string) {.noinline, noreturn.} = ## raises an `ESexpParsingError` exception. raise newException(SexpParsingError, errorMsgExpected(p, msg)) -proc newSString*(s: string): SexpNode = +proc newSString*(s: string): SexpNode {.procvar.}= ## Creates a new `SString SexpNode`. new(result) result.kind = SString @@ -304,36 +304,36 @@ proc newSStringMove(s: string): SexpNode = result.kind = SString shallowCopy(result.str, s) -proc newSInt*(n: BiggestInt): SexpNode = +proc newSInt*(n: BiggestInt): SexpNode {.procvar.} = ## Creates a new `SInt SexpNode`. new(result) result.kind = SInt result.num = n -proc newSFloat*(n: float): SexpNode = +proc newSFloat*(n: float): SexpNode {.procvar.} = ## Creates a new `SFloat SexpNode`. new(result) result.kind = SFloat result.fnum = n -proc newSNil*(): SexpNode = +proc newSNil*(): SexpNode {.procvar.} = ## Creates a new `SNil SexpNode`. new(result) -proc newSCons*(car, cdr: SexpNode): SexpNode = +proc newSCons*(car, cdr: SexpNode): SexpNode {.procvar.} = ## Creates a new `SCons SexpNode` new(result) result.kind = SCons result.car = car result.cdr = cdr -proc newSList*(): SexpNode = +proc newSList*(): SexpNode {.procvar.} = ## Creates a new `SList SexpNode` new(result) result.kind = SList result.elems = @[] -proc newSSymbol*(s: string): SexpNode = +proc newSSymbol*(s: string): SexpNode {.procvar.} = new(result) result.kind = SSymbol result.symbol = s @@ -387,25 +387,25 @@ proc getCons*(n: SexpNode, defaults: Cons = (newSNil(), newSNil())): Cons = elif n.kind == SList: return (n.elems[0], n.elems[1]) else: return defaults -proc `sexp`*(s: string): SexpNode = +proc sexp*(s: string): SexpNode = ## Generic constructor for SEXP data. Creates a new `SString SexpNode`. new(result) result.kind = SString result.str = s -proc `sexp`*(n: BiggestInt): SexpNode = +proc sexp*(n: BiggestInt): SexpNode = ## Generic constructor for SEXP data. Creates a new `SInt SexpNode`. new(result) result.kind = SInt result.num = n -proc `sexp`*(n: float): SexpNode = +proc sexp*(n: float): SexpNode = ## Generic constructor for SEXP data. Creates a new `SFloat SexpNode`. new(result) result.kind = SFloat result.fnum = n -proc `sexp`*(b: bool): SexpNode = +proc sexp*(b: bool): SexpNode = ## Generic constructor for SEXP data. Creates a new `SSymbol ## SexpNode` with value t or `SNil SexpNode`. new(result) @@ -415,7 +415,7 @@ proc `sexp`*(b: bool): SexpNode = else: result.kind = SNil -proc `sexp`*(elements: openArray[SexpNode]): SexpNode = +proc sexp*(elements: openArray[SexpNode]): SexpNode = ## Generic constructor for SEXP data. Creates a new `SList SexpNode` new(result) result.kind = SList From 147037bd1eddc447341b079020080e61f5ba3385 Mon Sep 17 00:00:00 2001 From: Simon Hafner Date: Sun, 26 Apr 2015 08:36:54 +0500 Subject: [PATCH 09/29] added id for sexp --- lib/pure/sexp.nim | 3 +++ 1 file changed, 3 insertions(+) diff --git a/lib/pure/sexp.nim b/lib/pure/sexp.nim index b5e67b90f..3c9fbc150 100644 --- a/lib/pure/sexp.nim +++ b/lib/pure/sexp.nim @@ -422,6 +422,9 @@ proc sexp*(elements: openArray[SexpNode]): SexpNode = newSeq(result.elems, elements.len) for i, p in pairs(elements): result.elems[i] = p +proc sexp*(s: SexpNode): SexpNode = + result = s + proc toSexp(x: NimNode): NimNode {.compiletime.} = case x.kind of nnkBracket: From 58570a4a3086852acbd6ee059c9e3427f504b2ab Mon Sep 17 00:00:00 2001 From: Simon Hafner Date: Sun, 26 Apr 2015 08:37:55 +0500 Subject: [PATCH 10/29] epc request/response barebones --- compiler/nimsuggest/nimsuggest.nim | 29 +++++++++++++++++++---------- 1 file changed, 19 insertions(+), 10 deletions(-) diff --git a/compiler/nimsuggest/nimsuggest.nim b/compiler/nimsuggest/nimsuggest.nim index d6bd01ce7..886590e35 100644 --- a/compiler/nimsuggest/nimsuggest.nim +++ b/compiler/nimsuggest/nimsuggest.nim @@ -80,9 +80,20 @@ proc sexp(s: seq[Suggest]): SexpNode = proc listEPC(): SexpNode = discard -proc executeEPC(body: SexpNode) = +proc executeEPC(section: Section, args: SexpNode) = + let + file = args[0].getStr + line = args[1].getNum + column = args[2].getNum + dirtyfile = args[3].getStr(nil) discard +proc returnEPC(socket: var Socket, uid: string, s: SexpNode) = + let response = $convertSexp([newSSymbol("return"), [uid, s]]) + socket.send(toHex(len(response), 6)) + socket.send(response) + socket.close() + proc action(cmd: string) = template toggle(sw) = if sw in gGlobalOptions: @@ -194,11 +205,12 @@ proc serve() = let body = message[1] case messageType: of "call": - executeEPC(body) - let response = $sexp(results) - client.send(toHex(len(response), 6)) - client.send(response) - client.close() + let + uid = body[0].getStr + section = parseEnum[Section](body[1].getStr) + args = body[2] + executeEPC(section, args) + returnEPC(client, uid, sexp(results)) of "return": raise newException(ValueError, "no return expected") of "return-error": @@ -207,10 +219,7 @@ proc serve() = stderr.writeln("recieved epc error: " & $messageBuffer) raise newException(ValueError, "epc error") of "methods": - let response = $listEPC() - client.send(toHex(len(response), 6)) - client.send(response) - client.close() + returnEPC(client, body[0].getStr, listEPC()) else: raise newException(ValueError, "unexpected call: " & messageType) From b9bbfb4cdfe081b0d0de52b28e1fef4d81fa5db3 Mon Sep 17 00:00:00 2001 From: Simon Hafner Date: Sun, 26 Apr 2015 08:41:23 +0500 Subject: [PATCH 11/29] always raise ValueException --- compiler/nimsuggest/nimsuggest.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/nimsuggest/nimsuggest.nim b/compiler/nimsuggest/nimsuggest.nim index 886590e35..97bbf6ea3 100644 --- a/compiler/nimsuggest/nimsuggest.nim +++ b/compiler/nimsuggest/nimsuggest.nim @@ -196,7 +196,7 @@ proc serve() = raise newException(ValueError, "didn't get all the hexbytes") var size = 0 if parseHex(sizeHex, size) == 0: - raiseRecoverableError("invalid size hex: " & $sizeHex) + raise newException(ValueError, "invalid size hex: " & $sizeHex) var messageBuffer = "" if client.recv(messageBuffer, size, 3000) != size: raise newException(ValueError, "didn't get all the bytes") From 6836d453988b2ad25783a3a9d64b0cc32cca5b0a Mon Sep 17 00:00:00 2001 From: Simon Hafner Date: Sun, 26 Apr 2015 08:42:28 +0500 Subject: [PATCH 12/29] findEPCPort defined, but not implemented --- compiler/nimsuggest/nimsuggest.nim | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/compiler/nimsuggest/nimsuggest.nim b/compiler/nimsuggest/nimsuggest.nim index 97bbf6ea3..6cc3bd49a 100644 --- a/compiler/nimsuggest/nimsuggest.nim +++ b/compiler/nimsuggest/nimsuggest.nim @@ -94,6 +94,9 @@ proc returnEPC(socket: var Socket, uid: string, s: SexpNode) = socket.send(response) socket.close() +proc findEPCPort(): int = + 98294 # guaranteed to be random + proc action(cmd: string) = template toggle(sw) = if sw in gGlobalOptions: @@ -178,7 +181,8 @@ proc serve() = stdoutSocket.send("\c\L") stdoutSocket.close() of mepc: - let port = 98294 # guaranteed to be random + let port = findEPCPort() + echo(port) var server = newSocket() server.bindaddr(port.Port, "localhost") var inp = "".TaintedString From 33edba665d45d148216d4a056d0588952e4547fc Mon Sep 17 00:00:00 2001 From: Simon Hafner Date: Sun, 26 Apr 2015 08:54:46 +0500 Subject: [PATCH 13/29] removed a few consts and rewired to IdeCmd --- compiler/options.nim | 19 +++++++++++++++++-- compiler/suggest.nim | 27 +++++++++++---------------- 2 files changed, 28 insertions(+), 18 deletions(-) diff --git a/compiler/options.nim b/compiler/options.nim index 65250f519..709959cc4 100644 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -83,11 +83,11 @@ type # please make sure we have under 32 options TGCMode* = enum # the selected GC gcNone, gcBoehm, gcMarkAndSweep, gcRefc, gcV2, gcGenerational - TIdeCmd* = enum + IdeCmd* = enum ideNone, ideSug, ideCon, ideDef, ideUse var - gIdeCmd*: TIdeCmd + gIdeCmd*: IdeCmd const ChecksOptions* = {optObjCheck, optFieldCheck, optRangeCheck, optNilCheck, @@ -395,3 +395,18 @@ template cnimdbg*: expr = p.module.module.fileIdx == gProjectMainIdx template pnimdbg*: expr = p.lex.fileIdx == gProjectMainIdx template lnimdbg*: expr = L.fileIdx == gProjectMainIdx +proc parseIdeCmd*(s: string): IdeCmd = + case s: + of "sug": ideSug + of "con": ideCon + of "def": ideDef + of "use": ideUse + else: ideNone + +proc `$`*(c: IdeCmd): string = + case c: + of ideSug: "sug" + of ideCon: "con" + of ideDef: "def" + of ideUse: "use" + of ideNone: "none" diff --git a/compiler/suggest.nim b/compiler/suggest.nim index 539d4bc5d..f1628db94 100644 --- a/compiler/suggest.nim +++ b/compiler/suggest.nim @@ -16,15 +16,10 @@ import algorithm, sequtils, strutils, ast, msgs, options, renderer, const sep = '\t' - sectionSuggest = "sug" - sectionDef = "def" - sectionContext = "con" - sectionUsage = "use" type - Section* = enum sug, def, con, use Suggest* = object - section*: Section + section*: IdeCmd qualifiedPath*: seq[string] filePath*: string line*: int # Starts at 1 @@ -41,7 +36,7 @@ var template origModuleName(m: PSym): string = m.name.s proc symToSuggest(s: PSym, isLocal: bool, section: string, li: TLineInfo): Suggest = - result.section = parseEnum[Section](section) + result.section = parseIdeCmd(section) if optIdeTerse in gGlobalOptions: if s.kind in routineKinds: result.symkind = parseEnum[TSymKind](renderTree(s.ast, {renderNoBody, renderNoComments, @@ -116,7 +111,7 @@ proc fieldVisible*(c: PContext, f: PSym): bool {.inline.} = proc suggestField(c: PContext, s: PSym, outputs: var int) = if filterSym(s) and fieldVisible(c, s): - suggestResult(symToSuggest(s, isLocal=true, sectionSuggest)) + suggestResult(symToSuggest(s, isLocal=true, $ideSug)) inc outputs template wholeSymTab(cond, section: expr) {.immediate.} = @@ -172,7 +167,7 @@ proc argsFit(c: PContext, candidate: PSym, n, nOrig: PNode): bool = proc suggestCall(c: PContext, n, nOrig: PNode, outputs: var int) = wholeSymTab(filterSym(it) and nameFits(c, it, n) and argsFit(c, it, n, nOrig), - sectionContext) + $ideCon) proc typeFits(c: PContext, s: PSym, firstArg: PType): bool {.inline.} = if s.typ != nil and sonsLen(s.typ) > 1 and s.typ.sons[1] != nil: @@ -189,7 +184,7 @@ proc typeFits(c: PContext, s: PSym, firstArg: PType): bool {.inline.} = proc suggestOperations(c: PContext, n: PNode, typ: PType, outputs: var int) = assert typ != nil - wholeSymTab(filterSymNoOpr(it) and typeFits(c, it, typ), sectionSuggest) + wholeSymTab(filterSymNoOpr(it) and typeFits(c, it, typ), $ideSug) proc suggestEverything(c: PContext, n: PNode, outputs: var int) = # do not produce too many symbols: @@ -198,7 +193,7 @@ proc suggestEverything(c: PContext, n: PNode, outputs: var int) = if scope == c.topLevelScope: isLocal = false for it in items(scope.symbols): if filterSym(it): - suggestResult(symToSuggest(it, isLocal = isLocal, sectionSuggest)) + suggestResult(symToSuggest(it, isLocal = isLocal, $ideSug)) inc outputs if scope == c.topLevelScope: break @@ -213,12 +208,12 @@ proc suggestFieldAccess(c: PContext, n: PNode, outputs: var int) = # all symbols accessible, because we are in the current module: for it in items(c.topLevelScope.symbols): if filterSym(it): - suggestResult(symToSuggest(it, isLocal=false, sectionSuggest)) + suggestResult(symToSuggest(it, isLocal=false, $ideSug)) inc outputs else: for it in items(n.sym.tab): if filterSym(it): - suggestResult(symToSuggest(it, isLocal=false, sectionSuggest)) + suggestResult(symToSuggest(it, isLocal=false, $ideSug)) inc outputs else: # fallback: @@ -295,16 +290,16 @@ var proc findUsages(info: TLineInfo; s: PSym) = if usageSym == nil and isTracked(info, s.name.s.len): usageSym = s - suggestResult(symToSuggest(s, isLocal=false, sectionUsage)) + suggestResult(symToSuggest(s, isLocal=false, $ideUse)) elif s == usageSym: if lastLineInfo != info: - suggestResult(symToSuggest(s, isLocal=false, sectionUsage, info)) + suggestResult(symToSuggest(s, isLocal=false, $ideUse, info)) lastLineInfo = info proc findDefinition(info: TLineInfo; s: PSym) = if s.isNil: return if isTracked(info, s.name.s.len): - suggestResult(symToSuggest(s, isLocal=false, sectionDef)) + suggestResult(symToSuggest(s, isLocal=false, $ideDef)) suggestQuit() proc ensureIdx[T](x: var T, y: int) = From bf4a92784e7c90018c6e8fbec3019df2a5dd5c23 Mon Sep 17 00:00:00 2001 From: Simon Hafner Date: Sun, 26 Apr 2015 08:56:22 +0500 Subject: [PATCH 14/29] final Section -> IdeCmd --- compiler/nimsuggest/nimsuggest.nim | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/compiler/nimsuggest/nimsuggest.nim b/compiler/nimsuggest/nimsuggest.nim index 6cc3bd49a..73325fa48 100644 --- a/compiler/nimsuggest/nimsuggest.nim +++ b/compiler/nimsuggest/nimsuggest.nim @@ -58,7 +58,7 @@ proc parseQuoted(cmd: string; outp: var string; start: int): int = let order: SexpNode = sexp(@["section", "symkind", "qualifiedPath", "filePath", "forth", "line", "column", "doc"].map(newSSymbol)) -proc sexp(s: Section): SexpNode = sexp($s) +proc sexp(s: IdeCmd): SexpNode = sexp($s) proc sexp(s: TSymKind): SexpNode = sexp($s) @@ -80,7 +80,7 @@ proc sexp(s: seq[Suggest]): SexpNode = proc listEPC(): SexpNode = discard -proc executeEPC(section: Section, args: SexpNode) = +proc executeEPC(cmd: IdeCmd, args: SexpNode) = let file = args[0].getStr line = args[1].getNum @@ -211,9 +211,9 @@ proc serve() = of "call": let uid = body[0].getStr - section = parseEnum[Section](body[1].getStr) + cmd = parseIdeCmd(body[1].getStr) args = body[2] - executeEPC(section, args) + executeEPC(cmd, args) returnEPC(client, uid, sexp(results)) of "return": raise newException(ValueError, "no return expected") From 7b7188a8ade72379d9d04c5ad3fdfd0aa0e11337 Mon Sep 17 00:00:00 2001 From: Simon Hafner Date: Sun, 26 Apr 2015 09:20:55 +0500 Subject: [PATCH 15/29] proc execute / return-error handling --- compiler/nimsuggest/nimsuggest.nim | 108 ++++++++++++++++------------- 1 file changed, 59 insertions(+), 49 deletions(-) diff --git a/compiler/nimsuggest/nimsuggest.nim b/compiler/nimsuggest/nimsuggest.nim index 73325fa48..2f95ceaae 100644 --- a/compiler/nimsuggest/nimsuggest.nim +++ b/compiler/nimsuggest/nimsuggest.nim @@ -45,6 +45,9 @@ const "type 'debug' to toggle debug mode on/off\n" & "type 'terse' to toggle terse mode on/off" +type + EUnexpectedCommand = object of Exception + proc parseQuoted(cmd: string; outp: var string; start: int): int = var i = start i += skipWhitespace(cmd, i) @@ -80,16 +83,38 @@ proc sexp(s: seq[Suggest]): SexpNode = proc listEPC(): SexpNode = discard +proc execute(cmd: IdeCmd, file, dirtyfile: string, line, col: int) = + gIdeCmd = cmd + if cmd == ideUse: + modules.resetAllModules() + var isKnownFile = true + let dirtyIdx = file.fileInfoIdx(isKnownFile) + + if dirtyfile.len != 0: msgs.setDirtyFile(dirtyIdx, dirtyfile) + else: msgs.setDirtyFile(dirtyIdx, nil) + + resetModule dirtyIdx + if dirtyIdx != gProjectMainIdx: + resetModule gProjectMainIdx + + gTrackPos = newLineInfo(dirtyIdx, line, col) + #echo dirtyfile, gDirtyBufferIdx, " project ", gProjectMainIdx + gErrorCounter = 0 + if not isKnownFile: + compileProject(dirtyIdx) + else: + compileProject() + proc executeEPC(cmd: IdeCmd, args: SexpNode) = let file = args[0].getStr line = args[1].getNum column = args[2].getNum dirtyfile = args[3].getStr(nil) - discard + execute(cmd, file, dirtyfile, int(line), int(column)) -proc returnEPC(socket: var Socket, uid: string, s: SexpNode) = - let response = $convertSexp([newSSymbol("return"), [uid, s]]) +proc returnEPC(socket: var Socket, uid: string, s: SexpNode, return_symbol = "return") = + let response = $convertSexp([newSSymbol(return_symbol), [uid, s]]) socket.send(toHex(len(response), 6)) socket.send(response) socket.close() @@ -97,7 +122,7 @@ proc returnEPC(socket: var Socket, uid: string, s: SexpNode) = proc findEPCPort(): int = 98294 # guaranteed to be random -proc action(cmd: string) = +proc parseCmdLine(cmd: string) = template toggle(sw) = if sw in gGlobalOptions: excl(gGlobalOptions, sw) @@ -115,9 +140,7 @@ proc action(cmd: string) = of "sug": gIdeCmd = ideSug of "con": gIdeCmd = ideCon of "def": gIdeCmd = ideDef - of "use": - modules.resetAllModules() - gIdeCmd = ideUse + of "use": gIdeCmd = ideUse of "quit": quit() of "debug": toggle optIdeDebug of "terse": toggle optIdeTerse @@ -133,23 +156,7 @@ proc action(cmd: string) = i += skipWhile(cmd, seps, i) i += parseInt(cmd, col, i) - var isKnownFile = true - if orig.len == 0: err() - let dirtyIdx = orig.fileInfoIdx(isKnownFile) - - if dirtyfile.len != 0: msgs.setDirtyFile(dirtyIdx, dirtyfile) - else: msgs.setDirtyFile(dirtyIdx, nil) - - resetModule dirtyIdx - if dirtyIdx != gProjectMainIdx: - resetModule gProjectMainIdx - gTrackPos = newLineInfo(dirtyIdx, line, col) - #echo dirtyfile, gDirtyBufferIdx, " project ", gProjectMainIdx - gErrorCounter = 0 - if not isKnownFile: - compileProject(dirtyIdx) - else: - compileProject() + execute(gIdeCmd, orig, dirtyfile, line, col) proc serve() = # do not stop after the first error: @@ -159,7 +166,7 @@ proc serve() = echo Help var line = "" while readLineFromStdin("> ", line): - action line + parseCmdLine line echo "" flushFile(stdout) of mtcp: @@ -176,7 +183,7 @@ proc serve() = accept(server, stdoutSocket) stdoutSocket.readLine(inp) - action inp.string + parseCmdLine inp.string stdoutSocket.send("\c\L") stdoutSocket.close() @@ -189,11 +196,7 @@ proc serve() = server.listen() echo(port) while true: - var results: seq[Suggest] = @[] var client = newSocket() - suggest.suggestionResultHook = proc (s: Suggest) = - results.add(s) - accept(server, client) var sizeHex = "" if client.recv(sizeHex, 6, 1000) != 6: @@ -207,25 +210,32 @@ proc serve() = let message = parseSexp($messageBuffer) let messageType = message[0].getSymbol let body = message[1] - case messageType: - of "call": - let - uid = body[0].getStr - cmd = parseIdeCmd(body[1].getStr) - args = body[2] - executeEPC(cmd, args) - returnEPC(client, uid, sexp(results)) - of "return": - raise newException(ValueError, "no return expected") - of "return-error": - raise newException(ValueError, "no return expected") - of "epc-error": - stderr.writeln("recieved epc error: " & $messageBuffer) - raise newException(ValueError, "epc error") - of "methods": - returnEPC(client, body[0].getStr, listEPC()) - else: - raise newException(ValueError, "unexpected call: " & messageType) + try: + case messageType: + of "call": + var results: seq[Suggest] = @[] + suggest.suggestionResultHook = proc (s: Suggest) = + results.add(s) + + let + uid = body[0].getStr + cmd = parseIdeCmd(body[1].getStr) + args = body[2] + executeEPC(cmd, args) + returnEPC(client, uid, sexp(results)) + of "return": + raise newException(EUnexpectedCommand, "no return expected") + of "return-error": + raise newException(EUnexpectedCommand, "no return expected") + of "epc-error": + stderr.writeln("recieved epc error: " & $messageBuffer) + raise newException(IOError, "epc error") + of "methods": + returnEPC(client, body[0].getStr, listEPC()) + else: + raise newException(EUnexpectedCommand, "unexpected call: " & messageType) + except: + returnEPC(client, body[0].getStr, sexp(getCurrentExceptionMsg()), "return-error") proc mainCommand = registerPass verbosePass From 53bc541666565ebb6103afea618b0e680a464dd0 Mon Sep 17 00:00:00 2001 From: Simon Hafner Date: Sun, 26 Apr 2015 09:44:22 +0500 Subject: [PATCH 16/29] rough guess how methods should look like --- compiler/nimsuggest/nimsuggest.nim | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/compiler/nimsuggest/nimsuggest.nim b/compiler/nimsuggest/nimsuggest.nim index 2f95ceaae..9f672d49b 100644 --- a/compiler/nimsuggest/nimsuggest.nim +++ b/compiler/nimsuggest/nimsuggest.nim @@ -81,7 +81,18 @@ proc sexp(s: seq[Suggest]): SexpNode = result = sexp(s) proc listEPC(): SexpNode = - discard + let + argspecs = sexp("file line column dirtyfile".split(" ").map(newSSymbol)) + docstring = sexp("line starts at 1, column at 0, dirtyfile is optional") + result = newSList() + for command in ["sug", "con", "def", "use"]: + let + cmd = sexp(command) + methodDesc = newSList() + methodDesc.add(cmd) + methodDesc.add(argspecs) + methodDesc.add(docstring) + result.add(methodDesc) proc execute(cmd: IdeCmd, file, dirtyfile: string, line, col: int) = gIdeCmd = cmd From 00fafd1dbe72e3f6c50f981d286e0c753207bc99 Mon Sep 17 00:00:00 2001 From: Simon Hafner Date: Mon, 27 Apr 2015 10:25:45 +0500 Subject: [PATCH 17/29] Fixed a few socket things --- compiler/nimsuggest/nimsuggest.nim | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/compiler/nimsuggest/nimsuggest.nim b/compiler/nimsuggest/nimsuggest.nim index 9f672d49b..4e4925fb8 100644 --- a/compiler/nimsuggest/nimsuggest.nim +++ b/compiler/nimsuggest/nimsuggest.nim @@ -9,7 +9,7 @@ ## Nimsuggest is a tool that helps to give editors IDE like capabilities. -import strutils, os, parseopt, parseutils, sequtils +import strutils, os, parseopt, parseutils, sequtils, net import options, commands, modules, sem, passes, passaux, msgs, nimconf, extccomp, condsyms, lists, net, rdstdin, sexp, suggest, ast @@ -131,7 +131,7 @@ proc returnEPC(socket: var Socket, uid: string, s: SexpNode, return_symbol = "re socket.close() proc findEPCPort(): int = - 98294 # guaranteed to be random + 55822 # guaranteed to be random proc parseCmdLine(cmd: string) = template toggle(sw) = @@ -200,9 +200,8 @@ proc serve() = stdoutSocket.close() of mepc: let port = findEPCPort() - echo(port) var server = newSocket() - server.bindaddr(port.Port, "localhost") + server.bindaddr(Port(port), "localhost") var inp = "".TaintedString server.listen() echo(port) From 39378823ae5c0df8d2baa4e1ee07b71786736bef Mon Sep 17 00:00:00 2001 From: Simon Hafner Date: Mon, 27 Apr 2015 20:06:17 +0500 Subject: [PATCH 18/29] epc waits on recv, not on accept --- compiler/nimsuggest/nimsuggest.nim | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/compiler/nimsuggest/nimsuggest.nim b/compiler/nimsuggest/nimsuggest.nim index 4e4925fb8..7e0d3a399 100644 --- a/compiler/nimsuggest/nimsuggest.nim +++ b/compiler/nimsuggest/nimsuggest.nim @@ -205,17 +205,18 @@ proc serve() = var inp = "".TaintedString server.listen() echo(port) + var client = newSocket() + # Wait for connection + accept(server, client) while true: - var client = newSocket() - accept(server, client) var sizeHex = "" - if client.recv(sizeHex, 6, 1000) != 6: + if client.recv(sizeHex, 6) != 6: raise newException(ValueError, "didn't get all the hexbytes") var size = 0 if parseHex(sizeHex, size) == 0: raise newException(ValueError, "invalid size hex: " & $sizeHex) var messageBuffer = "" - if client.recv(messageBuffer, size, 3000) != size: + if client.recv(messageBuffer, size) != size: raise newException(ValueError, "didn't get all the bytes") let message = parseSexp($messageBuffer) let messageType = message[0].getSymbol From a1ee169d09b8d8c001e868d889c517fad4d1efaa Mon Sep 17 00:00:00 2001 From: Simon Hafner Date: Sat, 2 May 2015 05:28:53 +0500 Subject: [PATCH 19/29] implemented nextFreePort for nimsuggest --- compiler/nimsuggest/nimsuggest.nim | 25 +++++++++++++++++++++---- 1 file changed, 21 insertions(+), 4 deletions(-) diff --git a/compiler/nimsuggest/nimsuggest.nim b/compiler/nimsuggest/nimsuggest.nim index 7e0d3a399..00b3bcbfd 100644 --- a/compiler/nimsuggest/nimsuggest.nim +++ b/compiler/nimsuggest/nimsuggest.nim @@ -13,6 +13,11 @@ import strutils, os, parseopt, parseutils, sequtils, net import options, commands, modules, sem, passes, passaux, msgs, nimconf, extccomp, condsyms, lists, net, rdstdin, sexp, suggest, ast +when defined(windows): + import winlean +else: + import posix + const Usage = """ Nimsuggest - Tool to give every editor IDE like capabilities for Nim Usage: @@ -130,8 +135,21 @@ proc returnEPC(socket: var Socket, uid: string, s: SexpNode, return_symbol = "re socket.send(response) socket.close() -proc findEPCPort(): int = - 55822 # guaranteed to be random +proc nextFreePort(server: Socket, host: string, start = 30000): int = + result = start + while true: + try: + server.bindaddr(Port(result), host) + return + except OsError: + when defined(windows): + let checkFor = WSAEADDRINUSE.OSErrorCode + else: + let checkFor = EADDRINUSE.OSErrorCode + if osLastError() != checkFor: + raise getCurrentException() + else: + result += 1 proc parseCmdLine(cmd: string) = template toggle(sw) = @@ -199,9 +217,8 @@ proc serve() = stdoutSocket.send("\c\L") stdoutSocket.close() of mepc: - let port = findEPCPort() var server = newSocket() - server.bindaddr(Port(port), "localhost") + let port = nextFreePort(server, "localhost") var inp = "".TaintedString server.listen() echo(port) From 7dfa93d3c0dc17541b5b54654f2af3da8bed91f5 Mon Sep 17 00:00:00 2001 From: Simon Hafner Date: Sat, 2 May 2015 08:04:17 +0500 Subject: [PATCH 20/29] fixed first round of bugs --- compiler/nimsuggest/nimsuggest.nim | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/compiler/nimsuggest/nimsuggest.nim b/compiler/nimsuggest/nimsuggest.nim index 00b3bcbfd..c14c89e16 100644 --- a/compiler/nimsuggest/nimsuggest.nim +++ b/compiler/nimsuggest/nimsuggest.nim @@ -133,7 +133,6 @@ proc returnEPC(socket: var Socket, uid: string, s: SexpNode, return_symbol = "re let response = $convertSexp([newSSymbol(return_symbol), [uid, s]]) socket.send(toHex(len(response), 6)) socket.send(response) - socket.close() proc nextFreePort(server: Socket, host: string, start = 30000): int = result = start @@ -235,9 +234,9 @@ proc serve() = var messageBuffer = "" if client.recv(messageBuffer, size) != size: raise newException(ValueError, "didn't get all the bytes") - let message = parseSexp($messageBuffer) - let messageType = message[0].getSymbol - let body = message[1] + let + message = parseSexp($messageBuffer) + messageType = message[0].getSymbol try: case messageType: of "call": @@ -246,9 +245,9 @@ proc serve() = results.add(s) let - uid = body[0].getStr - cmd = parseIdeCmd(body[1].getStr) - args = body[2] + uid = message[1].getStr + cmd = parseIdeCmd(message[2].getStr) + args = message[3] executeEPC(cmd, args) returnEPC(client, uid, sexp(results)) of "return": @@ -259,11 +258,11 @@ proc serve() = stderr.writeln("recieved epc error: " & $messageBuffer) raise newException(IOError, "epc error") of "methods": - returnEPC(client, body[0].getStr, listEPC()) + returnEPC(client, message[1].getStr, listEPC()) else: raise newException(EUnexpectedCommand, "unexpected call: " & messageType) except: - returnEPC(client, body[0].getStr, sexp(getCurrentExceptionMsg()), "return-error") + returnEPC(client, message[1].getStr, sexp(getCurrentException().getStackTrace()), "return-error") proc mainCommand = registerPass verbosePass From b50090797804ea3f669932c4eb9a26fe4899842c Mon Sep 17 00:00:00 2001 From: Simon Hafner Date: Sat, 2 May 2015 08:24:05 +0500 Subject: [PATCH 21/29] some more bugs in epc fixed - removed try --- compiler/nimsuggest/nimsuggest.nim | 59 ++++++++++++++++-------------- 1 file changed, 31 insertions(+), 28 deletions(-) diff --git a/compiler/nimsuggest/nimsuggest.nim b/compiler/nimsuggest/nimsuggest.nim index c14c89e16..68d89e6e9 100644 --- a/compiler/nimsuggest/nimsuggest.nim +++ b/compiler/nimsuggest/nimsuggest.nim @@ -83,7 +83,9 @@ proc sexp(s: Suggest): SexpNode = ]) proc sexp(s: seq[Suggest]): SexpNode = - result = sexp(s) + result = newSList() + for sug in s: + result.add(sexp(sug)) proc listEPC(): SexpNode = let @@ -126,11 +128,13 @@ proc executeEPC(cmd: IdeCmd, args: SexpNode) = file = args[0].getStr line = args[1].getNum column = args[2].getNum + var dirtyfile = "" + if len(args) > 3: dirtyfile = args[3].getStr(nil) execute(cmd, file, dirtyfile, int(line), int(column)) -proc returnEPC(socket: var Socket, uid: string, s: SexpNode, return_symbol = "return") = - let response = $convertSexp([newSSymbol(return_symbol), [uid, s]]) +proc returnEPC(socket: var Socket, uid: BiggestInt, s: SexpNode, return_symbol = "return") = + let response = $convertSexp([newSSymbol(return_symbol), uid, s]) socket.send(toHex(len(response), 6)) socket.send(response) @@ -237,32 +241,31 @@ proc serve() = let message = parseSexp($messageBuffer) messageType = message[0].getSymbol - try: - case messageType: - of "call": - var results: seq[Suggest] = @[] - suggest.suggestionResultHook = proc (s: Suggest) = - results.add(s) + echo("received message: " & $message) + case messageType: + of "call": + var results: seq[Suggest] = @[] + suggest.suggestionResultHook = proc (s: Suggest) = + echo(s) + results.add(s) - let - uid = message[1].getStr - cmd = parseIdeCmd(message[2].getStr) - args = message[3] - executeEPC(cmd, args) - returnEPC(client, uid, sexp(results)) - of "return": - raise newException(EUnexpectedCommand, "no return expected") - of "return-error": - raise newException(EUnexpectedCommand, "no return expected") - of "epc-error": - stderr.writeln("recieved epc error: " & $messageBuffer) - raise newException(IOError, "epc error") - of "methods": - returnEPC(client, message[1].getStr, listEPC()) - else: - raise newException(EUnexpectedCommand, "unexpected call: " & messageType) - except: - returnEPC(client, message[1].getStr, sexp(getCurrentException().getStackTrace()), "return-error") + let + uid = message[1].getNum + cmd = parseIdeCmd(message[2].getStr) + args = message[3] + executeEPC(cmd, args) + returnEPC(client, uid, sexp(results)) + of "return": + raise newException(EUnexpectedCommand, "no return expected") + of "return-error": + raise newException(EUnexpectedCommand, "no return expected") + of "epc-error": + stderr.writeln("recieved epc error: " & $messageBuffer) + raise newException(IOError, "epc error") + of "methods": + returnEPC(client, message[1].getNum, listEPC()) + else: + raise newException(EUnexpectedCommand, "unexpected call: " & messageType) proc mainCommand = registerPass verbosePass From 0ff5339590b0acde4c344af7799232ae2d44737f Mon Sep 17 00:00:00 2001 From: Simon Hafner Date: Sat, 2 May 2015 08:34:30 +0500 Subject: [PATCH 22/29] removed debug echos --- compiler/nimsuggest/nimsuggest.nim | 2 -- 1 file changed, 2 deletions(-) diff --git a/compiler/nimsuggest/nimsuggest.nim b/compiler/nimsuggest/nimsuggest.nim index 68d89e6e9..636d14e7f 100644 --- a/compiler/nimsuggest/nimsuggest.nim +++ b/compiler/nimsuggest/nimsuggest.nim @@ -241,12 +241,10 @@ proc serve() = let message = parseSexp($messageBuffer) messageType = message[0].getSymbol - echo("received message: " & $message) case messageType: of "call": var results: seq[Suggest] = @[] suggest.suggestionResultHook = proc (s: Suggest) = - echo(s) results.add(s) let From 2522049ed2991cf9c8c8c7026646e16472125271 Mon Sep 17 00:00:00 2001 From: Simon Hafner Date: Sat, 2 May 2015 09:14:02 +0500 Subject: [PATCH 23/29] method arrives as symbol --- compiler/nimsuggest/nimsuggest.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/nimsuggest/nimsuggest.nim b/compiler/nimsuggest/nimsuggest.nim index 636d14e7f..6975c87cc 100644 --- a/compiler/nimsuggest/nimsuggest.nim +++ b/compiler/nimsuggest/nimsuggest.nim @@ -249,7 +249,7 @@ proc serve() = let uid = message[1].getNum - cmd = parseIdeCmd(message[2].getStr) + cmd = parseIdeCmd(message[2].getSymbol) args = message[3] executeEPC(cmd, args) returnEPC(client, uid, sexp(results)) From 1e78c5445d2d2bd7cc5f39579dd8ff0f8bbcb65c Mon Sep 17 00:00:00 2001 From: Simon Hafner Date: Sat, 2 May 2015 09:22:26 +0500 Subject: [PATCH 24/29] manually reapplied d40098ad --- compiler/nimsuggest/nimsuggest.nim | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/compiler/nimsuggest/nimsuggest.nim b/compiler/nimsuggest/nimsuggest.nim index 6975c87cc..3f8943016 100644 --- a/compiler/nimsuggest/nimsuggest.nim +++ b/compiler/nimsuggest/nimsuggest.nim @@ -183,12 +183,13 @@ proc parseCmdLine(cmd: string) = if cmd[i] == ';': i = parseQuoted(cmd, dirtyfile, i+1) i += skipWhile(cmd, seps, i) - var line, col = -1 + var line = -1 + var col = 0 i += parseInt(cmd, line, i) i += skipWhile(cmd, seps, i) i += parseInt(cmd, col, i) - execute(gIdeCmd, orig, dirtyfile, line, col) + execute(gIdeCmd, orig, dirtyfile, line, col-1) proc serve() = # do not stop after the first error: From 2474c1bb111b38ddef64659c893722b357a27384 Mon Sep 17 00:00:00 2001 From: Simon Hafner Date: Fri, 8 May 2015 04:45:48 +0500 Subject: [PATCH 25/29] import sigmatch, not suggest in nimsuggest --- compiler/nimsuggest/nimsuggest.nim | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/compiler/nimsuggest/nimsuggest.nim b/compiler/nimsuggest/nimsuggest.nim index 3f8943016..7ba3379c3 100644 --- a/compiler/nimsuggest/nimsuggest.nim +++ b/compiler/nimsuggest/nimsuggest.nim @@ -10,8 +10,11 @@ ## Nimsuggest is a tool that helps to give editors IDE like capabilities. import strutils, os, parseopt, parseutils, sequtils, net +# Do NOT import suggest. It will lead to wierd bugs with +# suggestionResultHook, because suggest.nim is included by sigmatch. +# So we import that one instead. import options, commands, modules, sem, passes, passaux, msgs, nimconf, - extccomp, condsyms, lists, net, rdstdin, sexp, suggest, ast + extccomp, condsyms, lists, net, rdstdin, sexp, sigmatch, ast when defined(windows): import winlean @@ -116,7 +119,6 @@ proc execute(cmd: IdeCmd, file, dirtyfile: string, line, col: int) = resetModule gProjectMainIdx gTrackPos = newLineInfo(dirtyIdx, line, col) - #echo dirtyfile, gDirtyBufferIdx, " project ", gProjectMainIdx gErrorCounter = 0 if not isKnownFile: compileProject(dirtyIdx) @@ -245,7 +247,7 @@ proc serve() = case messageType: of "call": var results: seq[Suggest] = @[] - suggest.suggestionResultHook = proc (s: Suggest) = + suggestionResultHook = proc (s: Suggest) = results.add(s) let From c06672451ad34e1baf727dc4de68a0fa0566dae3 Mon Sep 17 00:00:00 2001 From: Simon Hafner Date: Fri, 8 May 2015 19:47:17 +0500 Subject: [PATCH 26/29] removed unnecessary deps --- compiler/suggest.nim | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/compiler/suggest.nim b/compiler/suggest.nim index f1628db94..a79d08f71 100644 --- a/compiler/suggest.nim +++ b/compiler/suggest.nim @@ -11,8 +11,7 @@ # included from sigmatch.nim -import algorithm, sequtils, strutils, ast, msgs, options, renderer, - types, docgen, lexer, semdata, astalgo, idents, sigmatch, lookups +import algorithm, sequtils const sep = '\t' From 13e8547fa7825aef0c8a07c67dcad78921c5a820 Mon Sep 17 00:00:00 2001 From: Simon Hafner Date: Fri, 8 May 2015 19:47:34 +0500 Subject: [PATCH 27/29] hardcoded order --- compiler/nimsuggest/nimsuggest.nim | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/compiler/nimsuggest/nimsuggest.nim b/compiler/nimsuggest/nimsuggest.nim index 7ba3379c3..ad3b5bf0d 100644 --- a/compiler/nimsuggest/nimsuggest.nim +++ b/compiler/nimsuggest/nimsuggest.nim @@ -65,15 +65,13 @@ proc parseQuoted(cmd: string; outp: var string; start: int): int = i += parseUntil(cmd, outp, seps, i) result = i -# make sure it's in the same order as the proc below -let order: SexpNode = - sexp(@["section", "symkind", "qualifiedPath", "filePath", "forth", "line", "column", "doc"].map(newSSymbol)) - proc sexp(s: IdeCmd): SexpNode = sexp($s) proc sexp(s: TSymKind): SexpNode = sexp($s) proc sexp(s: Suggest): SexpNode = + # If you change the oder here, make sure to change it over in + # nim-mode.el too. result = convertSexp([ s.section, s.symkind, From c56a00523861ec3d680478f0facf4a05ea753969 Mon Sep 17 00:00:00 2001 From: Simon Hafner Date: Wed, 13 May 2015 18:19:39 +0500 Subject: [PATCH 28/29] nextFreePort -> connectToNextFreePort --- compiler/nimsuggest/nimsuggest.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/compiler/nimsuggest/nimsuggest.nim b/compiler/nimsuggest/nimsuggest.nim index ad3b5bf0d..3b4aa658d 100644 --- a/compiler/nimsuggest/nimsuggest.nim +++ b/compiler/nimsuggest/nimsuggest.nim @@ -138,7 +138,7 @@ proc returnEPC(socket: var Socket, uid: BiggestInt, s: SexpNode, return_symbol = socket.send(toHex(len(response), 6)) socket.send(response) -proc nextFreePort(server: Socket, host: string, start = 30000): int = +proc connectToNextFreePort(server: Socket, host: string, start = 30000): int = result = start while true: try: @@ -222,7 +222,7 @@ proc serve() = stdoutSocket.close() of mepc: var server = newSocket() - let port = nextFreePort(server, "localhost") + let port = connectToNextFreePort(server, "localhost") var inp = "".TaintedString server.listen() echo(port) From 5e97780a9411a9702c285acec7922bfe865b8897 Mon Sep 17 00:00:00 2001 From: Simon Hafner Date: Wed, 13 May 2015 18:34:29 +0500 Subject: [PATCH 29/29] changed from parseEnum to direct copy Let's hope this doesn't break anything --- compiler/suggest.nim | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/compiler/suggest.nim b/compiler/suggest.nim index a79d08f71..659f1fa16 100644 --- a/compiler/suggest.nim +++ b/compiler/suggest.nim @@ -37,11 +37,7 @@ template origModuleName(m: PSym): string = m.name.s proc symToSuggest(s: PSym, isLocal: bool, section: string, li: TLineInfo): Suggest = result.section = parseIdeCmd(section) if optIdeTerse in gGlobalOptions: - if s.kind in routineKinds: - result.symkind = parseEnum[TSymKind](renderTree(s.ast, {renderNoBody, renderNoComments, - renderDocComments, renderNoPragmas})) - else: - result.symkind = parseEnum[TSymKind](s.name.s) + result.symkind = s.kind result.filePath = toFullPath(li) result.line = toLinenumber(li) result.column = toColumn(li)