jyapayne/Nim
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

next steps for the new parser/grammar

Browse source
This commit is contained in:
Araq 2013-04-20 01:59:39 +02:00
commit 2796121dd7
6 changed files with 286 additions and 266 deletions
Download patch file Download diff file Expand all files Collapse all files

2
compiler/docgen.nim
View file

@ -237,7 +237,7 @@ proc genItem(d: PDoc, n, nameNode: PNode, k: TSymKind) =
of tkSymbol: of tkSymbol:
dispA(result, "<span class=\"Identifier\">$1</span>", dispA(result, "<span class=\"Identifier\">$1</span>",
"\\spanIdentifier{$1}", [toRope(esc(d.target, literal))]) "\\spanIdentifier{$1}", [toRope(esc(d.target, literal))])
of tkInd, tkSpaces, tkInvalid: of tkSpaces, tkInvalid:
app(result, literal) app(result, literal)
of tkParLe, tkParRi, tkBracketLe, tkBracketRi, tkCurlyLe, tkCurlyRi, of tkParLe, tkParRi, tkBracketLe, tkBracketRi, tkCurlyLe, tkCurlyRi,
tkBracketDotLe, tkBracketDotRi, tkCurlyDotLe, tkCurlyDotRi, tkParDotLe, tkBracketDotLe, tkBracketDotRi, tkCurlyDotLe, tkCurlyDotRi, tkParDotLe,

52
compiler/lexer.nim
View file

@ -58,7 +58,7 @@ type
tkParDotLe, tkParDotRi, # (. and .) tkParDotLe, tkParDotRi, # (. and .)
tkComma, tkSemiColon, tkComma, tkSemiColon,
tkColon, tkColonColon, tkEquals, tkDot, tkDotDot, tkColon, tkColonColon, tkEquals, tkDot, tkDotDot,
tkOpr, tkComment, tkAccent, tkInd, tkOpr, tkComment, tkAccent,
tkSpaces, tkInfixOpr, tkPrefixOpr, tkPostfixOpr, tkSpaces, tkInfixOpr, tkPrefixOpr, tkPostfixOpr,
TTokTypes* = set[TTokType] TTokTypes* = set[TTokType]
@ -90,7 +90,7 @@ const
")", "[", "]", "{", "}", "[.", ".]", "{.", ".}", "(.", ".)", ")", "[", "]", "{", "}", "[.", ".]", "{.", ".}", "(.", ".)",
",", ";", ",", ";",
":", "::", "=", ".", "..", ":", "::", "=", ".", "..",
"tkOpr", "tkComment", "`", "[new indentation]", "tkOpr", "tkComment", "`",
"tkSpaces", "tkInfixOpr", "tkSpaces", "tkInfixOpr",
"tkPrefixOpr", "tkPostfixOpr"] "tkPrefixOpr", "tkPostfixOpr"]
@ -154,7 +154,7 @@ proc tokToStr*(tok: TToken): string =
of tkIntLit..tkInt64Lit: result = $tok.iNumber of tkIntLit..tkInt64Lit: result = $tok.iNumber
of tkFloatLit..tkFloat64Lit: result = $tok.fNumber of tkFloatLit..tkFloat64Lit: result = $tok.fNumber
of tkInvalid, tkStrLit..tkCharLit, tkComment: result = tok.literal of tkInvalid, tkStrLit..tkCharLit, tkComment: result = tok.literal
of tkParLe..tkColon, tkEof, tkInd, tkAccent: of tkParLe..tkColon, tkEof, tkAccent:
result = tokTypeToStr[tok.tokType] result = tokTypeToStr[tok.tokType]
else: else:
if tok.ident != nil: if tok.ident != nil:
@ -411,9 +411,10 @@ proc GetNumber(L: var TLexer): TToken =
result.tokType = tkInt64Lit result.tokType = tkInt64Lit
elif result.tokType != tkInt64Lit: elif result.tokType != tkInt64Lit:
lexMessage(L, errInvalidNumber, result.literal) lexMessage(L, errInvalidNumber, result.literal)
except EInvalidValue: lexMessage(L, errInvalidNumber, result.literal) except EInvalidValue:
except EOverflow: lexMessage(L, errNumberOutOfRange, result.literal) lexMessage(L, errInvalidNumber, result.literal)
except EOutOfRange: lexMessage(L, errNumberOutOfRange, result.literal) except EOverflow, EOutOfRange:
lexMessage(L, errNumberOutOfRange, result.literal)
L.bufpos = endpos L.bufpos = endpos
proc handleHexChar(L: var TLexer, xi: var int) = proc handleHexChar(L: var TLexer, xi: var int) =
@ -628,10 +629,6 @@ proc getOperator(L: var TLexer, tok: var TToken) =
Inc(pos) Inc(pos)
endOperator(L, tok, pos, h) endOperator(L, tok, pos, h)
proc handleIndentation(tok: var TToken, indent: int) {.inline.} =
tok.indent = indent
tok.tokType = tkInd
proc scanComment(L: var TLexer, tok: var TToken) = proc scanComment(L: var TLexer, tok: var TToken) =
var pos = L.bufpos var pos = L.bufpos
var buf = L.buf var buf = L.buf
@ -680,7 +677,7 @@ proc skip(L: var TLexer, tok: var TToken) =
Inc(pos) Inc(pos)
of Tabulator: of Tabulator:
lexMessagePos(L, errTabulatorsAreNotAllowed, pos) lexMessagePos(L, errTabulatorsAreNotAllowed, pos)
inc(pos) # BUGFIX inc(pos)
of CR, LF: of CR, LF:
pos = HandleCRLF(L, pos) pos = HandleCRLF(L, pos)
buf = L.buf buf = L.buf
@ -688,8 +685,8 @@ proc skip(L: var TLexer, tok: var TToken) =
while buf[pos] == ' ': while buf[pos] == ' ':
Inc(pos) Inc(pos)
Inc(indent) Inc(indent)
if (buf[pos] > ' '): if buf[pos] > ' ':
handleIndentation(tok, indent) tok.indent = indent
break break
else: else:
break # EndOfFile also leaves the loop break # EndOfFile also leaves the loop
@ -698,17 +695,14 @@ proc skip(L: var TLexer, tok: var TToken) =
proc rawGetTok(L: var TLexer, tok: var TToken) = proc rawGetTok(L: var TLexer, tok: var TToken) =
fillToken(tok) fillToken(tok)
if L.indentAhead >= 0: if L.indentAhead >= 0:
handleIndentation(tok, L.indentAhead) tok.indent = L.indentAhead
L.indentAhead = - 1 L.indentAhead = -1
return else:
tok.indent = -1
skip(L, tok) skip(L, tok)
# got an documentation comment or tkIndent, return that:
if tok.toktype != tkInvalid: return
var c = L.buf[L.bufpos] var c = L.buf[L.bufpos]
if c in SymStartChars - {'r', 'R', 'l'}: if c in SymStartChars - {'r', 'R', 'l'}:
getSymbol(L, tok) getSymbol(L, tok)
elif c in {'0'..'9'}:
tok = getNumber(L)
else: else:
case c case c
of '#': of '#':
@ -727,7 +721,7 @@ proc rawGetTok(L: var TLexer, tok: var TToken) =
of 'l': of 'l':
# if we parsed exactly one character and its a small L (l), this # if we parsed exactly one character and its a small L (l), this
# is treated as a warning because it may be confused with the number 1 # is treated as a warning because it may be confused with the number 1
if not (L.buf[L.bufpos + 1] in (SymChars + {'_'})): if L.buf[L.bufpos+1] notin (SymChars + {'_'}):
lexMessage(L, warnSmallLshouldNotBeUsed) lexMessage(L, warnSmallLshouldNotBeUsed)
getSymbol(L, tok) getSymbol(L, tok)
of 'r', 'R': of 'r', 'R':
@ -738,7 +732,7 @@ proc rawGetTok(L: var TLexer, tok: var TToken) =
getSymbol(L, tok) getSymbol(L, tok)
of '(': of '(':
Inc(L.bufpos) Inc(L.bufpos)
if (L.buf[L.bufPos] == '.') and (L.buf[L.bufPos + 1] != '.'): if L.buf[L.bufPos] == '.' and L.buf[L.bufPos+1] != '.':
tok.toktype = tkParDotLe tok.toktype = tkParDotLe
Inc(L.bufpos) Inc(L.bufpos)
else: else:
@ -748,7 +742,7 @@ proc rawGetTok(L: var TLexer, tok: var TToken) =
Inc(L.bufpos) Inc(L.bufpos)
of '[': of '[':
Inc(L.bufpos) Inc(L.bufpos)
if (L.buf[L.bufPos] == '.') and (L.buf[L.bufPos + 1] != '.'): if L.buf[L.bufPos] == '.' and L.buf[L.bufPos+1] != '.':
tok.toktype = tkBracketDotLe tok.toktype = tkBracketDotLe
Inc(L.bufpos) Inc(L.bufpos)
else: else:
@ -757,20 +751,20 @@ proc rawGetTok(L: var TLexer, tok: var TToken) =
tok.toktype = tkBracketRi tok.toktype = tkBracketRi
Inc(L.bufpos) Inc(L.bufpos)
of '.': of '.':
if L.buf[L.bufPos + 1] == ']': if L.buf[L.bufPos+1] == ']':
tok.tokType = tkBracketDotRi tok.tokType = tkBracketDotRi
Inc(L.bufpos, 2) Inc(L.bufpos, 2)
elif L.buf[L.bufPos + 1] == '}': elif L.buf[L.bufPos+1] == '}':
tok.tokType = tkCurlyDotRi tok.tokType = tkCurlyDotRi
Inc(L.bufpos, 2) Inc(L.bufpos, 2)
elif L.buf[L.bufPos + 1] == ')': elif L.buf[L.bufPos+1] == ')':
tok.tokType = tkParDotRi tok.tokType = tkParDotRi
Inc(L.bufpos, 2) Inc(L.bufpos, 2)
else: else:
getOperator(L, tok) getOperator(L, tok)
of '{': of '{':
Inc(L.bufpos) Inc(L.bufpos)
if (L.buf[L.bufPos] == '.') and (L.buf[L.bufPos+1] != '.'): if L.buf[L.bufPos] == '.' and L.buf[L.bufPos+1] != '.':
tok.toktype = tkCurlyDotLe tok.toktype = tkCurlyDotLe
Inc(L.bufpos) Inc(L.bufpos)
else: else:
@ -796,13 +790,15 @@ proc rawGetTok(L: var TLexer, tok: var TToken) =
tok.tokType = tkCharLit tok.tokType = tkCharLit
getCharacter(L, tok) getCharacter(L, tok)
tok.tokType = tkCharLit tok.tokType = tkCharLit
of '0'..'9':
tok = getNumber(L)
else: else:
if c in OpChars: if c in OpChars:
getOperator(L, tok) getOperator(L, tok)
elif c == lexbase.EndOfFile: elif c == lexbase.EndOfFile:
tok.toktype = tkEof tok.toktype = tkEof
else: else:
tok.literal = c & "" tok.literal = $c
tok.tokType = tkInvalid tok.tokType = tkInvalid
lexMessage(L, errInvalidToken, c & " (\\" & $(ord(c)) & ')') lexMessage(L, errInvalidToken, c & " (\\" & $(ord(c)) & ')')
Inc(L.bufpos) Inc(L.bufpos)

2
compiler/nimconf.nim
View file

@ -19,7 +19,7 @@ import
proc ppGetTok(L: var TLexer, tok: var TToken) = proc ppGetTok(L: var TLexer, tok: var TToken) =
# simple filter # simple filter
rawGetTok(L, tok) rawGetTok(L, tok)
while tok.tokType in {tkInd, tkComment}: rawGetTok(L, tok) while tok.tokType in {tkComment}: rawGetTok(L, tok)
proc parseExpr(L: var TLexer, tok: var TToken): bool proc parseExpr(L: var TLexer, tok: var TToken): bool
proc parseAtom(L: var TLexer, tok: var TToken): bool = proc parseAtom(L: var TLexer, tok: var TToken): bool =

360
compiler/parser.nim
View file

@ -30,9 +30,9 @@ import
type type
TParser*{.final.} = object # a TParser object represents a module that TParser*{.final.} = object # a TParser object represents a module that
# is being parsed # is being parsed
currInd: int # current indentation (for skipInd)
lex*: TLexer # the lexer that is used for parsing lex*: TLexer # the lexer that is used for parsing
tok*: TToken # the current token tok*: TToken # the current token
currInd: int # current indentation (for skipInd)
proc ParseAll*(p: var TParser): PNode proc ParseAll*(p: var TParser): PNode
proc openParser*(p: var TParser, filename: string, inputstream: PLLStream) proc openParser*(p: var TParser, filename: string, inputstream: PLLStream)
@ -97,10 +97,10 @@ template withInd(p: expr, body: stmt) {.immediate.} =
body body
p.currInd = oldInd p.currInd = oldInd
template realInd(p): bool = p.tok.tokType == tkInd and p.tok.ident > p.currInd template realInd(p): bool = p.tok.indent > p.currInd
template sameInd(p): bool = p.tok.tokType == tkInd and p.tok.ident == p.currInd template sameInd(p): bool = p.tok.indent == p.currInd
proc skipComment(p: var TParser, node: PNode) = proc rawSkipComment(p: var TParser, node: PNode) =
if p.tok.tokType == tkComment: if p.tok.tokType == tkComment:
if node != nil: if node != nil:
if node.comment == nil: node.comment = "" if node.comment == nil: node.comment = ""
@ -109,17 +109,27 @@ proc skipComment(p: var TParser, node: PNode) =
parMessage(p, errInternal, "skipComment") parMessage(p, errInternal, "skipComment")
getTok(p) getTok(p)
proc skipComment(p: var TParser, node: PNode) =
if p.tok.indent < 0: rawSkipComment(p, node)
proc skipInd(p: var TParser) = proc skipInd(p: var TParser) =
if realInd(p): getTok(p) if p.tok.indent >= 0:
if not realInd(p): parMessage(p, errInvalidIndentation)
proc optPar(p: var TParser) = proc optPar(p: var TParser) =
if p.tok.tokType == tkInd and p.tok.indent >= p.currInd: getTok(p) if p.tok.indent >= 0:
if p.tok.indent < p.currInd: parMessage(p, errInvalidIndentation)
proc optInd(p: var TParser, n: PNode) = proc optInd(p: var TParser, n: PNode) =
skipComment(p, n) skipComment(p, n)
skipInd(p) skipInd(p)
proc ExpectNl(p: TParser) = proc getTokNoInd(p: var TParser) =
getTok(p)
if p.tok.indent >= 0: parMessage(p, errInvalidIndentation)
when false:
proc ExpectNl(p: TParser) =
if p.tok.tokType notin {tkEof, tkInd, tkComment}: if p.tok.tokType notin {tkEof, tkInd, tkComment}:
lexMessage(p.lex, errNewlineExpected, prettyTok(p.tok)) lexMessage(p.lex, errNewlineExpected, prettyTok(p.tok))
@ -139,11 +149,9 @@ proc parLineInfo(p: TParser): TLineInfo =
result = getLineInfo(p.lex) result = getLineInfo(p.lex)
proc indAndComment(p: var TParser, n: PNode) = proc indAndComment(p: var TParser, n: PNode) =
if p.tok.tokType == tkInd and p.tok.indent > p.currInd: if p.tok.indent > p.currInd:
var info = parLineInfo(p) if p.tok.tokType == tkComment: rawSkipComment(p, n)
getTok(p) else: parMessage(p, errInvalidIndentation)
if p.tok.tokType == tkComment: skipComment(p, n)
else: LocalError(info, errInvalidIndentation)
else: else:
skipComment(p, n) skipComment(p, n)
@ -214,7 +222,7 @@ proc getPrecedence(tok: TToken): int =
proc isOperator(tok: TToken): bool = proc isOperator(tok: TToken): bool =
result = getPrecedence(tok) >= 0 result = getPrecedence(tok) >= 0
#| module = stmt? (IND{=} stmt)* #| module = stmt ^* (';' / IND{=})
#| #|
#| comma = ',' COMMENT? IND? #| comma = ',' COMMENT? IND?
#| semicolon = ';' COMMENT IND? #| semicolon = ';' COMMENT IND?
@ -242,6 +250,10 @@ proc isOperator(tok: TToken): bool =
#| mulExpr = dollarExpr (OP8 optInd dollarExpr)* #| mulExpr = dollarExpr (OP8 optInd dollarExpr)*
#| dollarExpr = primary (OP9 optInd primary)* #| dollarExpr = primary (OP9 optInd primary)*
proc colcom(p: var TParser, n: PNode) =
eat(p, tkColon)
skipComment(p, n)
proc parseSymbol(p: var TParser): PNode = proc parseSymbol(p: var TParser): PNode =
#| symbol = '`' (KEYW|IDENT|operator|'(' ')'|'[' ']'|'{' '}'|'='|literal)+ '`' #| symbol = '`' (KEYW|IDENT|operator|'(' ')'|'[' ']'|'{' '}'|'='|literal)+ '`'
#| | IDENT #| | IDENT
@ -291,7 +303,7 @@ proc indexExpr(p: var TParser): PNode =
proc indexExprList(p: var TParser, first: PNode, k: TNodeKind, proc indexExprList(p: var TParser, first: PNode, k: TNodeKind,
endToken: TTokType): PNode = endToken: TTokType): PNode =
#| indexExprList = indexExpr (comma indexExpr)* comma? #| indexExprList = indexExpr ^+ comma
result = newNodeP(k, p) result = newNodeP(k, p)
addSon(result, first) addSon(result, first)
getTok(p) getTok(p)
@ -324,7 +336,7 @@ proc exprColonEqExpr(p: var TParser): PNode =
result = a result = a
proc exprList(p: var TParser, endTok: TTokType, result: PNode) = proc exprList(p: var TParser, endTok: TTokType, result: PNode) =
#| exprList = expr (comma expr)* comma? #| exprList = expr ^+ comma
getTok(p) getTok(p)
optInd(p, result) optInd(p, result)
while (p.tok.tokType != endTok) and (p.tok.tokType != tkEof): while (p.tok.tokType != endTok) and (p.tok.tokType != tkEof):
@ -363,8 +375,7 @@ proc exprColonEqExprListAux(p: var TParser, endTok: TTokType, result: PNode) =
assert(endTok in {tkCurlyRi, tkCurlyDotRi, tkBracketRi, tkParRi}) assert(endTok in {tkCurlyRi, tkCurlyDotRi, tkBracketRi, tkParRi})
getTok(p) getTok(p)
optInd(p, result) optInd(p, result)
while p.tok.tokType != endTok and p.tok.tokType != tkEof and while p.tok.tokType != endTok and p.tok.tokType != tkEof:
not (p.tok.tokType == tkInd and p.tok.indent >= p.currInd):
var a = exprColonEqExpr(p) var a = exprColonEqExpr(p)
addSon(result, a) addSon(result, a)
if p.tok.tokType != tkComma: break if p.tok.tokType != tkComma: break
@ -388,7 +399,7 @@ proc setOrTableConstr(p: var TParser): PNode =
getTok(p) # skip ':' getTok(p) # skip ':'
result.kind = nkTableConstr result.kind = nkTableConstr
else: else:
while p.tok.tokType notin {tkCurlyRi, tkEof, tkInd}: while p.tok.tokType notin {tkCurlyRi, tkEof}:
var a = exprColonEqExpr(p) var a = exprColonEqExpr(p)
if a.kind == nkExprColonExpr: result.kind = nkTableConstr if a.kind == nkExprColonExpr: result.kind = nkTableConstr
addSon(result, a) addSon(result, a)
@ -722,7 +733,6 @@ proc parseTuple(p: var TParser, indentAllowed = false): PNode =
skipComment(p, result) skipComment(p, result)
if realInd(p): if realInd(p):
withInd(p): withInd(p):
getTok(p)
skipComment(p, result) skipComment(p, result)
while true: while true:
case p.tok.tokType case p.tok.tokType
@ -735,16 +745,15 @@ proc parseTuple(p: var TParser, indentAllowed = false): PNode =
parMessage(p, errIdentifierExpected, p.tok) parMessage(p, errIdentifierExpected, p.tok)
break break
if not sameInd(p): break if not sameInd(p): break
getTok(p)
proc parseParamList(p: var TParser, retColon = true): PNode = proc parseParamList(p: var TParser, retColon = true): PNode =
#| paramList = '(' (identColonEquals (comma/semicolon identColonEquals)*)? ')' #| paramList = '(' identColonEquals ^* (comma/semicolon) ')'
#| paramListArrow = paramList? ('->' optInd typeDesc)? #| paramListArrow = paramList? ('->' optInd typeDesc)?
#| paramListColon = paramList? (':' optInd typeDesc)? #| paramListColon = paramList? (':' optInd typeDesc)?
var a: PNode var a: PNode
result = newNodeP(nkFormalParams, p) result = newNodeP(nkFormalParams, p)
addSon(result, ast.emptyNode) # return type addSon(result, ast.emptyNode) # return type
if p.tok.tokType == tkParLe: if p.tok.tokType == tkParLe and p.tok.indent < 0:
getTok(p) getTok(p)
optInd(p, result) optInd(p, result)
while true: while true:
@ -764,14 +773,16 @@ proc parseParamList(p: var TParser, retColon = true): PNode =
eat(p, tkParRi) eat(p, tkParRi)
let hasRet = if retColon: p.tok.tokType == tkColon let hasRet = if retColon: p.tok.tokType == tkColon
else: p.tok.tokType == tkOpr and IdentEq(p.tok.ident, "->") else: p.tok.tokType == tkOpr and IdentEq(p.tok.ident, "->")
if hasRet: if hasRet and p.tok.indent < 0:
getTok(p) getTok(p)
optInd(p, result) optInd(p, result)
result.sons[0] = parseTypeDesc(p) result.sons[0] = parseTypeDesc(p)
proc optPragmas(p: var TParser): PNode = proc optPragmas(p: var TParser): PNode =
if p.tok.tokType == tkCurlyDotLe: result = parsePragma(p) if p.tok.tokType == tkCurlyDotLe and (p.tok.indent < 0 or realInd(p)):
else: result = ast.emptyNode result = parsePragma(p)
else:
result = ast.emptyNode
proc parseDoBlock(p: var TParser): PNode = proc parseDoBlock(p: var TParser): PNode =
#| doBlock = 'do' paramListArrow pragmas? colcom stmt #| doBlock = 'do' paramListArrow pragmas? colcom stmt
@ -786,21 +797,20 @@ proc parseDoBlock(p: var TParser): PNode =
pragmas = pragmas) pragmas = pragmas)
proc parseDoBlocks(p: var TParser, call: PNode) = proc parseDoBlocks(p: var TParser, call: PNode) =
#| doBlocks = doBlock* #| doBlocks = doBlock ^* IND{=}
while p.tok.tokType == tkDo: if p.tok.tokType == tkDo:
addSon(call, parseDoBlock(p))
while sameInd(p) and p.tok.tokType == tkDo:
addSon(call, parseDoBlock(p)) addSon(call, parseDoBlock(p))
proc parseProcExpr(p: var TParser, isExpr: bool): PNode = proc parseProcExpr(p: var TParser, isExpr: bool): PNode =
#| procExpr = 'proc' paramListColon pragmas? ('=' COMMENT? stmt)? #| procExpr = 'proc' paramListColon pragmas? ('=' COMMENT? stmt)?
# either a proc type or a anonymous proc # either a proc type or a anonymous proc
var let info = parLineInfo(p)
pragmas, params: PNode
info: TLineInfo
info = parLineInfo(p)
getTok(p) getTok(p)
let hasSignature = p.tok.tokType in {tkParLe, tkColon} let hasSignature = p.tok.tokType in {tkParLe, tkColon} and p.tok.indent < 0
params = parseParamList(p) let params = parseParamList(p)
pragmas = optPragmas(p) let pragmas = optPragmas(p)
if p.tok.tokType == tkEquals and isExpr: if p.tok.tokType == tkEquals and isExpr:
getTok(p) getTok(p)
skipComment(p, result) skipComment(p, result)
@ -831,10 +841,10 @@ proc parseTypeDescKAux(p: var TParser, kind: TNodeKind,
addSon(result, primary(p, mode)) addSon(result, primary(p, mode))
proc parseExpr(p: var TParser): PNode = proc parseExpr(p: var TParser): PNode =
#| expr = lowestExpr #| expr = (ifExpr
#| | ifExpr
#| | whenExpr #| | whenExpr
#| | caseExpr #| | caseExpr)
#| / lowestExpr
case p.tok.tokType: case p.tok.tokType:
of tkIf: result = parseIfExpr(p, nkIfExpr) of tkIf: result = parseIfExpr(p, nkIfExpr)
of tkWhen: result = parseIfExpr(p, nkWhenExpr) of tkWhen: result = parseIfExpr(p, nkWhenExpr)
@ -907,11 +917,11 @@ proc primary(p: var TParser, mode: TPrimaryMode): PNode =
getTok(p) getTok(p)
of tkAddr: of tkAddr:
result = newNodeP(nkAddr, p) result = newNodeP(nkAddr, p)
getTok(p) getTokNoInd(p)
addSon(result, primary(p, pmNormal)) addSon(result, primary(p, pmNormal))
of tkStatic: of tkStatic:
result = newNodeP(nkStaticExpr, p) result = newNodeP(nkStaticExpr, p)
getTok(p) getTokNoInd(p)
addSon(result, primary(p, pmNormal)) addSon(result, primary(p, pmNormal))
of tkBind: of tkBind:
result = newNodeP(nkBind, p) result = newNodeP(nkBind, p)
@ -933,10 +943,10 @@ proc parseExprStmt(p: var TParser): PNode =
#| exprStmt = lowestExpr ( #| exprStmt = lowestExpr (
#| '=' optInd expr #| '=' optInd expr
#| / doBlocks #| / doBlocks
#| / ':' stmt? ('of' exprList ':' stmt #| / ':' stmt? ( IND{=} 'of' exprList ':' stmt
#| | 'elif' expr ':' stmt #| | IND{=} 'elif' expr ':' stmt
#| | 'except' exprList ':' stmt #| | IND{=} 'except' exprList ':' stmt
#| | 'else' ':' stmt )? #| | IND{=} 'else' ':' stmt )*
#| ) #| )
var a = lowestExpr(p) var a = lowestExpr(p)
if p.tok.tokType == tkEquals: if p.tok.tokType == tkEquals:
@ -966,12 +976,10 @@ proc parseExprStmt(p: var TParser): PNode =
result = call result = call
getTok(p) getTok(p)
skipComment(p, result) skipComment(p, result)
if sameInd(p): getTok(p)
if p.tok.TokType notin {tkOf, tkElif, tkElse, tkExcept}: if p.tok.TokType notin {tkOf, tkElif, tkElse, tkExcept}:
let body = parseStmt(p) let body = parseStmt(p)
addSon(result, newProcNode(nkDo, body.info, body)) addSon(result, newProcNode(nkDo, body.info, body))
while true: while sameInd(p):
if sameInd(p): getTok(p)
var b: PNode var b: PNode
case p.tok.tokType case p.tok.tokType
of tkOf: of tkOf:
@ -999,7 +1007,7 @@ proc parseExprStmt(p: var TParser): PNode =
proc parseImport(p: var TParser, kind: TNodeKind): PNode = proc parseImport(p: var TParser, kind: TNodeKind): PNode =
#| importStmt = 'import' optInd expr #| importStmt = 'import' optInd expr
#| ((comma expr)* #| ((comma expr)*
#| / 'except' optInd expr (comma expr)*) #| / 'except' optInd (expr ^+ comma))
result = newNodeP(kind, p) result = newNodeP(kind, p)
getTok(p) # skip `import` or `export` getTok(p) # skip `import` or `export`
optInd(p, result) optInd(p, result)
@ -1018,10 +1026,10 @@ proc parseImport(p: var TParser, kind: TNodeKind): PNode =
if p.tok.tokType != tkComma: break if p.tok.tokType != tkComma: break
getTok(p) getTok(p)
optInd(p, a) optInd(p, a)
expectNl(p) #expectNl(p)
proc parseIncludeStmt(p: var TParser): PNode = proc parseIncludeStmt(p: var TParser): PNode =
#| includeStmt = 'include' optInd expr (comma expr)* #| includeStmt = 'include' optInd expr ^+ comma
result = newNodeP(nkIncludeStmt, p) result = newNodeP(nkIncludeStmt, p)
getTok(p) # skip `import` or `include` getTok(p) # skip `import` or `include`
optInd(p, result) optInd(p, result)
@ -1033,7 +1041,7 @@ proc parseIncludeStmt(p: var TParser): PNode =
if p.tok.tokType != tkComma: break if p.tok.tokType != tkComma: break
getTok(p) getTok(p)
optInd(p, a) optInd(p, a)
expectNl(p) #expectNl(p)
proc parseFromStmt(p: var TParser): PNode = proc parseFromStmt(p: var TParser): PNode =
#| fromStmt = 'from' expr 'import' optInd expr (comma expr)* #| fromStmt = 'from' expr 'import' optInd expr (comma expr)*
@ -1052,7 +1060,7 @@ proc parseFromStmt(p: var TParser): PNode =
if p.tok.tokType != tkComma: break if p.tok.tokType != tkComma: break
getTok(p) getTok(p)
optInd(p, a) optInd(p, a)
expectNl(p) #expectNl(p)
proc parseReturnOrRaise(p: var TParser, kind: TNodeKind): PNode = proc parseReturnOrRaise(p: var TParser, kind: TNodeKind): PNode =
#| returnStmt = 'return' optInd expr? #| returnStmt = 'return' optInd expr?
@ -1063,15 +1071,16 @@ proc parseReturnOrRaise(p: var TParser, kind: TNodeKind): PNode =
#| continueStmt = 'break' optInd expr? #| continueStmt = 'break' optInd expr?
result = newNodeP(kind, p) result = newNodeP(kind, p)
getTok(p) getTok(p)
optInd(p, result) if p.tok.indent >= 0 and p.tok.indent <= p.currInd or p.tok.tokType == tkEof:
case p.tok.tokType # NL terminates:
of tkEof, tkInd: addSon(result, ast.emptyNode) addSon(result, ast.emptyNode)
else: addSon(result, parseExpr(p)) else:
addSon(result, parseExpr(p))
proc parseIfOrWhen(p: var TParser, kind: TNodeKind): PNode = proc parseIfOrWhen(p: var TParser, kind: TNodeKind): PNode =
#| condStmt = expr colcom stmt COMMENT? #| condStmt = expr colcom stmt COMMENT?
#| ('elif' expr colcom stmt)* #| (IND{=} 'elif' expr colcom stmt)*
#| ('else' colcom stmt)? #| (IND{=} 'else' colcom stmt)?
#| ifStmt = 'if' condStmt #| ifStmt = 'if' condStmt
#| whenStmt = 'when' condStmt #| whenStmt = 'when' condStmt
result = newNodeP(kind, p) result = newNodeP(kind, p)
@ -1085,8 +1094,8 @@ proc parseIfOrWhen(p: var TParser, kind: TNodeKind): PNode =
addSon(branch, parseStmt(p)) addSon(branch, parseStmt(p))
skipComment(p, branch) skipComment(p, branch)
addSon(result, branch) addSon(result, branch)
if p.tok.tokType != tkElif: break if p.tok.tokType != tkElif or not sameInd(p): break
if p.tok.tokType == tkElse: if p.tok.tokType == tkElse and sameInd(p):
var branch = newNodeP(nkElse, p) var branch = newNodeP(nkElse, p)
eat(p, tkElse) eat(p, tkElse)
eat(p, tkColon) eat(p, tkColon)
@ -1100,8 +1109,7 @@ proc parseWhile(p: var TParser): PNode =
getTok(p) getTok(p)
optInd(p, result) optInd(p, result)
addSon(result, parseExpr(p)) addSon(result, parseExpr(p))
eat(p, tkColon) colcom(p, result)
skipComment(p, result)
addSon(result, parseStmt(p)) addSon(result, parseStmt(p))
proc parseCase(p: var TParser): PNode = proc parseCase(p: var TParser): PNode =
@ -1128,8 +1136,7 @@ proc parseCase(p: var TParser): PNode =
getTok(p) getTok(p)
wasIndented = true wasIndented = true
while true: while sameInd(p):
if sameInd(p): getTok(p)
case p.tok.tokType case p.tok.tokType
of tkOf: of tkOf:
if inElif: break if inElif: break
@ -1156,24 +1163,23 @@ proc parseCase(p: var TParser): PNode =
p.currInd = oldInd p.currInd = oldInd
proc parseTry(p: var TParser): PNode = proc parseTry(p: var TParser): PNode =
#| tryStmt = 'try' colcom stmt &('except'|'finally') #| tryStmt = 'try' colcom stmt &(IND{=} 'except'|'finally')
#| ('except' exprList colcom stmt)* #| (IND{=} 'except' exprList colcom stmt)*
#| ('finally' colcom stmt)? #| (IND{=} 'finally' colcom stmt)?
result = newNodeP(nkTryStmt, p) result = newNodeP(nkTryStmt, p)
getTok(p) getTok(p)
eat(p, tkColon) eat(p, tkColon)
skipComment(p, result) skipComment(p, result)
addSon(result, parseStmt(p)) addSon(result, parseStmt(p))
var b: PNode = nil var b: PNode = nil
while true: while sameInd(p):
if p.tok.tokType == tkSad: getTok(p)
case p.tok.tokType case p.tok.tokType
of tkExcept: of tkExcept:
b = newNodeP(nkExceptBranch, p) b = newNodeP(nkExceptBranch, p)
exprList(p, tkColon, b) exprList(p, tkColon, b)
of tkFinally: of tkFinally:
b = newNodeP(nkFinally, p) b = newNodeP(nkFinally, p)
getTok(p) getTokNoInd(p)
eat(p, tkColon) eat(p, tkColon)
else: break else: break
skipComment(p, b) skipComment(p, b)
@ -1185,15 +1191,14 @@ proc parseTry(p: var TParser): PNode =
proc parseExceptBlock(p: var TParser, kind: TNodeKind): PNode = proc parseExceptBlock(p: var TParser, kind: TNodeKind): PNode =
#| exceptBlock = 'except' colcom stmt #| exceptBlock = 'except' colcom stmt
result = newNodeP(kind, p) result = newNodeP(kind, p)
getTok(p) getTokNoInd(p)
eat(p, tkColon) colcom(p, result)
skipComment(p, result)
addSon(result, parseStmt(p)) addSon(result, parseStmt(p))
proc parseFor(p: var TParser): PNode = proc parseFor(p: var TParser): PNode =
#| forStmt = 'for' symbol (comma symbol)* 'in' expr colcom stmt #| forStmt = 'for' symbol (comma symbol)* 'in' expr colcom stmt
result = newNodeP(nkForStmt, p) result = newNodeP(nkForStmt, p)
getTok(p) getTokNoInd(p)
var a = parseSymbol(p) var a = parseSymbol(p)
addSon(result, a) addSon(result, a)
while p.tok.tokType == tkComma: while p.tok.tokType == tkComma:
@ -1203,32 +1208,29 @@ proc parseFor(p: var TParser): PNode =
addSon(result, a) addSon(result, a)
eat(p, tkIn) eat(p, tkIn)
addSon(result, parseExpr(p)) addSon(result, parseExpr(p))
eat(p, tkColon) colcom(p, result)
skipComment(p, result)
addSon(result, parseStmt(p)) addSon(result, parseStmt(p))
proc parseBlock(p: var TParser): PNode = proc parseBlock(p: var TParser): PNode =
#| blockStmt = 'block' symbol? colcom stmt #| blockStmt = 'block' symbol? colcom stmt
result = newNodeP(nkBlockStmt, p) result = newNodeP(nkBlockStmt, p)
getTok(p) getTokNoInd(p)
if p.tok.tokType = tkColon: addSon(result, ast.emptyNode) if p.tok.tokType == tkColon: addSon(result, ast.emptyNode)
else: addSon(result, parseSymbol(p)) else: addSon(result, parseSymbol(p))
eat(p, tkColon) colcom(p, result)
skipComment(p, result)
addSon(result, parseStmt(p)) addSon(result, parseStmt(p))
proc parseStatic(p: var TParser): PNode = proc parseStatic(p: var TParser): PNode =
#| staticStmt = 'static' colcom stmt #| staticStmt = 'static' colcom stmt
result = newNodeP(nkStaticStmt, p) result = newNodeP(nkStaticStmt, p)
getTok(p) getTokNoInd(p)
eat(p, tkColon) colcom(p, result)
skipComment(p, result)
addSon(result, parseStmt(p)) addSon(result, parseStmt(p))
proc parseAsm(p: var TParser): PNode = proc parseAsm(p: var TParser): PNode =
#| asmStmt = 'asm' pragma? (STR_LIT | RSTR_LIT | TRIPLE_STR_LIT) #| asmStmt = 'asm' pragma? (STR_LIT | RSTR_LIT | TRIPLE_STR_LIT)
result = newNodeP(nkAsmStmt, p) result = newNodeP(nkAsmStmt, p)
getTok(p) getTokNoInd(p)
if p.tok.tokType == tkCurlyDotLe: addSon(result, parsePragma(p)) if p.tok.tokType == tkCurlyDotLe: addSon(result, parsePragma(p))
else: addSon(result, ast.emptyNode) else: addSon(result, ast.emptyNode)
case p.tok.tokType case p.tok.tokType
@ -1271,11 +1273,11 @@ proc parseGenericParam(p: var TParser): PNode =
proc parseGenericParamList(p: var TParser): PNode = proc parseGenericParamList(p: var TParser): PNode =
#| genericParamList = '[' optInd #| genericParamList = '[' optInd
#| (genericParam (comma/semicolon genericParam)*)? optPar ']' #| genericParam ^* (comma/semicolon) optPar ']'
result = newNodeP(nkGenericParams, p) result = newNodeP(nkGenericParams, p)
getTok(p) getTok(p)
optInd(p, result) optInd(p, result)
while (p.tok.tokType == tkSymbol) or (p.tok.tokType == tkAccent): while p.tok.tokType in {tkSymbol, tkAccent}:
var a = parseGenericParam(p) var a = parseGenericParam(p)
addSon(result, a) addSon(result, a)
if p.tok.tokType notin {tkComma, tkSemicolon}: break if p.tok.tokType notin {tkComma, tkSemicolon}: break
@ -1290,6 +1292,9 @@ proc parsePattern(p: var TParser): PNode =
result = parseStmt(p) result = parseStmt(p)
eat(p, tkCurlyRi) eat(p, tkCurlyRi)
proc validInd(p: var TParser): bool =
result = p.tok.indent < 0 or p.tok.indent > p.currInd
proc parseRoutine(p: var TParser, kind: TNodeKind): PNode = proc parseRoutine(p: var TParser, kind: TNodeKind): PNode =
#| indAndComment = (IND{>} COMMENT)? | COMMENT? #| indAndComment = (IND{>} COMMENT)? | COMMENT?
#| routine = optInd identVis pattern? genericParamList? #| routine = optInd identVis pattern? genericParamList?
@ -1298,16 +1303,18 @@ proc parseRoutine(p: var TParser, kind: TNodeKind): PNode =
getTok(p) getTok(p)
optInd(p, result) optInd(p, result)
addSon(result, identVis(p)) addSon(result, identVis(p))
if p.tok.tokType == tkCurlyLe: addSon(result, parsePattern(p)) if p.tok.tokType == tkCurlyLe and p.validInd: addSon(result, p.parsePattern)
else: addSon(result, ast.emptyNode) else: addSon(result, ast.emptyNode)
if p.tok.tokType == tkBracketLe: addSon(result, parseGenericParamList(p)) if p.tok.tokType == tkBracketLe and p.validInd:
else: addSon(result, ast.emptyNode) result.add(p.parseGenericParamList)
addSon(result, parseParamList(p)) else:
if p.tok.tokType == tkCurlyDotLe: addSon(result, parsePragma(p)) addSon(result, ast.emptyNode)
addSon(result, p.parseParamList)
if p.tok.tokType == tkCurlyDotLe and p.validInd: addSon(result, p.parsePragma)
else: addSon(result, ast.emptyNode) else: addSon(result, ast.emptyNode)
# empty exception tracking: # empty exception tracking:
addSon(result, ast.emptyNode) addSon(result, ast.emptyNode)
if p.tok.tokType == tkEquals: if p.tok.tokType == tkEquals and p.validInd:
getTok(p) getTok(p)
skipComment(p, result) skipComment(p, result)
addSon(result, parseStmt(p)) addSon(result, parseStmt(p))
@ -1329,22 +1336,15 @@ proc parseSection(p: var TParser, kind: TNodeKind,
result = newNodeP(kind, p) result = newNodeP(kind, p)
getTok(p) getTok(p)
skipComment(p, result) skipComment(p, result)
case p.tok.tokType if realInd(p):
of tkInd:
if not realInd(p): parMessage(p, errInvalidIndentation)
withInd(p): withInd(p):
getTok(p)
skipComment(p, result) skipComment(p, result)
while true: while sameInd(p):
case p.tok.tokType case p.tok.tokType
of tkSad:
getTok(p)
of tkSymbol, tkAccent: of tkSymbol, tkAccent:
var a = defparser(p) var a = defparser(p)
skipComment(p, a) skipComment(p, a)
addSon(result, a) addSon(result, a)
of tkEof:
break
of tkComment: of tkComment:
var a = newCommentStmt(p) var a = newCommentStmt(p)
skipComment(p, a) skipComment(p, a)
@ -1352,10 +1352,12 @@ proc parseSection(p: var TParser, kind: TNodeKind,
else: else:
parMessage(p, errIdentifierExpected, p.tok) parMessage(p, errIdentifierExpected, p.tok)
break break
of tkSymbol, tkAccent, tkParLe: if result.len == 0: parMessage(p, errIdentifierExpected, p.tok)
elif p.tok.tokType in {tkSymbol, tkAccent, tkParLe} and p.tok.indent < 0:
# tkParLe is allowed for ``var (x, y) = ...`` tuple parsing # tkParLe is allowed for ``var (x, y) = ...`` tuple parsing
addSon(result, defparser(p)) addSon(result, defparser(p))
else: parMessage(p, errIdentifierExpected, p.tok) else:
parMessage(p, errIdentifierExpected, p.tok)
proc parseConstant(p: var TParser): PNode = proc parseConstant(p: var TParser): PNode =
#| constant = identWithPragma (colon typedesc)? '=' optInd expr indAndComment #| constant = identWithPragma (colon typedesc)? '=' optInd expr indAndComment
@ -1378,7 +1380,7 @@ proc parseEnum(p: var TParser): PNode =
getTok(p) getTok(p)
addSon(result, ast.emptyNode) addSon(result, ast.emptyNode)
optInd(p, result) optInd(p, result)
while p.tok.tokType notin {tkEof, tkInd}: while true:
var a = parseSymbol(p) var a = parseSymbol(p)
optInd(p, a) optInd(p, a)
if p.tok.tokType == tkEquals: if p.tok.tokType == tkEquals:
@ -1393,6 +1395,8 @@ proc parseEnum(p: var TParser): PNode =
getTok(p) getTok(p)
optInd(p, a) optInd(p, a)
addSon(result, a) addSon(result, a)
if p.tok.indent > 0 and p.tok.indent <= p.currInd or p.tok.tokType == tkEof:
break
if result.len <= 1: if result.len <= 1:
lexMessage(p.lex, errIdentifierExpected, prettyTok(p.tok)) lexMessage(p.lex, errIdentifierExpected, prettyTok(p.tok))
@ -1402,24 +1406,22 @@ proc parseObjectWhen(p: var TParser): PNode =
#| ('elif' expr colcom objectPart COMMENT?)* #| ('elif' expr colcom objectPart COMMENT?)*
#| ('else' colcom objectPart COMMENT?)? #| ('else' colcom objectPart COMMENT?)?
result = newNodeP(nkRecWhen, p) result = newNodeP(nkRecWhen, p)
while true: while sameInd(p):
getTok(p) # skip `when`, `elif` getTok(p) # skip `when`, `elif`
var branch = newNodeP(nkElifBranch, p) var branch = newNodeP(nkElifBranch, p)
optInd(p, branch) optInd(p, branch)
addSon(branch, parseExpr(p)) addSon(branch, parseExpr(p))
eat(p, tkColon) colcom(p, branch)
skipComment(p, branch)
addSon(branch, parseObjectPart(p)) addSon(branch, parseObjectPart(p))
skipComment(p, branch) skipComment(p, branch)
addSon(result, branch) addSon(result, branch)
if p.tok.tokType != tkElif: break if p.tok.tokType != tkElif: break
if p.tok.tokType == tkElse: if p.tok.tokType == tkElse and sameInd(p):
var branch = newNodeP(nkElse, p) var branch = newNodeP(nkElse, p)
eat(p, tkElse) eat(p, tkElse)
eat(p, tkColon) colcom(p, branch)
skipComment(p, branch)
addSon(branch, parseObjectPart(p)) addSon(branch, parseObjectPart(p))
# XXX no skipComment(p, branch) here? skipComment(p, branch)
addSon(result, branch) addSon(result, branch)
proc parseObjectCase(p: var TParser): PNode = proc parseObjectCase(p: var TParser): PNode =
@ -1427,11 +1429,11 @@ proc parseObjectCase(p: var TParser): PNode =
#| objectBranches = objectBranch (IND{=} objectBranch)* #| objectBranches = objectBranch (IND{=} objectBranch)*
#| (IND{=} 'elif' expr colcom objectPart)* #| (IND{=} 'elif' expr colcom objectPart)*
#| (IND{=} 'else' colcom objectPart)? #| (IND{=} 'else' colcom objectPart)?
#| objectCase = 'case' identWithPragma ':'? COMMENT? #| objectCase = 'case' identWithPragma ':' typeDesc ':'? COMMENT?
#| (IND{>} objectBranches #| (IND{>} objectBranches
#| | IND{=} objectBranches) #| | IND{=} objectBranches)
result = newNodeP(nkRecCase, p) result = newNodeP(nkRecCase, p)
getTok(p) getTokNoInd(p)
var a = newNodeP(nkIdentDefs, p) var a = newNodeP(nkIdentDefs, p)
addSon(a, identWithPragma(p)) addSon(a, identWithPragma(p))
eat(p, tkColon) eat(p, tkColon)
@ -1441,12 +1443,11 @@ proc parseObjectCase(p: var TParser): PNode =
if p.tok.tokType == tkColon: getTok(p) if p.tok.tokType == tkColon: getTok(p)
skipComment(p, result) skipComment(p, result)
var wasIndented = false var wasIndented = false
if p.tok.tokType == tkInd: let oldInd = p.currInd
pushInd(p.lex, p.tok.indent) if realInd(p):
getTok(p) p.currInd = p.tok.indent
wasIndented = true wasIndented = true
while true: while sameInd(p):
if p.tok.tokType == tkSad: getTok(p)
var b: PNode var b: PNode
case p.tok.tokType case p.tok.tokType
of tkOf: of tkOf:
@ -1466,31 +1467,24 @@ proc parseObjectCase(p: var TParser): PNode =
addSon(result, b) addSon(result, b)
if b.kind == nkElse: break if b.kind == nkElse: break
if wasIndented: if wasIndented:
eat(p, tkDed) p.currInd = oldInd
popInd(p.lex)
proc parseObjectPart(p: var TParser): PNode = proc parseObjectPart(p: var TParser): PNode =
case p.tok.tokType #| objectPart = IND{>} objectPart^+IND{=} DED
of tkInd: #| / objectWhen / objectCase / 'nil' / declColonEquals
if realInd(p):
result = newNodeP(nkRecList, p) result = newNodeP(nkRecList, p)
pushInd(p.lex, p.tok.indent) withInd(p):
getTok(p)
skipComment(p, result) skipComment(p, result)
while true: while sameInd(p):
case p.tok.tokType case p.tok.tokType
of tkSad:
getTok(p)
of tkCase, tkWhen, tkSymbol, tkAccent, tkNil: of tkCase, tkWhen, tkSymbol, tkAccent, tkNil:
addSon(result, parseObjectPart(p)) addSon(result, parseObjectPart(p))
of tkDed:
getTok(p)
break
of tkEof:
break
else: else:
parMessage(p, errIdentifierExpected, p.tok) parMessage(p, errIdentifierExpected, p.tok)
break break
popInd(p.lex) else:
case p.tok.tokType
of tkWhen: of tkWhen:
result = parseObjectWhen(p) result = parseObjectWhen(p)
of tkCase: of tkCase:
@ -1501,14 +1495,18 @@ proc parseObjectPart(p: var TParser): PNode =
of tkNil: of tkNil:
result = newNodeP(nkNilLit, p) result = newNodeP(nkNilLit, p)
getTok(p) getTok(p)
else: result = ast.emptyNode else:
result = ast.emptyNode
proc parseObject(p: var TParser): PNode = proc parseObject(p: var TParser): PNode =
#| object = 'object' pragma? ('of' typeDesc)? COMMENT? objectPart
result = newNodeP(nkObjectTy, p) result = newNodeP(nkObjectTy, p)
getTok(p) getTok(p)
if p.tok.tokType == tkCurlyDotLe: addSon(result, parsePragma(p)) if p.tok.tokType == tkCurlyDotLe and p.validInd:
else: addSon(result, ast.emptyNode) addSon(result, parsePragma(p))
if p.tok.tokType == tkOf: else:
addSon(result, ast.emptyNode)
if p.tok.tokType == tkOf and p.tok.indent < 0:
var a = newNodeP(nkOfInherit, p) var a = newNodeP(nkOfInherit, p)
getTok(p) getTok(p)
addSon(a, parseTypeDesc(p)) addSon(a, parseTypeDesc(p))
@ -1526,10 +1524,14 @@ proc parseDistinct(p: var TParser): PNode =
addSon(result, parseTypeDesc(p)) addSon(result, parseTypeDesc(p))
proc parseTypeDef(p: var TParser): PNode = proc parseTypeDef(p: var TParser): PNode =
#| typeDef = identWithPragma genericParamList? '=' optInd typeDefAux
#| indAndComment?
result = newNodeP(nkTypeDef, p) result = newNodeP(nkTypeDef, p)
addSon(result, identWithPragma(p)) addSon(result, identWithPragma(p))
if p.tok.tokType == tkBracketLe: addSon(result, parseGenericParamList(p)) if p.tok.tokType == tkBracketLe and p.validInd:
else: addSon(result, ast.emptyNode) addSon(result, parseGenericParamList(p))
else:
addSon(result, ast.emptyNode)
if p.tok.tokType == tkEquals: if p.tok.tokType == tkEquals:
getTok(p) getTok(p)
optInd(p, result) optInd(p, result)
@ -1539,10 +1541,11 @@ proc parseTypeDef(p: var TParser): PNode =
indAndComment(p, result) # special extension! indAndComment(p, result) # special extension!
proc parseVarTuple(p: var TParser): PNode = proc parseVarTuple(p: var TParser): PNode =
#| varTuple = '(' optInd identWithPragma ^+ comma optPar ')' '=' optInd expr
result = newNodeP(nkVarTuple, p) result = newNodeP(nkVarTuple, p)
getTok(p) # skip '(' getTok(p) # skip '('
optInd(p, result) optInd(p, result)
while (p.tok.tokType == tkSymbol) or (p.tok.tokType == tkAccent): while p.tok.tokType in {tkSymbol, tkAccent}:
var a = identWithPragma(p) var a = identWithPragma(p)
addSon(result, a) addSon(result, a)
if p.tok.tokType != tkComma: break if p.tok.tokType != tkComma: break
@ -1556,12 +1559,14 @@ proc parseVarTuple(p: var TParser): PNode =
addSon(result, parseExpr(p)) addSon(result, parseExpr(p))
proc parseVariable(p: var TParser): PNode = proc parseVariable(p: var TParser): PNode =
#| variable = (varTuple / identColonEquals) indAndComment
if p.tok.tokType == tkParLe: result = parseVarTuple(p) if p.tok.tokType == tkParLe: result = parseVarTuple(p)
else: result = parseIdentColonEquals(p, {withPragma}) else: result = parseIdentColonEquals(p, {withPragma})
indAndComment(p, result) # special extension! indAndComment(p, result)
proc parseBind(p: var TParser, k: TNodeKind): PNode = proc parseBind(p: var TParser, k: TNodeKind): PNode =
#| bindStmt = 'bind' optInd qualifiedIdent (comma qualifiedIdent)* #| bindStmt = 'bind' optInd qualifiedIdent ^+ comma
#| mixinStmt = 'mixin' optInd qualifiedIdent ^+ comma
result = newNodeP(k, p) result = newNodeP(k, p)
getTok(p) getTok(p)
optInd(p, result) optInd(p, result)
@ -1571,12 +1576,12 @@ proc parseBind(p: var TParser, k: TNodeKind): PNode =
if p.tok.tokType != tkComma: break if p.tok.tokType != tkComma: break
getTok(p) getTok(p)
optInd(p, a) optInd(p, a)
expectNl(p) #expectNl(p)
proc parseStmtPragma(p: var TParser): PNode = proc parseStmtPragma(p: var TParser): PNode =
#| pragmaStmt = pragma (':' COMMENT? stmt)? #| pragmaStmt = pragma (':' COMMENT? stmt)?
result = parsePragma(p) result = parsePragma(p)
if p.tok.tokType == tkColon: if p.tok.tokType == tkColon and p.tok.indent < 0:
let a = result let a = result
result = newNodeI(nkPragmaBlock, a.info) result = newNodeI(nkPragmaBlock, a.info)
getTok(p) getTok(p)
@ -1585,6 +1590,10 @@ proc parseStmtPragma(p: var TParser): PNode =
result.add parseStmt(p) result.add parseStmt(p)
proc simpleStmt(p: var TParser): PNode = proc simpleStmt(p: var TParser): PNode =
#| simpleStmt = ((returnStmt | raiseStmt | yieldStmt | discardStmt | breakStmt
#| | continueStmt | pragmaStmt | importStmt | exportStmt | fromStmt
#| | includeStmt | commentStmt) / exprStmt) COMMENT?
#|
case p.tok.tokType case p.tok.tokType
of tkReturn: result = parseReturnOrRaise(p, nkReturnStmt) of tkReturn: result = parseReturnOrRaise(p, nkReturnStmt)
of tkRaise: result = parseReturnOrRaise(p, nkRaiseStmt) of tkRaise: result = parseReturnOrRaise(p, nkRaiseStmt)
@ -1604,6 +1613,20 @@ proc simpleStmt(p: var TParser): PNode =
if result.kind != nkEmpty: skipComment(p, result) if result.kind != nkEmpty: skipComment(p, result)
proc complexOrSimpleStmt(p: var TParser): PNode = proc complexOrSimpleStmt(p: var TParser): PNode =
#| complexOrSimpleStmt = (ifStmt | whenStmt | whileStmt
#| | tryStmt | finallyStmt | exceptStmt | forStmt
#| | blockStmt | staticStmt | asmStmt
#| | 'proc' routine
#| | 'method' routine
#| | 'iterator' routine
#| | 'macro' routine
#| | 'template' routine
#| | 'converter' routine
#| | 'type' section(typeDef)
#| | 'const' section(constant)
#| | ('let' | 'var') section(variable)
#| | bindStmt | mixinStmt)
#| / simpleStmt
case p.tok.tokType case p.tok.tokType
of tkIf: result = parseIfOrWhen(p, nkIfStmt) of tkIf: result = parseIfOrWhen(p, nkIfStmt)
of tkWhile: result = parseWhile(p) of tkWhile: result = parseWhile(p)
@ -1631,25 +1654,22 @@ proc complexOrSimpleStmt(p: var TParser): PNode =
else: result = simpleStmt(p) else: result = simpleStmt(p)
proc parseStmt(p: var TParser): PNode = proc parseStmt(p: var TParser): PNode =
if p.tok.tokType == tkInd: #| stmt = (IND{>} complexOrSimpleStmt^+(IND{=} / ';') DED)
#| / simpleStmt
if p.tok.indent > p.currInd:
result = newNodeP(nkStmtList, p) result = newNodeP(nkStmtList, p)
pushInd(p.lex, p.tok.indent) withInd(p):
getTok(p)
while true: while true:
case p.tok.tokType if p.tok.indent == p.currInd:
of tkSad, tkSemicolon: getTok(p) nil
of tkEof: break elif p.tok.tokType == tkSemicolon:
of tkDed: while p.tok.tokType == tkSemicolon: getTok(p)
getTok(p)
break
else: else:
var a = complexOrSimpleStmt(p) if p.tok.indent > p.currInd:
if a.kind == nkEmpty: parMessage(p, errInvalidIndentation)
# XXX this needs a proper analysis;
if isKeyword(p.tok.tokType): parMessage(p, errInvalidIndentation)
break break
addSon(result, a) var a = complexOrSimpleStmt(p)
popInd(p.lex) if a.kind != nkEmpty: addSon(result, a)
else: else:
# the case statement is only needed for better error messages: # the case statement is only needed for better error messages:
case p.tok.tokType case p.tok.tokType
@ -1660,34 +1680,27 @@ proc parseStmt(p: var TParser): PNode =
else: else:
result = simpleStmt(p) result = simpleStmt(p)
if result.kind == nkEmpty: parMessage(p, errExprExpected, p.tok) if result.kind == nkEmpty: parMessage(p, errExprExpected, p.tok)
if p.tok.tokType == tkSemicolon: getTok(p) while p.tok.tokType == tkSemicolon: getTok(p)
if p.tok.tokType == tkSad: getTok(p)
proc parseAll(p: var TParser): PNode = proc parseAll(p: var TParser): PNode =
result = newNodeP(nkStmtList, p) result = newNodeP(nkStmtList, p)
while true: while p.tok.tokType != tkEof:
case p.tok.tokType if p.tok.indent != p.currInd:
of tkSad: getTok(p)
of tkDed, tkInd:
parMessage(p, errInvalidIndentation) parMessage(p, errInvalidIndentation)
getTok(p)
of tkEof: break
else:
var a = complexOrSimpleStmt(p) var a = complexOrSimpleStmt(p)
if a.kind == nkEmpty: if a.kind != nkEmpty:
addSon(result, a)
else:
parMessage(p, errExprExpected, p.tok) parMessage(p, errExprExpected, p.tok)
# bugfix: consume a token here to prevent an endless loop: # bugfix: consume a token here to prevent an endless loop:
getTok(p) getTok(p)
addSon(result, a)
proc parseTopLevelStmt(p: var TParser): PNode = proc parseTopLevelStmt(p: var TParser): PNode =
result = ast.emptyNode result = ast.emptyNode
while true: while true:
if p.tok.indent > 0: parMessage(p, errInvalidIndentation)
case p.tok.tokType case p.tok.tokType
of tkSad, tkSemicolon: getTok(p) of tkSemicolon: getTok(p)
of tkDed, tkInd:
parMessage(p, errInvalidIndentation)
getTok(p)
of tkEof: break of tkEof: break
else: else:
result = complexOrSimpleStmt(p) result = complexOrSimpleStmt(p)
@ -1703,4 +1716,3 @@ proc parseString(s: string, filename: string = "", line: int = 0): PNode =
result = parser.parseAll result = parser.parseAll
CloseParser(parser) CloseParser(parser)

4
compiler/renderer.nim
View file

@ -81,13 +81,13 @@ proc addTok(g: var TSrcGen, kind: TTokType, s: string) =
proc addPendingNL(g: var TSrcGen) = proc addPendingNL(g: var TSrcGen) =
if g.pendingNL >= 0: if g.pendingNL >= 0:
addTok(g, tkInd, "\n" & repeatChar(g.pendingNL)) addTok(g, tkSpaces, "\n" & repeatChar(g.pendingNL))
g.lineLen = g.pendingNL g.lineLen = g.pendingNL
g.pendingNL = - 1 g.pendingNL = - 1
proc putNL(g: var TSrcGen, indent: int) = proc putNL(g: var TSrcGen, indent: int) =
if g.pendingNL >= 0: addPendingNL(g) if g.pendingNL >= 0: addPendingNL(g)
else: addTok(g, tkInd, "\n") else: addTok(g, tkSpaces, "\n")
g.pendingNL = indent g.pendingNL = indent
g.lineLen = indent g.lineLen = indent

12
todo.txt
View file

@ -7,6 +7,18 @@ version 0.9.2
- acyclic vs prunable; introduce GC hints - acyclic vs prunable; introduce GC hints
- CGEN: ``restrict`` pragma + backend support; computed goto support - CGEN: ``restrict`` pragma + backend support; computed goto support
- document NimMain and check whether it works for threading - document NimMain and check whether it works for threading
- parser/grammar: enforce 'simpleExpr' more often --> doesn't work; tkProc is
part of primary!
* check that of branches can only receive even simpler expressions, don't
allow of (var x = 23; nkIdent)
* document the new grammar: ^+ ^* operators; indentation handling
* remove rules in the manual as it's too hard to keep it up to date
* improve rules to contain the AST structure
* the typeDesc/expr unification is weird and only necessary because of
the ambiguous a[T] construct: It would be easy to support a[expr] for
generics but require a[.typeDesc] if that's required; this would also
allow [.ref T.](x) for a more general type conversion construct; for
templates that would work too: T([.ref int])
Bugs Bugs