fixes merge conflicts

This commit is contained in:
Andreas Rumpf 2018-09-11 17:27:47 +02:00
commit f7d1902043
887 changed files with 48670 additions and 42944 deletions

View file

@ -17,6 +17,8 @@
# In fact the grammar is generated from this file:
when isMainModule:
# Leave a note in grammar.txt that it is generated:
#| # This file is generated by compiler/parser.nim.
import pegs
var outp = open("doc/grammar.txt", fmWrite)
for line in lines("compiler/parser.nim"):
@ -25,7 +27,10 @@ when isMainModule:
outp.close
import
llstream, lexer, idents, strutils, ast, astalgo, msgs
llstream, lexer, idents, strutils, ast, astalgo, msgs, options, lineinfos
when defined(nimpretty2):
import layouter
type
TParser* = object # A TParser object represents a file that
@ -38,10 +43,16 @@ type
tok*: TToken # The current token
inPragma*: int # Pragma level
inSemiStmtList*: int
emptyNode: PNode
when defined(nimpretty2):
em: Emitter
SymbolMode = enum
smNormal, smAllowNil, smAfterDot
TPrimaryMode = enum
pmNormal, pmTypeDesc, pmTypeDef, pmSkipSuffix
proc parseAll*(p: var TParser): PNode
proc closeParser*(p: var TParser)
proc parseTopLevelStmt*(p: var TParser): PNode
@ -73,6 +84,9 @@ proc parsePragma(p: var TParser): PNode
proc postExprBlocks(p: var TParser, x: PNode): PNode
proc parseExprStmt(p: var TParser): PNode
proc parseBlock(p: var TParser): PNode
proc primary(p: var TParser, mode: TPrimaryMode): PNode
proc simpleExprAux(p: var TParser, limit: int, mode: TPrimaryMode): PNode
# implementation
proc getTok(p: var TParser) =
@ -80,34 +94,48 @@ proc getTok(p: var TParser) =
## `tok` member.
rawGetTok(p.lex, p.tok)
p.hasProgress = true
when defined(nimpretty2):
emitTok(p.em, p.lex, p.tok)
while p.tok.tokType == tkComment:
rawGetTok(p.lex, p.tok)
emitTok(p.em, p.lex, p.tok)
proc openParser*(p: var TParser, fileIdx: int32, inputStream: PLLStream,
cache: IdentCache;
proc openParser*(p: var TParser, fileIdx: FileIndex, inputStream: PLLStream,
cache: IdentCache; config: ConfigRef;
strongSpaces=false) =
## Open a parser, using the given arguments to set up its internal state.
##
initToken(p.tok)
openLexer(p.lex, fileIdx, inputStream, cache)
openLexer(p.lex, fileIdx, inputStream, cache, config)
when defined(nimpretty2):
openEmitter(p.em, cache, config, fileIdx)
getTok(p) # read the first token
p.firstTok = true
p.strongSpaces = strongSpaces
p.emptyNode = newNode(nkEmpty)
proc openParser*(p: var TParser, filename: string, inputStream: PLLStream,
cache: IdentCache;
cache: IdentCache; config: ConfigRef;
strongSpaces=false) =
openParser(p, filename.fileInfoIdx, inputStream, cache, strongSpaces)
openParser(p, fileInfoIdx(config, filename), inputStream, cache, config, strongSpaces)
proc closeParser(p: var TParser) =
## Close a parser, freeing up its resources.
closeLexer(p.lex)
when defined(nimpretty2):
closeEmitter(p.em)
proc parMessage(p: TParser, msg: TMsgKind, arg = "") =
## Produce and emit the parser message `arg` to output.
lexMessageTok(p.lex, msg, p.tok, arg)
proc parMessage(p: TParser, msg: TMsgKind, tok: TToken) =
proc parMessage(p: TParser, msg: string, tok: TToken) =
## Produce and emit a parser message to output about the token `tok`
parMessage(p, msg, prettyTok(tok))
parMessage(p, errGenerated, msg % prettyTok(tok))
proc parMessage(p: TParser, arg: string) =
## Produce and emit the parser message `arg` to output.
lexMessageTok(p.lex, errGenerated, p.tok, arg)
template withInd(p, body: untyped) =
let oldInd = p.currInd
@ -122,8 +150,15 @@ template sameOrNoInd(p): bool = p.tok.indent == p.currInd or p.tok.indent < 0
proc rawSkipComment(p: var TParser, node: PNode) =
if p.tok.tokType == tkComment:
if node != nil:
if node.comment == nil: node.comment = ""
add(node.comment, p.tok.literal)
when not defined(nimNoNilSeqs):
if node.comment == nil: node.comment = ""
when defined(nimpretty):
if p.tok.commentOffsetB > p.tok.commentOffsetA:
add node.comment, fileSection(p.lex.config, p.lex.fileIdx, p.tok.commentOffsetA, p.tok.commentOffsetB)
else:
add node.comment, p.tok.literal
else:
add(node.comment, p.tok.literal)
else:
parMessage(p, errInternal, "skipComment")
getTok(p)
@ -134,6 +169,12 @@ proc skipComment(p: var TParser, node: PNode) =
proc flexComment(p: var TParser, node: PNode) =
if p.tok.indent < 0 or realInd(p): rawSkipComment(p, node)
const
errInvalidIndentation = "invalid indentation"
errIdentifierExpected = "identifier expected, but got '$1'"
errExprExpected = "expression expected, but found '$1'"
errTokenExpected = "'$1' expected"
proc skipInd(p: var TParser) =
if p.tok.indent >= 0:
if not realInd(p): parMessage(p, errInvalidIndentation)
@ -152,11 +193,11 @@ proc getTokNoInd(p: var TParser) =
proc expectIdentOrKeyw(p: TParser) =
if p.tok.tokType != tkSymbol and not isKeyword(p.tok.tokType):
lexMessage(p.lex, errIdentifierExpected, prettyTok(p.tok))
lexMessage(p.lex, errGenerated, errIdentifierExpected % prettyTok(p.tok))
proc expectIdent(p: TParser) =
if p.tok.tokType != tkSymbol:
lexMessage(p.lex, errIdentifierExpected, prettyTok(p.tok))
lexMessage(p.lex, errGenerated, errIdentifierExpected % prettyTok(p.tok))
proc eat(p: var TParser, tokType: TTokType) =
## Move the parser to the next token if the current token is of type
@ -164,7 +205,8 @@ proc eat(p: var TParser, tokType: TTokType) =
if p.tok.tokType == tokType:
getTok(p)
else:
lexMessageTok(p.lex, errTokenExpected, p.tok, TokTypeToStr[tokType])
lexMessage(p.lex, errGenerated,
"expected: '" & TokTypeToStr[tokType] & "', but got: '" & prettyTok(p.tok) & "'")
proc parLineInfo(p: TParser): TLineInfo =
## Retrieve the line information associated with the parser's current state.
@ -210,41 +252,6 @@ proc isRightAssociative(tok: TToken): bool {.inline.} =
result = tok.tokType == tkOpr and tok.ident.s[0] == '^'
# or (let L = tok.ident.s.len; L > 1 and tok.ident.s[L-1] == '>'))
proc getPrecedence(tok: TToken, strongSpaces: bool): int =
## Calculates the precedence of the given token.
template considerStrongSpaces(x): untyped =
x + (if strongSpaces: 100 - tok.strongSpaceA.int*10 else: 0)
case tok.tokType
of tkOpr:
let L = tok.ident.s.len
let relevantChar = tok.ident.s[0]
# arrow like?
if L > 1 and tok.ident.s[L-1] == '>' and
tok.ident.s[L-2] in {'-', '~', '='}: return considerStrongSpaces(1)
template considerAsgn(value: untyped) =
result = if tok.ident.s[L-1] == '=': 1 else: value
case relevantChar
of '$', '^': considerAsgn(10)
of '*', '%', '/', '\\': considerAsgn(9)
of '~': result = 8
of '+', '-', '|': considerAsgn(8)
of '&': considerAsgn(7)
of '=', '<', '>', '!': result = 5
of '.': considerAsgn(6)
of '?': result = 2
else: considerAsgn(2)
of tkDiv, tkMod, tkShl, tkShr: result = 9
of tkIn, tkNotin, tkIs, tkIsnot, tkNot, tkOf, tkAs: result = 5
of tkDotDot: result = 6
of tkAnd: result = 4
of tkOr, tkXor, tkPtr, tkRef: result = 3
else: return -10
result = considerStrongSpaces(result)
proc isOperator(tok: TToken): bool =
## Determines if the given token is an operator type token.
tok.tokType in {tkOpr, tkDiv, tkMod, tkShl, tkShr, tkIn, tkNotin, tkIs,
@ -260,13 +267,9 @@ proc isUnary(p: TParser): bool =
proc checkBinary(p: TParser) {.inline.} =
## Check if the current parser token is a binary operator.
# we don't check '..' here as that's too annoying
if p.strongSpaces and p.tok.tokType == tkOpr:
if p.tok.tokType == tkOpr:
if p.tok.strongSpaceB > 0 and p.tok.strongSpaceA != p.tok.strongSpaceB:
parMessage(p, errGenerated,
"Number of spaces around '$#' not consistent" %
prettyTok(p.tok))
elif p.tok.strongSpaceA notin {0,1,2,4,8}:
parMessage(p, errGenerated, "Number of spaces must be 0,1,2,4 or 8")
parMessage(p, warnInconsistentSpacing, prettyTok(p.tok))
#| module = stmt ^* (';' / IND{=})
#|
@ -301,6 +304,8 @@ proc colcom(p: var TParser, n: PNode) =
eat(p, tkColon)
skipComment(p, n)
const tkBuiltInMagics = {tkType, tkStatic, tkAddr}
proc parseSymbol(p: var TParser, mode = smNormal): PNode =
#| symbol = '`' (KEYW|IDENT|literal|(operator|'('|')'|'['|']'|'{'|'}'|'=')+)+ '`'
#| | IDENT | KEYW
@ -309,7 +314,7 @@ proc parseSymbol(p: var TParser, mode = smNormal): PNode =
result = newIdentNodeP(p.tok.ident, p)
getTok(p)
of tokKeywordLow..tokKeywordHigh:
if p.tok.tokType == tkAddr or p.tok.tokType == tkType or mode == smAfterDot:
if p.tok.tokType in tkBuiltInMagics or mode == smAfterDot:
# for backwards compatibility these 2 are always valid:
result = newIdentNodeP(p.tok.ident, p)
getTok(p)
@ -318,7 +323,7 @@ proc parseSymbol(p: var TParser, mode = smNormal): PNode =
getTok(p)
else:
parMessage(p, errIdentifierExpected, p.tok)
result = ast.emptyNode
result = p.emptyNode
of tkAccent:
result = newNodeP(nkAccQuoted, p)
getTok(p)
@ -349,7 +354,7 @@ proc parseSymbol(p: var TParser, mode = smNormal): PNode =
# But: this really sucks for idetools and keywords, so we don't do it
# if it is a keyword:
#if not isKeyword(p.tok.tokType): getTok(p)
result = ast.emptyNode
result = p.emptyNode
proc colonOrEquals(p: var TParser, a: PNode): PNode =
if p.tok.tokType == tkColon:
@ -377,6 +382,8 @@ proc exprColonEqExpr(p: var TParser): PNode =
proc exprList(p: var TParser, endTok: TTokType, result: PNode) =
#| exprList = expr ^+ comma
when defined(nimpretty2):
inc p.em.doIndentMore
getTok(p)
optInd(p, result)
# progress guaranteed
@ -386,20 +393,8 @@ proc exprList(p: var TParser, endTok: TTokType, result: PNode) =
if p.tok.tokType != tkComma: break
getTok(p)
optInd(p, a)
proc dotExpr(p: var TParser, a: PNode): PNode =
#| dotExpr = expr '.' optInd symbol
var info = p.parLineInfo
getTok(p)
result = newNodeI(nkDotExpr, info)
optInd(p, result)
addSon(result, a)
addSon(result, parseSymbol(p, smAfterDot))
proc qualifiedIdent(p: var TParser): PNode =
#| qualifiedIdent = symbol ('.' optInd symbol)?
result = parseSymbol(p)
if p.tok.tokType == tkDot: result = dotExpr(p, result)
when defined(nimpretty2):
dec p.em.doIndentMore
proc exprColonEqExprListAux(p: var TParser, endTok: TTokType, result: PNode) =
assert(endTok in {tkCurlyRi, tkCurlyDotRi, tkBracketRi, tkParRi})
@ -411,6 +406,9 @@ proc exprColonEqExprListAux(p: var TParser, endTok: TTokType, result: PNode) =
addSon(result, a)
if p.tok.tokType != tkComma: break
getTok(p)
# (1,) produces a tuple expression
if endTok == tkParRi and p.tok.tokType == tkParRi and result.kind == nkPar:
result.kind = nkTupleConstr
skipComment(p, a)
optPar(p)
eat(p, endTok)
@ -421,6 +419,33 @@ proc exprColonEqExprList(p: var TParser, kind: TNodeKind,
result = newNodeP(kind, p)
exprColonEqExprListAux(p, endTok, result)
proc dotExpr(p: var TParser, a: PNode): PNode =
#| dotExpr = expr '.' optInd (symbol | '[:' exprList ']')
#| explicitGenericInstantiation = '[:' exprList ']' ( '(' exprColonEqExpr ')' )?
var info = p.parLineInfo
getTok(p)
result = newNodeI(nkDotExpr, info)
optInd(p, result)
addSon(result, a)
addSon(result, parseSymbol(p, smAfterDot))
if p.tok.tokType == tkBracketLeColon and p.tok.strongSpaceA <= 0:
var x = newNodeI(nkBracketExpr, p.parLineInfo)
# rewrite 'x.y[:z]()' to 'y[z](x)'
x.add result[1]
exprList(p, tkBracketRi, x)
eat(p, tkBracketRi)
var y = newNodeI(nkCall, p.parLineInfo)
y.add x
y.add result[0]
if p.tok.tokType == tkParLe and p.tok.strongSpaceA <= 0:
exprColonEqExprListAux(p, tkParRi, y)
result = y
proc qualifiedIdent(p: var TParser): PNode =
#| qualifiedIdent = symbol ('.' optInd symbol)?
result = parseSymbol(p)
if p.tok.tokType == tkDot: result = dotExpr(p, result)
proc setOrTableConstr(p: var TParser): PNode =
#| setOrTableConstr = '{' ((exprColonEqExpr comma)* | ':' ) '}'
result = newNodeP(nkCurly, p)
@ -478,9 +503,6 @@ proc parseGStrLit(p: var TParser, a: PNode): PNode =
else:
result = a
type
TPrimaryMode = enum pmNormal, pmTypeDesc, pmTypeDef, pmSkipSuffix
proc complexOrSimpleStmt(p: var TParser): PNode
proc simpleExpr(p: var TParser, mode = pmNormal): PNode
@ -512,6 +534,7 @@ proc parsePar(p: var TParser): PNode =
result = newNodeP(nkPar, p)
getTok(p)
optInd(p, result)
flexComment(p, result)
if p.tok.tokType in {tkDiscard, tkInclude, tkIf, tkWhile, tkCase,
tkTry, tkDefer, tkFinally, tkExcept, tkFor, tkBlock,
tkConst, tkLet, tkWhen, tkVar, tkFor,
@ -551,6 +574,9 @@ proc parsePar(p: var TParser): PNode =
if p.tok.tokType == tkComma:
getTok(p)
skipComment(p, a)
# (1,) produces a tuple expression:
if p.tok.tokType == tkParRi:
result.kind = nkTupleConstr
# progress guaranteed
while p.tok.tokType != tkParRi and p.tok.tokType != tkEof:
var a = exprColonEqExpr(p)
@ -575,7 +601,7 @@ proc identOrLiteral(p: var TParser, mode: TPrimaryMode): PNode =
#| tupleConstr = '(' optInd (exprColonEqExpr comma?)* optPar ')'
#| arrayConstr = '[' optInd (exprColonEqExpr comma?)* optPar ']'
case p.tok.tokType
of tkSymbol, tkType, tkAddr:
of tkSymbol, tkBuiltInMagics:
result = newIdentNodeP(p.tok.ident, p)
getTok(p)
result = parseGStrLit(p, result)
@ -669,7 +695,7 @@ proc identOrLiteral(p: var TParser, mode: TPrimaryMode): PNode =
else:
parMessage(p, errExprExpected, p.tok)
getTok(p) # we must consume a token here to prevend endless loops!
result = ast.emptyNode
result = p.emptyNode
proc namedParams(p: var TParser, callee: PNode,
kind: TNodeKind, endTok: TTokType): PNode =
@ -679,12 +705,24 @@ proc namedParams(p: var TParser, callee: PNode,
# progress guaranteed
exprColonEqExprListAux(p, endTok, result)
proc commandParam(p: var TParser): PNode =
proc commandParam(p: var TParser, isFirstParam: var bool): PNode =
result = parseExpr(p)
if p.tok.tokType == tkDo:
result = postExprBlocks(p, result)
elif p.tok.tokType == tkEquals and not isFirstParam:
let lhs = result
result = newNodeP(nkExprEqExpr, p)
getTok(p)
addSon(result, lhs)
addSon(result, parseExpr(p))
isFirstParam = false
proc primarySuffix(p: var TParser, r: PNode, baseIndent: int): PNode =
const
tkTypeClasses = {tkRef, tkPtr, tkVar, tkStatic, tkType,
tkEnum, tkTuple, tkObject, tkProc}
proc primarySuffix(p: var TParser, r: PNode,
baseIndent: int, mode: TPrimaryMode): PNode =
#| primarySuffix = '(' (exprColonEqExpr comma?)* ')' doBlocks?
#| | doBlocks
#| | '.' optInd symbol generalizedLit?
@ -701,7 +739,14 @@ proc primarySuffix(p: var TParser, r: PNode, baseIndent: int): PNode =
case p.tok.tokType
of tkParLe:
# progress guaranteed
somePar()
if p.tok.strongSpaceA > 0:
# inside type sections, expressions such as `ref (int, bar)`
# are parsed as a nkCommand with a single tuple argument (nkPar)
if mode == pmTypeDef:
result = newNodeP(nkCommand, p)
result.addSon r
result.addSon primary(p, pmNormal)
break
result = namedParams(p, result, nkCall, tkParRi)
if result.len > 1 and result.sons[1].kind == nkExprColonExpr:
result.kind = nkObjConstr
@ -717,17 +762,24 @@ proc primarySuffix(p: var TParser, r: PNode, baseIndent: int): PNode =
# progress guaranteed
somePar()
result = namedParams(p, result, nkCurlyExpr, tkCurlyRi)
of tkSymbol, tkAccent, tkIntLit..tkCharLit, tkNil, tkCast, tkAddr, tkType:
if p.inPragma == 0:
of tkSymbol, tkAccent, tkIntLit..tkCharLit, tkNil, tkCast,
tkOpr, tkDotDot, tkTypeClasses - {tkRef, tkPtr}:
# XXX: In type sections we allow the free application of the
# command syntax, with the exception of expressions such as
# `foo ref` or `foo ptr`. Unfortunately, these two are also
# used as infix operators for the memory regions feature and
# the current parsing rules don't play well here.
if p.inPragma == 0 and (isUnary(p) or p.tok.tokType notin {tkOpr, tkDotDot}):
# actually parsing {.push hints:off.} as {.push(hints:off).} is a sweet
# solution, but pragmas.nim can't handle that
let a = result
result = newNodeP(nkCommand, p)
addSon(result, a)
var isFirstParam = true
when true:
# progress NOT guaranteed
p.hasProgress = false
addSon result, commandParam(p)
addSon result, commandParam(p, isFirstParam)
if not p.hasProgress: break
else:
while p.tok.tokType != tkEof:
@ -741,9 +793,6 @@ proc primarySuffix(p: var TParser, r: PNode, baseIndent: int): PNode =
else:
break
proc primary(p: var TParser, mode: TPrimaryMode): PNode
proc simpleExprAux(p: var TParser, limit: int, mode: TPrimaryMode): PNode
proc parseOperators(p: var TParser, headNode: PNode,
limit: int, mode: TPrimaryMode): PNode =
result = headNode
@ -778,7 +827,11 @@ proc simpleExprAux(p: var TParser, limit: int, mode: TPrimaryMode): PNode =
result = parseOperators(p, result, limit, mode)
proc simpleExpr(p: var TParser, mode = pmNormal): PNode =
when defined(nimpretty2):
inc p.em.doIndentMore
result = simpleExprAux(p, -1, mode)
when defined(nimpretty2):
dec p.em.doIndentMore
proc parseIfExpr(p: var TParser, kind: TNodeKind): PNode =
#| condExpr = expr colcom expr optInd
@ -818,7 +871,6 @@ proc parseIfExpr(p: var TParser, kind: TNodeKind): PNode =
if realInd(p):
p.currInd = p.tok.indent
wasIndented = true
echo result.info, " yes ", p.currInd
addSon(branch, parseExpr(p))
result.add branch
while sameInd(p) or not wasIndented:
@ -854,8 +906,12 @@ proc parsePragma(p: var TParser): PNode =
getTok(p)
skipComment(p, a)
optPar(p)
if p.tok.tokType in {tkCurlyDotRi, tkCurlyRi}: getTok(p)
else: parMessage(p, errTokenExpected, ".}")
if p.tok.tokType in {tkCurlyDotRi, tkCurlyRi}:
when defined(nimpretty2):
if p.tok.tokType == tkCurlyRi: curlyRiWasPragma(p.em)
getTok(p)
else:
parMessage(p, "expected '.}'")
dec p.inPragma
proc identVis(p: var TParser; allowDot=false): PNode =
@ -863,6 +919,8 @@ proc identVis(p: var TParser; allowDot=false): PNode =
#| identVisDot = symbol '.' optInd symbol opr?
var a = parseSymbol(p)
if p.tok.tokType == tkOpr:
when defined(nimpretty2):
starWasExportMarker(p.em)
result = newNodeP(nkPostfix, p)
addSon(result, newIdentNodeP(p.tok.ident, p))
addSon(result, a)
@ -916,7 +974,7 @@ proc parseIdentColonEquals(p: var TParser, flags: TDeclaredIdentFlags): PNode =
else:
addSon(result, newNodeP(nkEmpty, p))
if p.tok.tokType != tkEquals and withBothOptional notin flags:
parMessage(p, errColonOrEqualsExpected, p.tok)
parMessage(p, "':' or '=' expected, but got '$1'", p.tok)
if p.tok.tokType == tkEquals:
getTok(p)
optInd(p, result)
@ -940,6 +998,8 @@ proc parseTuple(p: var TParser, indentAllowed = false): PNode =
var a = parseIdentColonEquals(p, {})
addSon(result, a)
if p.tok.tokType notin {tkComma, tkSemiColon}: break
when defined(nimpretty2):
commaWasSemicolon(p.em)
getTok(p)
skipComment(p, a)
optPar(p)
@ -962,6 +1022,8 @@ proc parseTuple(p: var TParser, indentAllowed = false): PNode =
parMessage(p, errIdentifierExpected, p.tok)
break
if not sameInd(p): break
elif p.tok.tokType == tkParLe:
parMessage(p, errGenerated, "the syntax for tuple types is 'tuple[...]', not 'tuple(...)'")
else:
result = newNodeP(nkTupleClassTy, p)
@ -971,7 +1033,9 @@ proc parseParamList(p: var TParser, retColon = true): PNode =
#| paramListColon = paramList? (':' optInd typeDesc)?
var a: PNode
result = newNodeP(nkFormalParams, p)
addSon(result, ast.emptyNode) # return type
addSon(result, p.emptyNode) # return type
when defined(nimpretty2):
inc p.em.doIndentMore
let hasParLe = p.tok.tokType == tkParLe and p.tok.indent < 0
if hasParLe:
getTok(p)
@ -983,11 +1047,16 @@ proc parseParamList(p: var TParser, retColon = true): PNode =
a = parseIdentColonEquals(p, {withBothOptional, withPragma})
of tkParRi:
break
of tkVar:
parMessage(p, errGenerated, "the syntax is 'parameter: var T', not 'var parameter: T'")
break
else:
parMessage(p, errTokenExpected, ")")
parMessage(p, "expected closing ')'")
break
addSon(result, a)
if p.tok.tokType notin {tkComma, tkSemiColon}: break
when defined(nimpretty2):
commaWasSemicolon(p.em)
getTok(p)
skipComment(p, a)
optPar(p)
@ -1000,13 +1069,15 @@ proc parseParamList(p: var TParser, retColon = true): PNode =
result.sons[0] = parseTypeDesc(p)
elif not retColon and not hasParle:
# Mark as "not there" in order to mark for deprecation in the semantic pass:
result = ast.emptyNode
result = p.emptyNode
when defined(nimpretty2):
dec p.em.doIndentMore
proc optPragmas(p: var TParser): PNode =
if p.tok.tokType == tkCurlyDotLe and (p.tok.indent < 0 or realInd(p)):
result = parsePragma(p)
else:
result = ast.emptyNode
result = p.emptyNode
proc parseDoBlock(p: var TParser; info: TLineInfo): PNode =
#| doBlock = 'do' paramListArrow pragmas? colcom stmt
@ -1015,7 +1086,9 @@ proc parseDoBlock(p: var TParser; info: TLineInfo): PNode =
colcom(p, result)
result = parseStmt(p)
if params.kind != nkEmpty:
result = newProcNode(nkDo, info, result, params = params, pragmas = pragmas)
result = newProcNode(nkDo, info,
body = result, params = params, name = p.emptyNode, pattern = p.emptyNode,
genericParams = p.emptyNode, pragmas = pragmas, exceptions = p.emptyNode)
proc parseProcExpr(p: var TParser; isExpr: bool; kind: TNodeKind): PNode =
#| procExpr = 'proc' paramListColon pragmas? ('=' COMMENT? stmt)?
@ -1028,9 +1101,9 @@ proc parseProcExpr(p: var TParser; isExpr: bool; kind: TNodeKind): PNode =
if p.tok.tokType == tkEquals and isExpr:
getTok(p)
skipComment(p, result)
result = newProcNode(kind, info, parseStmt(p),
params = params,
pragmas = pragmas)
result = newProcNode(kind, info, body = parseStmt(p),
params = params, name = p.emptyNode, pattern = p.emptyNode,
genericParams = p.emptyNode, pragmas = pragmas, exceptions = p.emptyNode)
else:
result = newNodeI(nkProcTy, info)
if hasSignature:
@ -1040,9 +1113,9 @@ proc parseProcExpr(p: var TParser; isExpr: bool; kind: TNodeKind): PNode =
proc isExprStart(p: TParser): bool =
case p.tok.tokType
of tkSymbol, tkAccent, tkOpr, tkNot, tkNil, tkCast, tkIf, tkFor,
tkProc, tkFunc, tkIterator, tkBind, tkAddr,
tkProc, tkFunc, tkIterator, tkBind, tkBuiltInMagics,
tkParLe, tkBracketLe, tkCurlyLe, tkIntLit..tkCharLit, tkVar, tkRef, tkPtr,
tkTuple, tkObject, tkType, tkWhen, tkCase, tkOut:
tkTuple, tkObject, tkWhen, tkCase, tkOut:
result = true
else: result = false
@ -1061,6 +1134,7 @@ proc parseTypeDescKAux(p: var TParser, kind: TNodeKind,
#| distinct = 'distinct' optInd typeDesc
result = newNodeP(kind, p)
getTok(p)
if p.tok.indent != -1 and p.tok.indent <= p.currInd: return
optInd(p, result)
if not isOperator(p.tok) and isExprStart(p):
addSon(result, primary(p, mode))
@ -1121,7 +1195,6 @@ proc primary(p: var TParser, mode: TPrimaryMode): PNode =
#| | 'proc' | 'iterator' | 'distinct' | 'object' | 'enum'
#| primary = typeKeyw typeDescK
#| / prefixOperator* identOrLiteral primarySuffix*
#| / 'static' primary
#| / 'bind' primary
if isOperator(p.tok):
let isSigil = isSigilLike(p.tok)
@ -1134,7 +1207,7 @@ proc primary(p: var TParser, mode: TPrimaryMode): PNode =
#XXX prefix operators
let baseInd = p.lex.currLineIndent
addSon(result, primary(p, pmSkipSuffix))
result = primarySuffix(p, result, baseInd)
result = primarySuffix(p, result, baseInd, mode)
else:
addSon(result, primary(p, pmNormal))
return
@ -1163,15 +1236,7 @@ proc primary(p: var TParser, mode: TPrimaryMode): PNode =
if mode == pmTypeDef:
result = parseTypeClass(p)
else:
parMessage(p, errInvalidToken, p.tok)
of tkStatic:
let info = parLineInfo(p)
getTokNoInd(p)
let next = primary(p, pmNormal)
if next.kind == nkBracket and next.sonsLen == 1:
result = newNode(nkStaticTy, info, @[next.sons[0]])
else:
result = newNode(nkStaticExpr, info, @[next])
parMessage(p, "the 'concept' keyword is only valid in 'type' sections")
of tkBind:
result = newNodeP(nkBind, p)
getTok(p)
@ -1186,7 +1251,7 @@ proc primary(p: var TParser, mode: TPrimaryMode): PNode =
let baseInd = p.lex.currLineIndent
result = identOrLiteral(p, mode)
if mode != pmSkipSuffix:
result = primarySuffix(p, result, baseInd)
result = primarySuffix(p, result, baseInd, mode)
proc parseTypeDesc(p: var TParser): PNode =
#| typeDesc = simpleExpr
@ -1215,8 +1280,8 @@ proc postExprBlocks(p: var TParser, x: PNode): PNode =
if p.tok.indent >= 0: return
var
openingParams = emptyNode
openingPragmas = emptyNode
openingParams = p.emptyNode
openingPragmas = p.emptyNode
if p.tok.tokType == tkDo:
getTok(p)
@ -1235,8 +1300,12 @@ proc postExprBlocks(p: var TParser, x: PNode): PNode =
stmtList.flags.incl nfBlockArg
if openingParams.kind != nkEmpty:
result.add newProcNode(nkDo, stmtList.info, stmtList,
params = openingParams, pragmas = openingPragmas)
result.add newProcNode(nkDo, stmtList.info, body = stmtList,
params = openingParams,
name = p.emptyNode, pattern = p.emptyNode,
genericParams = p.emptyNode,
pragmas = openingPragmas,
exceptions = p.emptyNode)
else:
result.add stmtList
@ -1273,7 +1342,7 @@ proc postExprBlocks(p: var TParser, x: PNode): PNode =
if nextBlock.kind == nkElse: break
else:
if openingParams.kind != nkEmpty:
parMessage(p, errTokenExpected, ":")
parMessage(p, "expected ':'")
proc parseExprStmt(p: var TParser): PNode =
#| exprStmt = simpleExpr
@ -1293,17 +1362,18 @@ proc parseExprStmt(p: var TParser): PNode =
addSon(result, b)
else:
# simpleExpr parsed 'p a' from 'p a, b'?
var isFirstParam = false
if p.tok.indent < 0 and p.tok.tokType == tkComma and a.kind == nkCommand:
result = a
while true:
getTok(p)
optInd(p, result)
addSon(result, commandParam(p))
addSon(result, commandParam(p, isFirstParam))
if p.tok.tokType != tkComma: break
elif p.tok.indent < 0 and isExprStart(p):
result = newNode(nkCommand, a.info, @[a])
while true:
addSon(result, commandParam(p))
addSon(result, commandParam(p, isFirstParam))
if p.tok.tokType != tkComma: break
getTok(p)
optInd(p, result)
@ -1394,10 +1464,10 @@ proc parseReturnOrRaise(p: var TParser, kind: TNodeKind): PNode =
getTok(p)
if p.tok.tokType == tkComment:
skipComment(p, result)
addSon(result, ast.emptyNode)
addSon(result, p.emptyNode)
elif p.tok.indent >= 0 and p.tok.indent <= p.currInd or not isExprStart(p):
# NL terminates:
addSon(result, ast.emptyNode)
addSon(result, p.emptyNode)
else:
var e = parseExpr(p)
e = postExprBlocks(p, e)
@ -1508,7 +1578,7 @@ proc parseTry(p: var TParser; isExpr: bool): PNode =
addSon(b, parseStmt(p))
addSon(result, b)
if b.kind == nkFinally: break
if b == nil: parMessage(p, errTokenExpected, "except")
if b == nil: parMessage(p, "expected 'except'")
proc parseExceptBlock(p: var TParser, kind: TNodeKind): PNode =
#| exceptBlock = 'except' colcom stmt
@ -1522,7 +1592,7 @@ proc parseBlock(p: var TParser): PNode =
#| blockExpr = 'block' symbol? colcom stmt
result = newNodeP(nkBlockStmt, p)
getTokNoInd(p)
if p.tok.tokType == tkColon: addSon(result, ast.emptyNode)
if p.tok.tokType == tkColon: addSon(result, p.emptyNode)
else: addSon(result, parseSymbol(p))
colcom(p, result)
addSon(result, parseStmt(p))
@ -1536,19 +1606,19 @@ proc parseStaticOrDefer(p: var TParser; k: TNodeKind): PNode =
addSon(result, parseStmt(p))
proc parseAsm(p: var TParser): PNode =
#| asmStmt = 'asm' pragma? (STR_LIT | RSTR_LIT | TRIPLE_STR_LIT)
#| asmStmt = 'asm' pragma? (STR_LIT | RSTR_LIT | TRIPLESTR_LIT)
result = newNodeP(nkAsmStmt, p)
getTokNoInd(p)
if p.tok.tokType == tkCurlyDotLe: addSon(result, parsePragma(p))
else: addSon(result, ast.emptyNode)
else: addSon(result, p.emptyNode)
case p.tok.tokType
of tkStrLit: addSon(result, newStrNodeP(nkStrLit, p.tok.literal, p))
of tkRStrLit: addSon(result, newStrNodeP(nkRStrLit, p.tok.literal, p))
of tkTripleStrLit: addSon(result,
newStrNodeP(nkTripleStrLit, p.tok.literal, p))
else:
parMessage(p, errStringLiteralExpected)
addSon(result, ast.emptyNode)
parMessage(p, "the 'asm' statement takes a string literal")
addSon(result, p.emptyNode)
return
getTok(p)
@ -1579,13 +1649,13 @@ proc parseGenericParam(p: var TParser): PNode =
optInd(p, result)
addSon(result, parseExpr(p))
else:
addSon(result, ast.emptyNode)
addSon(result, p.emptyNode)
if p.tok.tokType == tkEquals:
getTok(p)
optInd(p, result)
addSon(result, parseExpr(p))
else:
addSon(result, ast.emptyNode)
addSon(result, p.emptyNode)
proc parseGenericParamList(p: var TParser): PNode =
#| genericParamList = '[' optInd
@ -1598,6 +1668,8 @@ proc parseGenericParamList(p: var TParser): PNode =
var a = parseGenericParam(p)
addSon(result, a)
if p.tok.tokType notin {tkComma, tkSemiColon}: break
when defined(nimpretty2):
commaWasSemicolon(p.em)
getTok(p)
skipComment(p, a)
optPar(p)
@ -1621,22 +1693,22 @@ proc parseRoutine(p: var TParser, kind: TNodeKind): PNode =
optInd(p, result)
addSon(result, identVis(p))
if p.tok.tokType == tkCurlyLe and p.validInd: addSon(result, p.parsePattern)
else: addSon(result, ast.emptyNode)
else: addSon(result, p.emptyNode)
if p.tok.tokType == tkBracketLe and p.validInd:
result.add(p.parseGenericParamList)
else:
addSon(result, ast.emptyNode)
addSon(result, p.emptyNode)
addSon(result, p.parseParamList)
if p.tok.tokType == tkCurlyDotLe and p.validInd: addSon(result, p.parsePragma)
else: addSon(result, ast.emptyNode)
else: addSon(result, p.emptyNode)
# empty exception tracking:
addSon(result, ast.emptyNode)
addSon(result, p.emptyNode)
if p.tok.tokType == tkEquals and p.validInd:
getTok(p)
skipComment(p, result)
addSon(result, parseStmt(p))
else:
addSon(result, ast.emptyNode)
addSon(result, p.emptyNode)
indAndComment(p, result)
proc newCommentStmt(p: var TParser): PNode =
@ -1678,7 +1750,7 @@ proc parseSection(p: var TParser, kind: TNodeKind,
parMessage(p, errIdentifierExpected, p.tok)
proc parseConstant(p: var TParser): PNode =
#| constant = identWithPragma (colon typedesc)? '=' optInd expr indAndComment
#| constant = identWithPragma (colon typeDesc)? '=' optInd expr indAndComment
result = newNodeP(nkConstDef, p)
addSon(result, identWithPragma(p))
if p.tok.tokType == tkColon:
@ -1686,7 +1758,7 @@ proc parseConstant(p: var TParser): PNode =
optInd(p, result)
addSon(result, parseTypeDesc(p))
else:
addSon(result, ast.emptyNode)
addSon(result, p.emptyNode)
eat(p, tkEquals)
optInd(p, result)
addSon(result, parseExpr(p))
@ -1696,7 +1768,7 @@ proc parseEnum(p: var TParser): PNode =
#| enum = 'enum' optInd (symbol optInd ('=' optInd expr COMMENT?)? comma?)+
result = newNodeP(nkEnumTy, p)
getTok(p)
addSon(result, ast.emptyNode)
addSon(result, p.emptyNode)
optInd(p, result)
flexComment(p, result)
# progress guaranteed
@ -1726,7 +1798,7 @@ proc parseEnum(p: var TParser): PNode =
p.tok.tokType == tkEof:
break
if result.len <= 1:
lexMessageTok(p.lex, errIdentifierExpected, p.tok, prettyTok(p.tok))
parMessage(p, errIdentifierExpected, p.tok)
proc parseObjectPart(p: var TParser): PNode
proc parseObjectWhen(p: var TParser): PNode =
@ -1767,7 +1839,7 @@ proc parseObjectCase(p: var TParser): PNode =
addSon(a, identWithPragma(p))
eat(p, tkColon)
addSon(a, parseTypeDesc(p))
addSon(a, ast.emptyNode)
addSon(a, p.emptyNode)
addSon(result, a)
if p.tok.tokType == tkColon: getTok(p)
flexComment(p, result)
@ -1826,7 +1898,7 @@ proc parseObjectPart(p: var TParser): PNode =
result = newNodeP(nkNilLit, p)
getTok(p)
else:
result = ast.emptyNode
result = p.emptyNode
proc parseObject(p: var TParser): PNode =
#| object = 'object' pragma? ('of' typeDesc)? COMMENT? objectPart
@ -1835,19 +1907,19 @@ proc parseObject(p: var TParser): PNode =
if p.tok.tokType == tkCurlyDotLe and p.validInd:
addSon(result, parsePragma(p))
else:
addSon(result, ast.emptyNode)
addSon(result, p.emptyNode)
if p.tok.tokType == tkOf and p.tok.indent < 0:
var a = newNodeP(nkOfInherit, p)
getTok(p)
addSon(a, parseTypeDesc(p))
addSon(result, a)
else:
addSon(result, ast.emptyNode)
addSon(result, p.emptyNode)
if p.tok.tokType == tkComment:
skipComment(p, result)
# an initial IND{>} HAS to follow:
if not realInd(p):
addSon(result, emptyNode)
addSon(result, p.emptyNode)
return
addSon(result, parseObjectPart(p))
@ -1882,7 +1954,7 @@ proc parseTypeClass(p: var TParser): PNode =
if p.tok.tokType == tkCurlyDotLe and p.validInd:
addSon(result, parsePragma(p))
else:
addSon(result, ast.emptyNode)
addSon(result, p.emptyNode)
if p.tok.tokType == tkOf and p.tok.indent < 0:
var a = newNodeP(nkOfInherit, p)
getTok(p)
@ -1893,12 +1965,12 @@ proc parseTypeClass(p: var TParser): PNode =
getTok(p)
addSon(result, a)
else:
addSon(result, ast.emptyNode)
addSon(result, p.emptyNode)
if p.tok.tokType == tkComment:
skipComment(p, result)
# an initial IND{>} HAS to follow:
if not realInd(p):
addSon(result, emptyNode)
addSon(result, p.emptyNode)
else:
addSon(result, parseStmt(p))
@ -1911,14 +1983,14 @@ proc parseTypeDef(p: var TParser): PNode =
if p.tok.tokType == tkBracketLe and p.validInd:
addSon(result, parseGenericParamList(p))
else:
addSon(result, ast.emptyNode)
addSon(result, p.emptyNode)
if p.tok.tokType == tkEquals:
result.info = parLineInfo(p)
getTok(p)
optInd(p, result)
addSon(result, parseTypeDefAux(p))
else:
addSon(result, ast.emptyNode)
addSon(result, p.emptyNode)
indAndComment(p, result) # special extension!
proc parseVarTuple(p: var TParser): PNode =
@ -1933,7 +2005,7 @@ proc parseVarTuple(p: var TParser): PNode =
if p.tok.tokType != tkComma: break
getTok(p)
skipComment(p, a)
addSon(result, ast.emptyNode) # no type desc
addSon(result, p.emptyNode) # no type desc
optPar(p)
eat(p, tkParRi)
eat(p, tkEquals)
@ -1994,7 +2066,7 @@ proc simpleStmt(p: var TParser): PNode =
of tkComment: result = newCommentStmt(p)
else:
if isExprStart(p): result = parseExprStmt(p)
else: result = ast.emptyNode
else: result = p.emptyNode
if result.kind notin {nkEmpty, nkCommentStmt}: skipComment(p, result)
proc complexOrSimpleStmt(p: var TParser): PNode =
@ -2089,8 +2161,8 @@ proc parseStmt(p: var TParser): PNode =
case p.tok.tokType
of tkIf, tkWhile, tkCase, tkTry, tkFor, tkBlock, tkAsm, tkProc, tkFunc,
tkIterator, tkMacro, tkType, tkConst, tkWhen, tkVar:
parMessage(p, errComplexStmtRequiresInd)
result = ast.emptyNode
parMessage(p, "complex statement requires indentation")
result = p.emptyNode
else:
if p.inSemiStmtList > 0:
result = simpleStmt(p)
@ -2127,13 +2199,18 @@ proc parseAll(p: var TParser): PNode =
proc parseTopLevelStmt(p: var TParser): PNode =
## Implements an iterator which, when called repeatedly, returns the next
## top-level statement or emptyNode if end of stream.
result = ast.emptyNode
result = p.emptyNode
# progress guaranteed
while true:
if p.tok.indent != 0:
if p.firstTok and p.tok.indent < 0: discard
elif p.tok.tokType != tkSemiColon:
parMessage(p, errInvalidIndentation)
# special casing for better error messages:
if p.tok.tokType == tkOpr and p.tok.ident.s == "*":
parMessage(p, errGenerated,
"invalid indentation; an export marker '*' follows the declared identifier")
else:
parMessage(p, errInvalidIndentation)
p.firstTok = false
case p.tok.tokType
of tkSemiColon:
@ -2147,8 +2224,8 @@ proc parseTopLevelStmt(p: var TParser): PNode =
if result.kind == nkEmpty: parMessage(p, errExprExpected, p.tok)
break
proc parseString*(s: string; cache: IdentCache; filename: string = "";
line: int = 0;
proc parseString*(s: string; cache: IdentCache; config: ConfigRef;
filename: string = ""; line: int = 0;
errorHandler: TErrorHandler = nil): PNode =
## Parses a string into an AST, returning the top node.
## `filename` and `line`, although optional, provide info so that the
@ -2161,7 +2238,7 @@ proc parseString*(s: string; cache: IdentCache; filename: string = "";
# XXX for now the builtin 'parseStmt/Expr' functions do not know about strong
# spaces...
parser.lex.errorHandler = errorHandler
openParser(parser, filename, stream, cache, false)
openParser(parser, filename, stream, cache, config, false)
result = parser.parseAll
closeParser(parser)