jyapayne/Nim
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

fixes #10665 (#13141) [backport]

Browse source
This commit is contained in:
Andreas Rumpf 2020-01-14 13:51:28 +01:00 • committed by GitHub
commit fadbd8f70d
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 101 additions and 45 deletions
Download patch file Download diff file Expand all files Collapse all files

48
compiler/parser.nim
View file

@ -26,6 +26,9 @@ when isMainModule:
outp.write matches[0], "\L" outp.write matches[0], "\L"
outp.close outp.close
import ".." / tools / grammar_nanny
checkGrammarFile()
import import
llstream, lexer, idents, strutils, ast, msgs, options, lineinfos, llstream, lexer, idents, strutils, ast, msgs, options, lineinfos,
pathutils pathutils
@ -750,11 +753,10 @@ proc commandExpr(p: var TParser; r: PNode; mode: TPrimaryMode): PNode =
proc primarySuffix(p: var TParser, r: PNode, proc primarySuffix(p: var TParser, r: PNode,
baseIndent: int, mode: TPrimaryMode): PNode = baseIndent: int, mode: TPrimaryMode): PNode =
#| primarySuffix = '(' (exprColonEqExpr comma?)* ')' doBlocks? #| primarySuffix = '(' (exprColonEqExpr comma?)* ')'
#| | doBlocks
#| | '.' optInd symbol generalizedLit? #| | '.' optInd symbol generalizedLit?
#| | '[' optInd indexExprList optPar ']' #| | '[' optInd exprColonEqExprList optPar ']'
#| | '{' optInd indexExprList optPar '}' #| | '{' optInd exprColonEqExprList optPar '}'
#| | &( '`'|IDENT|literal|'cast'|'addr'|'type') expr # command syntax #| | &( '`'|IDENT|literal|'cast'|'addr'|'type') expr # command syntax
result = r result = r
@ -908,7 +910,7 @@ proc parseIfExpr(p: var TParser, kind: TNodeKind): PNode =
p.currInd = oldInd p.currInd = oldInd
proc parsePragma(p: var TParser): PNode = proc parsePragma(p: var TParser): PNode =
#| pragma = '{.' optInd (exprColonExpr comma?)* optPar ('.}' | '}') #| pragma = '{.' optInd (exprColonEqExpr comma?)* optPar ('.}' | '}')
result = newNodeP(nkPragma, p) result = newNodeP(nkPragma, p)
inc p.inPragma inc p.inPragma
when defined(nimpretty): when defined(nimpretty):
@ -937,8 +939,8 @@ proc parsePragma(p: var TParser): PNode =
dec p.em.keepIndents dec p.em.keepIndents
proc identVis(p: var TParser; allowDot=false): PNode = proc identVis(p: var TParser; allowDot=false): PNode =
#| identVis = symbol opr? # postfix position #| identVis = symbol OPR? # postfix position
#| identVisDot = symbol '.' optInd symbol opr? #| identVisDot = symbol '.' optInd symbol OPR?
var a = parseSymbol(p) var a = parseSymbol(p)
if p.tok.tokType == tkOpr: if p.tok.tokType == tkOpr:
when defined(nimpretty): when defined(nimpretty):
@ -973,7 +975,7 @@ type
proc parseIdentColonEquals(p: var TParser, flags: TDeclaredIdentFlags): PNode = proc parseIdentColonEquals(p: var TParser, flags: TDeclaredIdentFlags): PNode =
#| declColonEquals = identWithPragma (comma identWithPragma)* comma? #| declColonEquals = identWithPragma (comma identWithPragma)* comma?
#| (':' optInd typeDesc)? ('=' optInd expr)? #| (':' optInd typeDesc)? ('=' optInd expr)?
#| identColonEquals = ident (comma ident)* comma? #| identColonEquals = IDENT (comma IDENT)* comma?
#| (':' optInd typeDesc)? ('=' optInd expr)?) #| (':' optInd typeDesc)? ('=' optInd expr)?)
var a: PNode var a: PNode
result = newNodeP(nkIdentDefs, p) result = newNodeP(nkIdentDefs, p)
@ -1006,7 +1008,7 @@ proc parseIdentColonEquals(p: var TParser, flags: TDeclaredIdentFlags): PNode =
proc parseTuple(p: var TParser, indentAllowed = false): PNode = proc parseTuple(p: var TParser, indentAllowed = false): PNode =
#| inlTupleDecl = 'tuple' #| inlTupleDecl = 'tuple'
#| [' optInd (identColonEquals (comma/semicolon)?)* optPar ']' #| '[' optInd (identColonEquals (comma/semicolon)?)* optPar ']'
#| extTupleDecl = 'tuple' #| extTupleDecl = 'tuple'
#| COMMENT? (IND{>} identColonEquals (IND{=} identColonEquals)*)? #| COMMENT? (IND{>} identColonEquals (IND{=} identColonEquals)*)?
#| tupleClass = 'tuple' #| tupleClass = 'tuple'
@ -1104,7 +1106,7 @@ proc optPragmas(p: var TParser): PNode =
result = p.emptyNode result = p.emptyNode
proc parseDoBlock(p: var TParser; info: TLineInfo): PNode = proc parseDoBlock(p: var TParser; info: TLineInfo): PNode =
#| doBlock = 'do' paramListArrow pragmas? colcom stmt #| doBlock = 'do' paramListArrow pragma? colcom stmt
let params = parseParamList(p, retColon=false) let params = parseParamList(p, retColon=false)
let pragmas = optPragmas(p) let pragmas = optPragmas(p)
colcom(p, result) colcom(p, result)
@ -1115,7 +1117,7 @@ proc parseDoBlock(p: var TParser; info: TLineInfo): PNode =
genericParams = p.emptyNode, pragmas = pragmas, exceptions = p.emptyNode) genericParams = p.emptyNode, pragmas = pragmas, exceptions = p.emptyNode)
proc parseProcExpr(p: var TParser; isExpr: bool; kind: TNodeKind): PNode = proc parseProcExpr(p: var TParser; isExpr: bool; kind: TNodeKind): PNode =
#| procExpr = 'proc' paramListColon pragmas? ('=' COMMENT? stmt)? #| procExpr = 'proc' paramListColon pragma? ('=' COMMENT? stmt)?
# either a proc type or a anonymous proc # either a proc type or a anonymous proc
let info = parLineInfo(p) let info = parLineInfo(p)
getTok(p) getTok(p)
@ -1214,11 +1216,11 @@ proc parseExpr(p: var TParser): PNode =
#| expr = (blockExpr #| expr = (blockExpr
#| | ifExpr #| | ifExpr
#| | whenExpr #| | whenExpr
#| | caseExpr #| | caseStmt
#| | forExpr #| | forExpr
#| | tryExpr) #| | tryExpr)
#| / simpleExpr #| / simpleExpr
case p.tok.tokType: case p.tok.tokType
of tkBlock: of tkBlock:
nimprettyDontTouch: nimprettyDontTouch:
result = parseBlock(p) result = parseBlock(p)
@ -1248,7 +1250,7 @@ proc parseTypeClass(p: var TParser): PNode
proc primary(p: var TParser, mode: TPrimaryMode): PNode = proc primary(p: var TParser, mode: TPrimaryMode): PNode =
#| typeKeyw = 'var' | 'out' | 'ref' | 'ptr' | 'shared' | 'tuple' #| typeKeyw = 'var' | 'out' | 'ref' | 'ptr' | 'shared' | 'tuple'
#| | 'proc' | 'iterator' | 'distinct' | 'object' | 'enum' #| | 'proc' | 'iterator' | 'distinct' | 'object' | 'enum'
#| primary = typeKeyw typeDescK #| primary = typeKeyw optInd typeDesc
#| / prefixOperator* identOrLiteral primarySuffix* #| / prefixOperator* identOrLiteral primarySuffix*
#| / 'bind' primary #| / 'bind' primary
if isOperator(p.tok): if isOperator(p.tok):
@ -1420,8 +1422,7 @@ proc parseExprStmt(p: var TParser): PNode =
#| exprStmt = simpleExpr #| exprStmt = simpleExpr
#| (( '=' optInd expr colonBody? ) #| (( '=' optInd expr colonBody? )
#| / ( expr ^+ comma #| / ( expr ^+ comma
#| doBlocks #| postExprBlocks
#| / macroColon
#| ))? #| ))?
var a = simpleExpr(p) var a = simpleExpr(p)
if p.tok.tokType == tkEquals: if p.tok.tokType == tkEquals:
@ -1468,6 +1469,9 @@ proc parseImport(p: var TParser, kind: TNodeKind): PNode =
#| importStmt = 'import' optInd expr #| importStmt = 'import' optInd expr
#| ((comma expr)* #| ((comma expr)*
#| / 'except' optInd (expr ^+ comma)) #| / 'except' optInd (expr ^+ comma))
#| exportStmt = 'export' optInd expr
#| ((comma expr)*
#| / 'except' optInd (expr ^+ comma))
result = newNodeP(kind, p) result = newNodeP(kind, p)
getTok(p) # skip `import` or `export` getTok(p) # skip `import` or `export`
optInd(p, result) optInd(p, result)
@ -1506,7 +1510,7 @@ proc parseIncludeStmt(p: var TParser): PNode =
#expectNl(p) #expectNl(p)
proc parseFromStmt(p: var TParser): PNode = proc parseFromStmt(p: var TParser): PNode =
#| fromStmt = 'from' moduleName 'import' optInd expr (comma expr)* #| fromStmt = 'from' expr 'import' optInd expr (comma expr)*
result = newNodeP(nkFromStmt, p) result = newNodeP(nkFromStmt, p)
getTok(p) # skip `from` getTok(p) # skip `from`
optInd(p, result) optInd(p, result)
@ -1790,7 +1794,7 @@ type
proc parseSection(p: var TParser, kind: TNodeKind, proc parseSection(p: var TParser, kind: TNodeKind,
defparser: TDefParser): PNode = defparser: TDefParser): PNode =
#| section(p) = COMMENT? p / (IND{>} (p / COMMENT)^+IND{=} DED) #| section(RULE) = COMMENT? RULE / (IND{>} (RULE / COMMENT)^+IND{=} DED)
result = newNodeP(kind, p) result = newNodeP(kind, p)
if kind != nkTypeSection: getTok(p) if kind != nkTypeSection: getTok(p)
skipComment(p, result) skipComment(p, result)
@ -1818,7 +1822,7 @@ proc parseSection(p: var TParser, kind: TNodeKind,
parMessage(p, errIdentifierExpected, p.tok) parMessage(p, errIdentifierExpected, p.tok)
proc parseEnum(p: var TParser): PNode = proc parseEnum(p: var TParser): PNode =
#| enum = 'enum' optInd (symbol optPragmas optInd ('=' optInd expr COMMENT?)? comma?)+ #| enum = 'enum' optInd (symbol pragma? optInd ('=' optInd expr COMMENT?)? comma?)+
result = newNodeP(nkEnumTy, p) result = newNodeP(nkEnumTy, p)
getTok(p) getTok(p)
result.add(p.emptyNode) result.add(p.emptyNode)
@ -2104,7 +2108,7 @@ proc parseVarTuple(p: var TParser): PNode =
eat(p, tkParRi) eat(p, tkParRi)
proc parseVariable(p: var TParser): PNode = proc parseVariable(p: var TParser): PNode =
#| colonBody = colcom stmt doBlocks? #| colonBody = colcom stmt postExprBlocks?
#| variable = (varTuple / identColonEquals) colonBody? indAndComment #| variable = (varTuple / identColonEquals) colonBody? indAndComment
if p.tok.tokType == tkParLe: if p.tok.tokType == tkParLe:
result = parseVarTuple(p) result = parseVarTuple(p)
@ -2116,7 +2120,7 @@ proc parseVariable(p: var TParser): PNode =
indAndComment(p, result) indAndComment(p, result)
proc parseConstant(p: var TParser): PNode = proc parseConstant(p: var TParser): PNode =
#| constant = (parseVarTuple / identWithPragma) (colon typeDesc)? '=' optInd expr indAndComment #| constant = (varTuple / identWithPragma) (colon typeDesc)? '=' optInd expr indAndComment
if p.tok.tokType == tkParLe: result = parseVarTuple(p) if p.tok.tokType == tkParLe: result = parseVarTuple(p)
else: else:
result = newNodeP(nkConstDef, p) result = newNodeP(nkConstDef, p)

48
doc/grammar.txt
View file

@ -54,53 +54,52 @@ identOrLiteral = generalizedLit | symbol | literal
| castExpr | castExpr
tupleConstr = '(' optInd (exprColonEqExpr comma?)* optPar ')' tupleConstr = '(' optInd (exprColonEqExpr comma?)* optPar ')'
arrayConstr = '[' optInd (exprColonEqExpr comma?)* optPar ']' arrayConstr = '[' optInd (exprColonEqExpr comma?)* optPar ']'
primarySuffix = '(' (exprColonEqExpr comma?)* ')' doBlocks? primarySuffix = '(' (exprColonEqExpr comma?)* ')'
| doBlocks
| '.' optInd symbol generalizedLit? | '.' optInd symbol generalizedLit?
| '[' optInd indexExprList optPar ']' | '[' optInd exprColonEqExprList optPar ']'
| '{' optInd indexExprList optPar '}' | '{' optInd exprColonEqExprList optPar '}'
| &( '`'|IDENT|literal|'cast'|'addr'|'type') expr # command syntax | &( '`'|IDENT|literal|'cast'|'addr'|'type') expr # command syntax
condExpr = expr colcom expr optInd condExpr = expr colcom expr optInd
('elif' expr colcom expr optInd)* ('elif' expr colcom expr optInd)*
'else' colcom expr 'else' colcom expr
ifExpr = 'if' condExpr ifExpr = 'if' condExpr
whenExpr = 'when' condExpr whenExpr = 'when' condExpr
pragma = '{.' optInd (exprColonExpr comma?)* optPar ('.}' | '}') pragma = '{.' optInd (exprColonEqExpr comma?)* optPar ('.}' | '}')
identVis = symbol opr? # postfix position identVis = symbol OPR? # postfix position
identVisDot = symbol '.' optInd symbol opr? identVisDot = symbol '.' optInd symbol OPR?
identWithPragma = identVis pragma? identWithPragma = identVis pragma?
identWithPragmaDot = identVisDot pragma? identWithPragmaDot = identVisDot pragma?
declColonEquals = identWithPragma (comma identWithPragma)* comma? declColonEquals = identWithPragma (comma identWithPragma)* comma?
(':' optInd typeDesc)? ('=' optInd expr)? (':' optInd typeDesc)? ('=' optInd expr)?
identColonEquals = ident (comma ident)* comma? identColonEquals = IDENT (comma IDENT)* comma?
(':' optInd typeDesc)? ('=' optInd expr)?) (':' optInd typeDesc)? ('=' optInd expr)?)
inlTupleDecl = 'tuple' inlTupleDecl = 'tuple'
[' optInd (identColonEquals (comma/semicolon)?)* optPar ']' '[' optInd (identColonEquals (comma/semicolon)?)* optPar ']'
extTupleDecl = 'tuple' extTupleDecl = 'tuple'
COMMENT? (IND{>} identColonEquals (IND{=} identColonEquals)*)? COMMENT? (IND{>} identColonEquals (IND{=} identColonEquals)*)?
tupleClass = 'tuple' tupleClass = 'tuple'
paramList = '(' declColonEquals ^* (comma/semicolon) ')' paramList = '(' declColonEquals ^* (comma/semicolon) ')'
paramListArrow = paramList? ('->' optInd typeDesc)? paramListArrow = paramList? ('->' optInd typeDesc)?
paramListColon = paramList? (':' optInd typeDesc)? paramListColon = paramList? (':' optInd typeDesc)?
doBlock = 'do' paramListArrow pragmas? colcom stmt doBlock = 'do' paramListArrow pragma? colcom stmt
procExpr = 'proc' paramListColon pragmas? ('=' COMMENT? stmt)? procExpr = 'proc' paramListColon pragma? ('=' COMMENT? stmt)?
distinct = 'distinct' optInd typeDesc distinct = 'distinct' optInd typeDesc
forStmt = 'for' (identWithPragma ^+ comma) 'in' expr colcom stmt forStmt = 'for' (identWithPragma ^+ comma) 'in' expr colcom stmt
forExpr = forStmt forExpr = forStmt
expr = (blockExpr expr = (blockExpr
| ifExpr | ifExpr
| whenExpr | whenExpr
| caseExpr | caseStmt
| forExpr | forExpr
| tryExpr) | tryExpr)
/ simpleExpr / simpleExpr
typeKeyw = 'var' | 'out' | 'ref' | 'ptr' | 'shared' | 'tuple' typeKeyw = 'var' | 'out' | 'ref' | 'ptr' | 'shared' | 'tuple'
| 'proc' | 'iterator' | 'distinct' | 'object' | 'enum' | 'proc' | 'iterator' | 'distinct' | 'object' | 'enum'
primary = typeKeyw typeDescK primary = typeKeyw optInd typeDesc
/ prefixOperator* identOrLiteral primarySuffix* / prefixOperator* identOrLiteral primarySuffix*
/ 'bind' primary / 'bind' primary
typeDesc = simpleExpr typeDesc = simpleExpr ('not' expr)?
typeDefAux = simpleExpr typeDefAux = simpleExpr ('not' expr)?
| 'concept' typeClass | 'concept' typeClass
postExprBlocks = ':' stmt? ( IND{=} doBlock postExprBlocks = ':' stmt? ( IND{=} doBlock
| IND{=} 'of' exprList ':' stmt | IND{=} 'of' exprList ':' stmt
@ -110,14 +109,16 @@ postExprBlocks = ':' stmt? ( IND{=} doBlock
exprStmt = simpleExpr exprStmt = simpleExpr
(( '=' optInd expr colonBody? ) (( '=' optInd expr colonBody? )
/ ( expr ^+ comma / ( expr ^+ comma
doBlocks postExprBlocks
/ macroColon
))? ))?
importStmt = 'import' optInd expr importStmt = 'import' optInd expr
((comma expr)* ((comma expr)*
/ 'except' optInd (expr ^+ comma)) / 'except' optInd (expr ^+ comma))
exportStmt = 'export' optInd expr
((comma expr)*
/ 'except' optInd (expr ^+ comma))
includeStmt = 'include' optInd expr ^+ comma includeStmt = 'include' optInd expr ^+ comma
fromStmt = 'from' moduleName 'import' optInd expr (comma expr)* fromStmt = 'from' expr 'import' optInd expr (comma expr)*
returnStmt = 'return' optInd expr? returnStmt = 'return' optInd expr?
raiseStmt = 'raise' optInd expr? raiseStmt = 'raise' optInd expr?
yieldStmt = 'yield' optInd expr? yieldStmt = 'yield' optInd expr?
@ -157,9 +158,8 @@ indAndComment = (IND{>} COMMENT)? | COMMENT?
routine = optInd identVis pattern? genericParamList? routine = optInd identVis pattern? genericParamList?
paramListColon pragma? ('=' COMMENT? stmt)? indAndComment paramListColon pragma? ('=' COMMENT? stmt)? indAndComment
commentStmt = COMMENT commentStmt = COMMENT
section(p) = COMMENT? p / (IND{>} (p / COMMENT)^+IND{=} DED) section(RULE) = COMMENT? RULE / (IND{>} (RULE / COMMENT)^+IND{=} DED)
constant = identWithPragma (colon typeDesc)? '=' optInd expr indAndComment enum = 'enum' optInd (symbol pragma? optInd ('=' optInd expr COMMENT?)? comma?)+
enum = 'enum' optInd (symbol optInd ('=' optInd expr COMMENT?)? comma?)+
objectWhen = 'when' expr colcom objectPart COMMENT? objectWhen = 'when' expr colcom objectPart COMMENT?
('elif' expr colcom objectPart COMMENT?)* ('elif' expr colcom objectPart COMMENT?)*
('else' colcom objectPart COMMENT?)? ('else' colcom objectPart COMMENT?)?
@ -177,10 +177,12 @@ typeClassParam = ('var' | 'out')? symbol
typeClass = typeClassParam ^* ',' (pragma)? ('of' typeDesc ^* ',')? typeClass = typeClassParam ^* ',' (pragma)? ('of' typeDesc ^* ',')?
&IND{>} stmt &IND{>} stmt
typeDef = identWithPragmaDot genericParamList? '=' optInd typeDefAux typeDef = identWithPragmaDot genericParamList? '=' optInd typeDefAux
indAndComment? / identVisDot genericParamList? pragma '=' optInd typeDefAux
indAndComment? indAndComment?
varTuple = '(' optInd identWithPragma ^+ comma optPar ')' '=' optInd expr varTuple = '(' optInd identWithPragma ^+ comma optPar ')' '=' optInd expr
colonBody = colcom stmt doBlocks? colonBody = colcom stmt postExprBlocks?
variable = (varTuple / identColonEquals) colonBody? indAndComment variable = (varTuple / identColonEquals) colonBody? indAndComment
constant = (varTuple / identWithPragma) (colon typeDesc)? '=' optInd expr indAndComment
bindStmt = 'bind' optInd qualifiedIdent ^+ comma bindStmt = 'bind' optInd qualifiedIdent ^+ comma
mixinStmt = 'mixin' optInd qualifiedIdent ^+ comma mixinStmt = 'mixin' optInd qualifiedIdent ^+ comma
pragmaStmt = pragma (':' COMMENT? stmt)? pragmaStmt = pragma (':' COMMENT? stmt)?

3
doc/manual.rst
View file

@ -555,6 +555,9 @@ following characters::
@ $ ~ & % | @ $ ~ & % |
! ? ^ . : \ ! ? ^ . : \
(The grammar uses the terminal OPR to refer to operator symbols as
defined here.)
These keywords are also operators: These keywords are also operators:
``and or not xor shl shr div mod in notin is isnot of``. ``and or not xor shl shr div mod in notin is isnot of``.

47
tools/grammar_nanny.nim Normal file
View file

@ -0,0 +1,47 @@
## Simple tool to check for obvious mistakes in Nim's
## grammar.txt file.
import std / [strutils, sets]
import ".." / compiler / [
llstream, ast, lexer, options, msgs, idents,
lineinfos, pathutils]
proc checkGrammarFileImpl(cache: IdentCache, config: ConfigRef) =
var f = AbsoluteFile"doc/grammar.txt"
let data = readFile(f.string).multiReplace({"IND{=}": "SAME_IND", "'": "\""})
var stream = llStreamOpen(data)
var declaredSyms = initHashSet[string]()
var usedSyms = initHashSet[string]()
if stream != nil:
declaredSyms.incl "section" # special case for 'section(RULE)' in the grammar
var
L: TLexer
tok: TToken
initToken(tok)
openLexer(L, f, stream, cache, config)
# load the first token:
rawGetTok(L, tok)
var word = ""
while tok.tokType != tkEof:
#printTok(config, tok)
if isKeyword(tok.tokType) or tok.tokType == tkSymbol:
word = tok.ident.s
rawGetTok(L, tok)
if tok.tokType == tkEquals:
declaredSyms.incl word
rawGetTok(L, tok)
elif not allCharsInSet(word, {'A'..'Z', '0'..'9', '_'}):
usedSyms.incl word
else:
rawGetTok(L, tok)
for u in usedSyms:
if u notin declaredSyms:
echo "Undeclared non-terminal: ", u
closeLexer(L)
else:
rawMessage(config, errGenerated, "cannot open file: " & f.string)
proc checkGrammarFile* =
checkGrammarFileImpl(newIdentCache(), newConfigRef())