nimState -> gState

This commit is contained in:
Joey Yakimowich-Payne 2020-04-22 20:20:34 -06:00
commit 1b4591df35
5 changed files with 136 additions and 164 deletions

View file

@ -104,9 +104,9 @@ proc newConstDef(gState: State, node: TSNode, fname = "", fval = ""): PNode =
if name.Bl:
# Name skipped or overridden since blank
result = nimState.getOverrideOrSkip(node, origname, nskConst)
result = gState.getOverrideOrSkip(node, origname, nskConst)
elif valident.kind != nkNone:
if nimState.addNewIdentifer(name):
if gState.addNewIdentifer(name):
# const X* = Y
#
# nkConstDef(
@ -1400,9 +1400,9 @@ proc addEnum(gState: State, node: TSNode) =
for (fname, fval, cexprNode) in fvalSections:
var fval = fval
if cexprNode.isSome:
fval = "(" & $nimState.parseCExpression(nimState.getNodeVal(cexprNode.get()), name) & ")." & name
fval = "(" & $gState.parseCExpression(gState.getNodeVal(cexprNode.get()), name) & ")." & name
# Cannot use newConstDef() since parseString(fval) adds backticks to and/or
nimState.constSection.add nimState.parseString(&"const {fname}* = {fval}")[0][0]
gState.constSection.add gState.parseString(&"const {fname}* = {fval}")[0][0]
# Add other names
if node.getName() == "type_definition" and node.len > 1:

View file

@ -7,12 +7,12 @@ proc handleError*(conf: ConfigRef, info: TLineInfo, msg: TMsgKind, arg: string)
if msg < warnMin:
raise newException(Exception, msgKindToString(msg))
proc parseString*(nimState: NimState, str: string): PNode =
proc parseString*(gState: State, str: string): PNode =
# Parse a string into Nim AST - use custom error handler that raises
# an exception rather than exiting on failure
try:
result = parseString(
str, nimState.identCache, nimState.config, errorHandler = handleError
str, gState.identCache, gState.config, errorHandler = handleError
)
except:
decho getCurrentExceptionMsg()

View file

@ -30,28 +30,12 @@ import "."/[globals, getters, comphelp, tshelp]
# for where Nim can't (such as uint + -int)
type
ExprParser* = ref object
state*: NimState
code*: string
name*: string
ExprParseError* = object of CatchableError
proc newExprParser*(state: NimState, code: string, name = ""): ExprParser =
ExprParser(state: state, code: code, name: name)
template techo(msg: varargs[string, `$`]) =
block:
let nimState {.inject.} = exprParser.state
decho join(msg, "")
template val(node: TSNode): string =
exprParser.code.getNodeVal(node)
gState.currentExpr.getNodeVal(node)
proc mode(exprParser: ExprParser): string =
exprParser.state.gState.mode
proc getIdent(exprParser: ExprParser, identName: string, kind = nskConst, parent = ""): PNode =
proc getExprIdent*(gState: State, identName: string, kind = nskConst, parent = ""): PNode =
## Gets a cPlugin transformed identifier from `identName`
##
## Returns PNode(nkNone) if the identifier is blank
@ -59,19 +43,19 @@ proc getIdent(exprParser: ExprParser, identName: string, kind = nskConst, parent
var ident = identName
if ident != "_":
# Process the identifier through cPlugin
ident = exprParser.state.getIdentifier(ident, kind, parent)
ident = gState.getIdentifier(ident, kind, parent)
if kind == nskType:
result = exprParser.state.getIdent(ident)
elif ident.nBl and ident in exprParser.state.constIdentifiers:
if exprParser.name.nBl:
ident = ident & "." & exprParser.name
result = exprParser.state.getIdent(ident)
result = gState.getIdent(ident)
elif ident.nBl and ident in gState.constIdentifiers:
if gState.currentTyCastName.nBl:
ident = ident & "." & gState.currentTyCastName
result = gState.getIdent(ident)
proc getIdent(exprParser: ExprParser, node: TSNode, kind = nskConst, parent = ""): PNode =
proc getExprIdent*(gState: State, node: TSNode, kind = nskConst, parent = ""): PNode =
## Gets a cPlugin transformed identifier from `identName`
##
## Returns PNode(nkNone) if the identifier is blank
exprParser.getIdent(node.val, kind, parent)
gState.getExprIdent(node.val, kind, parent)
proc parseChar(charStr: string): uint8 {.inline.} =
## Parses a character literal out of a string. This is needed
@ -125,53 +109,60 @@ proc getCharLit(charStr: string): PNode {.inline.} =
result = newNode(nkCharLit)
result.intVal = parseChar(charStr).int64
proc getFloatNode(number, suffix: string): PNode {.inline.} =
## Get a Nim float node from a C float expression + suffix
let floatSuffix = number[number.len-1]
try:
case floatSuffix
of 'l', 'L':
# TODO: handle long double (128 bits)
# result = newNode(nkFloat128Lit)
result = newFloatNode(nkFloat64Lit, parseFloat(number[0 ..< number.len - 1]))
of 'f', 'F':
result = newFloatNode(nkFloat64Lit, parseFloat(number[0 ..< number.len - 1]))
else:
result = newFloatNode(nkFloatLit, parseFloat(number))
except ValueError:
raise newException(ExprParseError, &"Could not parse float value \"{number}\".")
proc getIntNode(number, suffix: string): PNode {.inline.} =
## Get a Nim int node from a C integer expression + suffix
case suffix
of "u", "U":
result = newNode(nkUintLit)
of "l", "L":
result = newNode(nkInt32Lit)
of "ul", "UL":
result = newNode(nkUint32Lit)
of "ll", "LL":
result = newNode(nkInt64Lit)
of "ull", "ULL":
result = newNode(nkUint64Lit)
else:
result = newNode(nkIntLit)
# I realize these regex are wasteful on performance, but
# couldn't come up with a better idea.
if number.contains(re"0[xX]"):
result.intVal = parseHexInt(number)
result.flags = {nfBase16}
elif number.contains(re"0[bB]"):
result.intVal = parseBinInt(number)
result.flags = {nfBase2}
elif number.contains(re"0[oO]"):
result.intVal = parseOctInt(number)
result.flags = {nfBase8}
else:
result.intVal = parseInt(number)
proc getNumNode(number, suffix: string): PNode {.inline.} =
## Convert a C number to a Nim number PNode
result = newNode(nkNone)
if number.contains("."):
let floatSuffix = number[number.len-1]
try:
case floatSuffix
of 'l', 'L':
# TODO: handle long double (128 bits)
# result = newNode(nkFloat128Lit)
result = newFloatNode(nkFloat64Lit, parseFloat(number[0 ..< number.len - 1]))
of 'f', 'F':
result = newFloatNode(nkFloat64Lit, parseFloat(number[0 ..< number.len - 1]))
else:
result = newFloatNode(nkFloatLit, parseFloat(number))
except ValueError:
raise newException(ExprParseError, &"Could not parse float value \"{number}\".")
getFloatNode(number, suffix)
else:
case suffix
of "u", "U":
result = newNode(nkUintLit)
of "l", "L":
result = newNode(nkInt32Lit)
of "ul", "UL":
result = newNode(nkUint32Lit)
of "ll", "LL":
result = newNode(nkInt64Lit)
of "ull", "ULL":
result = newNode(nkUint64Lit)
else:
result = newNode(nkIntLit)
getIntNode(number, suffix)
# I realize these regex are wasteful on performance, but
# couldn't come up with a better idea.
if number.contains(re"0[xX]"):
result.intVal = parseHexInt(number)
result.flags = {nfBase16}
elif number.contains(re"0[bB]"):
result.intVal = parseBinInt(number)
result.flags = {nfBase2}
elif number.contains(re"0[oO]"):
result.intVal = parseOctInt(number)
result.flags = {nfBase8}
else:
result.intVal = parseInt(number)
proc processNumberLiteral(exprParser: ExprParser, node: TSNode): PNode =
proc processNumberLiteral(gState: State, node: TSNode): PNode =
## Parse a number literal from a TSNode. Can be a float, hex, long, etc
result = newNode(nkNone)
let nodeVal = node.val
@ -189,17 +180,17 @@ proc processNumberLiteral(exprParser: ExprParser, node: TSNode): PNode =
if result.kind != nkNone and prefix == "-":
result = nkPrefix.newTree(
exprParser.state.getIdent("-"),
gState.getIdent("-"),
result
)
else:
raise newException(ExprParseError, &"Could not find a number in number_literal: \"{nodeVal}\"")
proc processCharacterLiteral(exprParser: ExprParser, node: TSNode): PNode =
proc processCharacterLiteral(gState: State, node: TSNode): PNode =
let val = node.val
result = getCharLit(val[1 ..< val.len - 1])
proc processStringLiteral(exprParser: ExprParser, node: TSNode): PNode =
proc processStringLiteral(gState: State, node: TSNode): PNode =
let
nodeVal = node.val
strVal = nodeVal[1 ..< nodeVal.len - 1]
@ -215,9 +206,9 @@ proc processStringLiteral(exprParser: ExprParser, node: TSNode): PNode =
result = newStrNode(nkStrLit, nimStr)
proc processTSNode(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode
proc processTSNode(gState: State, node: TSNode, typeofNode: var PNode): PNode
proc processShiftExpression(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode =
proc processShiftExpression(gState: State, node: TSNode, typeofNode: var PNode): PNode =
result = newNode(nkInfix)
let
left = node[0]
@ -227,24 +218,24 @@ proc processShiftExpression(exprParser: ExprParser, node: TSNode, typeofNode: va
case shiftSym
of "<<":
result.add exprParser.state.getIdent("shl")
result.add gState.getIdent("shl")
of ">>":
result.add exprParser.state.getIdent("shr")
result.add gState.getIdent("shr")
else:
raise newException(ExprParseError, &"Unsupported shift symbol \"{shiftSym}\"")
let leftNode = exprParser.processTSNode(left, typeofNode)
let leftNode = gState.processTSNode(left, typeofNode)
# If the typeofNode is nil, set it
# to be the leftNode because C's type coercion
# happens left to right, and we want to emulate it
if typeofNode.isNil:
typeofNode = nkCall.newTree(
exprParser.state.getIdent("typeof"),
gState.getIdent("typeof"),
leftNode
)
let rightNode = exprParser.processTSNode(right, typeofNode)
let rightNode = gState.processTSNode(right, typeofNode)
result.add leftNode
result.add nkCall.newTree(
@ -252,15 +243,15 @@ proc processShiftExpression(exprParser: ExprParser, node: TSNode, typeofNode: va
rightNode
)
proc processParenthesizedExpr(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode =
proc processParenthesizedExpr(gState: State, node: TSNode, typeofNode: var PNode): PNode =
result = newNode(nkPar)
for i in 0 ..< node.len():
result.add(exprParser.processTSNode(node[i], typeofNode))
result.add(gState.processTSNode(node[i], typeofNode))
proc processCastExpression(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode =
proc processCastExpression(gState: State, node: TSNode, typeofNode: var PNode): PNode =
result = nkCast.newTree(
exprParser.processTSNode(node[0], typeofNode),
exprParser.processTSNode(node[1], typeofNode)
gState.processTSNode(node[0], typeofNode),
gState.processTSNode(node[1], typeofNode)
)
proc getNimUnarySym(csymbol: string): string =
@ -292,7 +283,7 @@ proc getNimBinarySym(csymbol: string): string =
else:
raise newException(ExprParseError, &"Unsupported binary symbol \"{csymbol}\"")
proc processBinaryExpression(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode =
proc processBinaryExpression(gState: State, node: TSNode, typeofNode: var PNode): PNode =
# Node has left and right children ie: (2 + 7)
result = newNode(nkInfix)
@ -302,16 +293,16 @@ proc processBinaryExpression(exprParser: ExprParser, node: TSNode, typeofNode: v
binarySym = node.tsNodeChild(1).val.strip()
nimSym = getNimBinarySym(binarySym)
result.add exprParser.state.getIdent(nimSym)
let leftNode = exprParser.processTSNode(left, typeofNode)
result.add gState.getIdent(nimSym)
let leftNode = gState.processTSNode(left, typeofNode)
if typeofNode.isNil:
typeofNode = nkCall.newTree(
exprParser.state.getIdent("typeof"),
gState.getIdent("typeof"),
leftNode
)
let rightNode = exprParser.processTSNode(right, typeofNode)
let rightNode = gState.processTSNode(right, typeofNode)
result.add leftNode
result.add nkCall.newTree(
@ -319,7 +310,7 @@ proc processBinaryExpression(exprParser: ExprParser, node: TSNode, typeofNode: v
rightNode
)
proc processUnaryExpression(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode =
proc processUnaryExpression(gState: State, node: TSNode, typeofNode: var PNode): PNode =
result = newNode(nkPar)
let
@ -333,31 +324,31 @@ proc processUnaryExpression(exprParser: ExprParser, node: TSNode, typeofNode: va
# Might be bad because we are overwriting the type
# There's probably a better way of doing this
if typeofNode.isNil:
typeofNode = exprParser.state.getIdent("int64")
typeofNode = gState.getIdent("int64")
result.add nkPrefix.newTree(
exprParser.state.getIdent(unarySym),
gState.getIdent(unarySym),
nkPar.newTree(
nkCall.newTree(
exprParser.state.getIdent("int64"),
exprParser.processTSNode(child, typeofNode)
gState.getIdent("int64"),
gState.processTSNode(child, typeofNode)
)
)
)
else:
result.add nkPrefix.newTree(
exprParser.state.getIdent(nimSym),
exprParser.processTSNode(child, typeofNode)
gState.getIdent(nimSym),
gState.processTSNode(child, typeofNode)
)
proc processUnaryOrBinaryExpression(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode =
proc processUnaryOrBinaryExpression(gState: State, node: TSNode, typeofNode: var PNode): PNode =
if node.len > 1:
# Node has left and right children ie: (2 + 7)
# Make sure the statement is of the same type as the left
# hand argument, since some expressions return a differing
# type than the input types (2/3 == float)
let binExpr = processBinaryExpression(exprParser, node, typeofNode)
let binExpr = processBinaryExpression(gState, node, typeofNode)
# Note that this temp var binExpr is needed for some reason, or else we get a segfault
result = nkCall.newTree(
typeofNode,
@ -366,37 +357,37 @@ proc processUnaryOrBinaryExpression(exprParser: ExprParser, node: TSNode, typeof
elif node.len() == 1:
# Node has only one child, ie -(20 + 7)
result = processUnaryExpression(exprParser, node, typeofNode)
result = processUnaryExpression(gState, node, typeofNode)
else:
raise newException(ExprParseError, &"Invalid {node.getName()} \"{node.val}\"")
proc processSizeofExpression(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode =
proc processSizeofExpression(gState: State, node: TSNode, typeofNode: var PNode): PNode =
result = nkCall.newTree(
exprParser.state.getIdent("sizeof"),
exprParser.processTSNode(node[0], typeofNode)
gState.getIdent("sizeof"),
gState.processTSNode(node[0], typeofNode)
)
proc processTSNode(exprParser: ExprParser, node: TSNode, typeofNode: var PNode): PNode =
proc processTSNode(gState: State, node: TSNode, typeofNode: var PNode): PNode =
## Handle all of the types of expressions here. This proc gets called recursively
## in the processX procs and will drill down to sub nodes.
result = newNode(nkNone)
let nodeName = node.getName()
techo "NODE: ", nodeName, ", VAL: ", node.val
decho "NODE: ", nodeName, ", VAL: ", node.val
case nodeName
of "number_literal":
# Input -> 0x1234FE, 1231, 123u, 123ul, 123ull, 1.334f
# Output -> 0x1234FE, 1231, 123'u, 123'u32, 123'u64, 1.334
result = exprParser.processNumberLiteral(node)
result = gState.processNumberLiteral(node)
of "string_literal":
# Input -> "foo\0\x42"
# Output -> "foo\0"
result = exprParser.processStringLiteral(node)
result = gState.processStringLiteral(node)
of "char_literal":
# Input -> 'F', '\034' // Octal, '\x5A' // Hex, '\r' // escape sequences
# Output ->
result = exprParser.processCharacterLiteral(node)
result = gState.processCharacterLiteral(node)
of "expression_statement", "ERROR", "translation_unit":
# Note that we're parsing partial expressions, so the TSNode might contain
# an ERROR node. If that's the case, they usually contain children with
@ -405,21 +396,21 @@ proc processTSNode(exprParser: ExprParser, node: TSNode, typeofNode: var PNode):
# Input (top level statement) -> ((1 + 3 - IDENT) - (int)400.0)
# Output -> (1 + typeof(1)(3) - typeof(1)(IDENT) - typeof(1)(cast[int](400.0))) # Type casting in case some args differ
if node.len == 1:
result = exprParser.processTSNode(node[0], typeofNode)
result = gState.processTSNode(node[0], typeofNode)
elif node.len > 1:
result = newNode(nkStmtListExpr)
for i in 0 ..< node.len:
result.add exprParser.processTSNode(node[i], typeofNode)
result.add gState.processTSNode(node[i], typeofNode)
else:
raise newException(ExprParseError, &"Node type \"{nodeName}\" has no children")
of "parenthesized_expression":
# Input -> (IDENT - OTHERIDENT)
# Output -> (IDENT - typeof(IDENT)(OTHERIDENT)) # Type casting in case OTHERIDENT is a slightly different type (uint vs int)
result = exprParser.processParenthesizedExpr(node, typeofNode)
result = gState.processParenthesizedExpr(node, typeofNode)
of "sizeof_expression":
# Input -> sizeof(char)
# Output -> sizeof(cchar)
result = exprParser.processSizeofExpression(node, typeofNode)
result = gState.processSizeofExpression(node, typeofNode)
# binary_expression from the new treesitter upgrade should work here
# once we upgrade
of "math_expression", "logical_expression", "relational_expression",
@ -434,55 +425,61 @@ proc processTSNode(exprParser: ExprParser, node: TSNode, typeofNode: var PNode):
# typeof(a)(a > typeof(a)(b))
# typeof(a)(a <= typeof(a)(b))
# typeof(a)(a >= typeof(a)(b))
result = exprParser.processUnaryOrBinaryExpression(node, typeofNode)
result = gState.processUnaryOrBinaryExpression(node, typeofNode)
of "shift_expression":
# Input -> a >> b, a << b
# Output -> a shr typeof(a)(b), a shl typeof(a)(b)
result = exprParser.processShiftExpression(node, typeofNode)
result = gState.processShiftExpression(node, typeofNode)
of "cast_expression":
# Input -> (int) a
# Output -> cast[cint](a)
result = exprParser.processCastExpression(node, typeofNode)
result = gState.processCastExpression(node, typeofNode)
# Why are these node types named true/false?
of "true", "false":
# Input -> true, false
# Output -> true, false
result = exprParser.state.parseString(node.val)
result = gState.parseString(node.val)
of "type_descriptor", "sized_type_specifier":
# Input -> int, unsigned int, long int, etc
# Output -> cint, cuint, clong, etc
let ty = getType(node.val)
if ty.len > 0:
# If ty is not empty, one of C's builtin types has been found
result = exprParser.getIdent(ty, nskType, parent=node.getName())
result = gState.getExprIdent(ty, nskType, parent=node.getName())
else:
result = exprParser.getIdent(node.val, nskType, parent=node.getName())
result = gState.getExprIdent(node.val, nskType, parent=node.getName())
if result.kind == nkNone:
raise newException(ExprParseError, &"Missing type specifier \"{node.val}\"")
of "identifier":
# Input -> IDENT
# Output -> IDENT (if found in sym table, else error)
result = exprParser.getIdent(node, parent=node.getName())
result = gState.getExprIdent(node, parent=node.getName())
if result.kind == nkNone:
raise newException(ExprParseError, &"Missing identifier \"{node.val}\"")
else:
raise newException(ExprParseError, &"Unsupported node type \"{nodeName}\" for node \"{node.val}\"")
techo "NODE RESULT: ", result
decho "NODE RESULT: ", result
proc parseCExpression*(state: NimState, code: string, name = ""): PNode =
proc parseCExpression*(gState: State, code: string, name = ""): PNode =
## Convert the C string to a nim PNode tree
gState.currentExpr = code
gState.currentTyCastName = name
result = newNode(nkNone)
# This is used for keeping track of the type of the first
# symbol used for type casting
var tnode: PNode = nil
let exprParser = newExprParser(state, code, name)
try:
withCodeAst(exprParser.code, exprParser.mode):
result = exprParser.processTSNode(root, tnode)
withCodeAst(gState.currentExpr, gState.mode):
result = gState.processTSNode(root, tnode)
except ExprParseError as e:
techo e.msg
decho e.msg
result = newNode(nkNone)
except Exception as e:
techo "UNEXPECTED EXCEPTION: ", e.msg
result = newNode(nkNone)
decho "UNEXPECTED EXCEPTION: ", e.msg
result = newNode(nkNone)
# Clear the state
gState.currentExpr = ""
gState.currentTyCastName = ""

View file

@ -1,4 +1,4 @@
import sequtils, sets, tables
import sequtils, sets, tables, strutils
import regex
@ -93,6 +93,9 @@ type
currentHeader*, impShort*, sourceFile*: string
# Used for the exprparser.nim module
currentExpr*, currentTyCastName*: string
data*: seq[tuple[name, val: string]]
nodeBranch*: seq[string]
@ -119,6 +122,6 @@ when not declared(CIMPORT):
else:
gState.outputHandle.writeLine(args)
template decho*(str: untyped): untyped =
template decho*(args: varargs[string, `$`]): untyped =
if gState.debug:
gecho str.getCommented()
gecho join(args, "").getCommented()

View file

@ -15,44 +15,16 @@ proc process(gState: State, path: string, astTable: AstTable) =
else:
gState.code = readFile(path)
<<<<<<< HEAD
doAssert gState.code.nBl, "Empty file or preprocessor error"
if gState.mode == "c":
doAssert parser.tsParserSetLanguage(treeSitterC()), "Failed to load C parser"
elif gState.mode == "cpp":
doAssert parser.tsParserSetLanguage(treeSitterCpp()), "Failed to load C++ parser"
else:
doAssert false, &"Invalid parser {gState.mode}"
var
tree = parser.tsParserParseString(nil, gState.code.cstring, gState.code.len.uint32)
root = tree.tsTreeRootNode()
defer:
tree.tsTreeDelete()
if gState.past:
gecho gState.printLisp(root)
elif gState.pnim:
if Feature.ast2 in gState.feature:
ast2.parseNim(gState, path, root)
else:
ast.parseNim(gState, path, root, astTable)
elif gState.preprocess:
gecho gState.code
=======
withCodeAst(gState.code, gState.mode):
if gState.past:
gecho gState.printLisp(root)
elif gState.pnim:
if Feature.ast2 in gState.feature:
ast2.printNim(gState, path, root)
ast2.parseNim(gState, path, root)
else:
ast.printNim(gState, path, root, astTable)
ast.parseNim(gState, path, root, astTable)
elif gState.preprocess:
gecho gState.code
>>>>>>> Update based on comments from review. Need to add more docs and reorg to use gstate
# CLI processing with default values
proc main(