Merge branch 'devel' into bugfix-2858-testament-sources-system-wide
This commit is contained in:
Oscar Campbell
commit
cdf631c508
130 changed files with 5899 additions and 5730 deletions
|
|
@ -842,7 +842,7 @@ type
|
|||
data*: TIdNodePairSeq
|
||||
|
||||
TNodePair* = object
|
||||
h*: THash # because it is expensive to compute!
|
||||
h*: Hash # because it is expensive to compute!
|
||||
key*: PNode
|
||||
val*: int
|
||||
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@
|
|||
import
|
||||
ast, hashes, intsets, strutils, options, msgs, ropes, idents, rodutils
|
||||
|
||||
proc hashNode*(p: RootRef): THash
|
||||
proc hashNode*(p: RootRef): Hash
|
||||
proc treeToYaml*(n: PNode, indent: int = 0, maxRecDepth: int = - 1): Rope
|
||||
# Convert a tree into its YAML representation; this is used by the
|
||||
# YAML code generator and it is invaluable for debugging purposes.
|
||||
|
|
@ -49,7 +49,7 @@ proc strTableGet*(t: TStrTable, name: PIdent): PSym
|
|||
|
||||
type
|
||||
TTabIter*{.final.} = object # consider all fields here private
|
||||
h*: THash # current hash
|
||||
h*: Hash # current hash
|
||||
|
||||
proc initTabIter*(ti: var TTabIter, tab: TStrTable): PSym
|
||||
proc nextIter*(ti: var TTabIter, tab: TStrTable): PSym
|
||||
|
|
@ -65,7 +65,7 @@ proc nextIter*(ti: var TTabIter, tab: TStrTable): PSym
|
|||
|
||||
type
|
||||
TIdentIter*{.final.} = object # iterator over all syms with same identifier
|
||||
h*: THash # current hash
|
||||
h*: Hash # current hash
|
||||
name*: PIdent
|
||||
|
||||
|
||||
|
|
@ -94,7 +94,7 @@ proc getSymFromList*(list: PNode, ident: PIdent, start: int = 0): PSym
|
|||
proc lookupInRecord*(n: PNode, field: PIdent): PSym
|
||||
proc getModule*(s: PSym): PSym
|
||||
proc mustRehash*(length, counter: int): bool
|
||||
proc nextTry*(h, maxHash: THash): THash {.inline.}
|
||||
proc nextTry*(h, maxHash: Hash): Hash {.inline.}
|
||||
|
||||
# ------------- table[int, int] ---------------------------------------------
|
||||
const
|
||||
|
|
@ -196,7 +196,7 @@ proc getSymFromList(list: PNode, ident: PIdent, start: int = 0): PSym =
|
|||
else: internalError(list.info, "getSymFromList")
|
||||
result = nil
|
||||
|
||||
proc hashNode(p: RootRef): THash =
|
||||
proc hashNode(p: RootRef): Hash =
|
||||
result = hash(cast[pointer](p))
|
||||
|
||||
proc mustRehash(length, counter: int): bool =
|
||||
|
|
@ -466,7 +466,7 @@ proc debug(n: PNode) =
|
|||
const
|
||||
EmptySeq = @[]
|
||||
|
||||
proc nextTry(h, maxHash: THash): THash =
|
||||
proc nextTry(h, maxHash: Hash): Hash =
|
||||
result = ((5 * h) + 1) and maxHash
|
||||
# For any initial h in range(maxHash), repeating that maxHash times
|
||||
# generates each int in range(maxHash) exactly once (see any text on
|
||||
|
|
@ -474,7 +474,7 @@ proc nextTry(h, maxHash: THash): THash =
|
|||
|
||||
proc objectSetContains(t: TObjectSet, obj: RootRef): bool =
|
||||
# returns true whether n is in t
|
||||
var h: THash = hashNode(obj) and high(t.data) # start with real hash value
|
||||
var h: Hash = hashNode(obj) and high(t.data) # start with real hash value
|
||||
while t.data[h] != nil:
|
||||
if t.data[h] == obj:
|
||||
return true
|
||||
|
|
@ -482,7 +482,7 @@ proc objectSetContains(t: TObjectSet, obj: RootRef): bool =
|
|||
result = false
|
||||
|
||||
proc objectSetRawInsert(data: var TObjectSeq, obj: RootRef) =
|
||||
var h: THash = hashNode(obj) and high(data)
|
||||
var h: Hash = hashNode(obj) and high(data)
|
||||
while data[h] != nil:
|
||||
assert(data[h] != obj)
|
||||
h = nextTry(h, high(data))
|
||||
|
|
@ -503,7 +503,7 @@ proc objectSetIncl(t: var TObjectSet, obj: RootRef) =
|
|||
|
||||
proc objectSetContainsOrIncl(t: var TObjectSet, obj: RootRef): bool =
|
||||
# returns true if obj is already in the string table:
|
||||
var h: THash = hashNode(obj) and high(t.data)
|
||||
var h: Hash = hashNode(obj) and high(t.data)
|
||||
while true:
|
||||
var it = t.data[h]
|
||||
if it == nil: break
|
||||
|
|
@ -520,7 +520,7 @@ proc objectSetContainsOrIncl(t: var TObjectSet, obj: RootRef): bool =
|
|||
result = false
|
||||
|
||||
proc tableRawGet(t: TTable, key: RootRef): int =
|
||||
var h: THash = hashNode(key) and high(t.data) # start with real hash value
|
||||
var h: Hash = hashNode(key) and high(t.data) # start with real hash value
|
||||
while t.data[h].key != nil:
|
||||
if t.data[h].key == key:
|
||||
return h
|
||||
|
|
@ -529,7 +529,7 @@ proc tableRawGet(t: TTable, key: RootRef): int =
|
|||
|
||||
proc tableSearch(t: TTable, key, closure: RootRef,
|
||||
comparator: TCmpProc): RootRef =
|
||||
var h: THash = hashNode(key) and high(t.data) # start with real hash value
|
||||
var h: Hash = hashNode(key) and high(t.data) # start with real hash value
|
||||
while t.data[h].key != nil:
|
||||
if t.data[h].key == key:
|
||||
if comparator(t.data[h].val, closure):
|
||||
|
|
@ -544,7 +544,7 @@ proc tableGet(t: TTable, key: RootRef): RootRef =
|
|||
else: result = nil
|
||||
|
||||
proc tableRawInsert(data: var TPairSeq, key, val: RootRef) =
|
||||
var h: THash = hashNode(key) and high(data)
|
||||
var h: Hash = hashNode(key) and high(data)
|
||||
while data[h].key != nil:
|
||||
assert(data[h].key != key)
|
||||
h = nextTry(h, high(data))
|
||||
|
|
@ -569,7 +569,7 @@ proc tablePut(t: var TTable, key, val: RootRef) =
|
|||
inc(t.counter)
|
||||
|
||||
proc strTableContains(t: TStrTable, n: PSym): bool =
|
||||
var h: THash = n.name.h and high(t.data) # start with real hash value
|
||||
var h: Hash = n.name.h and high(t.data) # start with real hash value
|
||||
while t.data[h] != nil:
|
||||
if (t.data[h] == n):
|
||||
return true
|
||||
|
|
@ -577,7 +577,7 @@ proc strTableContains(t: TStrTable, n: PSym): bool =
|
|||
result = false
|
||||
|
||||
proc strTableRawInsert(data: var TSymSeq, n: PSym) =
|
||||
var h: THash = n.name.h and high(data)
|
||||
var h: Hash = n.name.h and high(data)
|
||||
if sfImmediate notin n.flags:
|
||||
# fast path:
|
||||
while data[h] != nil:
|
||||
|
|
@ -606,7 +606,7 @@ proc strTableRawInsert(data: var TSymSeq, n: PSym) =
|
|||
|
||||
proc symTabReplaceRaw(data: var TSymSeq, prevSym: PSym, newSym: PSym) =
|
||||
assert prevSym.name.h == newSym.name.h
|
||||
var h: THash = prevSym.name.h and high(data)
|
||||
var h: Hash = prevSym.name.h and high(data)
|
||||
while data[h] != nil:
|
||||
if data[h] == prevSym:
|
||||
data[h] = newSym
|
||||
|
|
@ -640,7 +640,7 @@ proc strTableIncl*(t: var TStrTable, n: PSym): bool {.discardable.} =
|
|||
# It is essential that `n` is written nevertheless!
|
||||
# This way the newest redefinition is picked by the semantic analyses!
|
||||
assert n.name != nil
|
||||
var h: THash = n.name.h and high(t.data)
|
||||
var h: Hash = n.name.h and high(t.data)
|
||||
var replaceSlot = -1
|
||||
while true:
|
||||
var it = t.data[h]
|
||||
|
|
@ -666,7 +666,7 @@ proc strTableIncl*(t: var TStrTable, n: PSym): bool {.discardable.} =
|
|||
result = false
|
||||
|
||||
proc strTableGet(t: TStrTable, name: PIdent): PSym =
|
||||
var h: THash = name.h and high(t.data)
|
||||
var h: Hash = name.h and high(t.data)
|
||||
while true:
|
||||
result = t.data[h]
|
||||
if result == nil: break
|
||||
|
|
@ -694,7 +694,7 @@ proc nextIdentIter(ti: var TIdentIter, tab: TStrTable): PSym =
|
|||
|
||||
proc nextIdentExcluding*(ti: var TIdentIter, tab: TStrTable,
|
||||
excluding: IntSet): PSym =
|
||||
var h: THash = ti.h and high(tab.data)
|
||||
var h: Hash = ti.h and high(tab.data)
|
||||
var start = h
|
||||
result = tab.data[h]
|
||||
while result != nil:
|
||||
|
|
@ -743,7 +743,7 @@ proc hasEmptySlot(data: TIdPairSeq): bool =
|
|||
result = false
|
||||
|
||||
proc idTableRawGet(t: TIdTable, key: int): int =
|
||||
var h: THash
|
||||
var h: Hash
|
||||
h = key and high(t.data) # start with real hash value
|
||||
while t.data[h].key != nil:
|
||||
if t.data[h].key.id == key:
|
||||
|
|
@ -772,7 +772,7 @@ iterator pairs*(t: TIdTable): tuple[key: int, value: RootRef] =
|
|||
yield (t.data[i].key.id, t.data[i].val)
|
||||
|
||||
proc idTableRawInsert(data: var TIdPairSeq, key: PIdObj, val: RootRef) =
|
||||
var h: THash
|
||||
var h: Hash
|
||||
h = key.id and high(data)
|
||||
while data[h].key != nil:
|
||||
assert(data[h].key.id != key.id)
|
||||
|
|
@ -805,7 +805,7 @@ iterator idTablePairs*(t: TIdTable): tuple[key: PIdObj, val: RootRef] =
|
|||
if not isNil(t.data[i].key): yield (t.data[i].key, t.data[i].val)
|
||||
|
||||
proc idNodeTableRawGet(t: TIdNodeTable, key: PIdObj): int =
|
||||
var h: THash
|
||||
var h: Hash
|
||||
h = key.id and high(t.data) # start with real hash value
|
||||
while t.data[h].key != nil:
|
||||
if t.data[h].key.id == key.id:
|
||||
|
|
@ -824,7 +824,7 @@ proc idNodeTableGetLazy*(t: TIdNodeTable, key: PIdObj): PNode =
|
|||
result = idNodeTableGet(t, key)
|
||||
|
||||
proc idNodeTableRawInsert(data: var TIdNodePairSeq, key: PIdObj, val: PNode) =
|
||||
var h: THash
|
||||
var h: Hash
|
||||
h = key.id and high(data)
|
||||
while data[h].key != nil:
|
||||
assert(data[h].key.id != key.id)
|
||||
|
|
@ -863,7 +863,7 @@ proc initIITable(x: var TIITable) =
|
|||
for i in countup(0, StartSize - 1): x.data[i].key = InvalidKey
|
||||
|
||||
proc iiTableRawGet(t: TIITable, key: int): int =
|
||||
var h: THash
|
||||
var h: Hash
|
||||
h = key and high(t.data) # start with real hash value
|
||||
while t.data[h].key != InvalidKey:
|
||||
if t.data[h].key == key: return h
|
||||
|
|
@ -876,7 +876,7 @@ proc iiTableGet(t: TIITable, key: int): int =
|
|||
else: result = InvalidKey
|
||||
|
||||
proc iiTableRawInsert(data: var TIIPairSeq, key, val: int) =
|
||||
var h: THash
|
||||
var h: Hash
|
||||
h = key and high(data)
|
||||
while data[h].key != InvalidKey:
|
||||
assert(data[h].key != key)
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ import
|
|||
|
||||
type
|
||||
TSections = array[TSymKind, Rope]
|
||||
TDocumentor = object of rstgen.TRstGenerator
|
||||
TDocumentor = object of rstgen.RstGenerator
|
||||
modDesc: Rope # module description
|
||||
id: int # for generating IDs
|
||||
toc, section: TSections
|
||||
|
|
@ -29,7 +29,7 @@ type
|
|||
PDoc* = ref TDocumentor ## Alias to type less.
|
||||
|
||||
proc compilerMsgHandler(filename: string, line, col: int,
|
||||
msgKind: rst.TMsgKind, arg: string) {.procvar.} =
|
||||
msgKind: rst.MsgKind, arg: string) {.procvar.} =
|
||||
# translate msg kind:
|
||||
var k: msgs.TMsgKind
|
||||
case msgKind
|
||||
|
|
@ -53,7 +53,7 @@ proc docgenFindFile(s: string): string {.procvar.} =
|
|||
|
||||
proc parseRst(text, filename: string,
|
||||
line, column: int, hasToc: var bool,
|
||||
rstOptions: TRstParseOptions): PRstNode =
|
||||
rstOptions: RstParseOptions): PRstNode =
|
||||
result = rstParse(text, filename, line, column, hasToc, rstOptions,
|
||||
docgenFindFile, compilerMsgHandler)
|
||||
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@
|
|||
# id. This module is essential for the compiler's performance.
|
||||
|
||||
import
|
||||
hashes, strutils
|
||||
hashes, strutils, etcpriv
|
||||
|
||||
type
|
||||
TIdObj* = object of RootObj
|
||||
|
|
@ -23,7 +23,7 @@ type
|
|||
TIdent*{.acyclic.} = object of TIdObj
|
||||
s*: string
|
||||
next*: PIdent # for hash-table chaining
|
||||
h*: THash # hash value of s
|
||||
h*: Hash # hash value of s
|
||||
|
||||
var firstCharIsCS*: bool = true
|
||||
var buckets*: array[0..4096 * 2 - 1, PIdent]
|
||||
|
|
@ -37,6 +37,8 @@ proc cmpIgnoreStyle(a, b: cstring, blen: int): int =
|
|||
while j < blen:
|
||||
while a[i] == '_': inc(i)
|
||||
while b[j] == '_': inc(j)
|
||||
while isMagicIdentSeparatorRune(a, i): inc(i, magicIdentSeparatorRuneByteWidth)
|
||||
while isMagicIdentSeparatorRune(b, j): inc(j, magicIdentSeparatorRuneByteWidth)
|
||||
# tolower inlined:
|
||||
var aa = a[i]
|
||||
var bb = b[j]
|
||||
|
|
@ -65,7 +67,7 @@ proc cmpExact(a, b: cstring, blen: int): int =
|
|||
|
||||
var wordCounter = 1
|
||||
|
||||
proc getIdent*(identifier: cstring, length: int, h: THash): PIdent =
|
||||
proc getIdent*(identifier: cstring, length: int, h: Hash): PIdent =
|
||||
var idx = h and high(buckets)
|
||||
result = buckets[idx]
|
||||
var last: PIdent = nil
|
||||
|
|
@ -99,7 +101,7 @@ proc getIdent*(identifier: string): PIdent =
|
|||
result = getIdent(cstring(identifier), len(identifier),
|
||||
hashIgnoreStyle(identifier))
|
||||
|
||||
proc getIdent*(identifier: string, h: THash): PIdent =
|
||||
proc getIdent*(identifier: string, h: Hash): PIdent =
|
||||
result = getIdent(cstring(identifier), len(identifier), h)
|
||||
|
||||
proc identEq*(id: PIdent, name: string): bool =
|
||||
|
|
|
|||
|
|
@ -986,6 +986,15 @@ proc genAddr(p: PProc, n: PNode, r: var TCompRes) =
|
|||
else: internalError(n.sons[0].info, "expr(nkBracketExpr, " & $ty.kind & ')')
|
||||
else: internalError(n.sons[0].info, "genAddr")
|
||||
|
||||
proc genProcForSymIfNeeded(p: PProc, s: PSym) =
|
||||
if not p.g.generatedSyms.containsOrIncl(s.id):
|
||||
let newp = genProc(p, s)
|
||||
var owner = p
|
||||
while owner != nil and owner.prc != s.owner:
|
||||
owner = owner.up
|
||||
if owner != nil: add(owner.locals, newp)
|
||||
else: add(p.g.code, newp)
|
||||
|
||||
proc genSym(p: PProc, n: PNode, r: var TCompRes) =
|
||||
var s = n.sym
|
||||
case s.kind
|
||||
|
|
@ -1021,13 +1030,8 @@ proc genSym(p: PProc, n: PNode, r: var TCompRes) =
|
|||
discard
|
||||
elif sfForward in s.flags:
|
||||
p.g.forwarded.add(s)
|
||||
elif not p.g.generatedSyms.containsOrIncl(s.id):
|
||||
let newp = genProc(p, s)
|
||||
var owner = p
|
||||
while owner != nil and owner.prc != s.owner:
|
||||
owner = owner.up
|
||||
if owner != nil: add(owner.locals, newp)
|
||||
else: add(p.g.code, newp)
|
||||
else:
|
||||
genProcForSymIfNeeded(p, s)
|
||||
else:
|
||||
if s.loc.r == nil:
|
||||
internalError(n.info, "symbol has no generated name: " & s.name.s)
|
||||
|
|
@ -1394,6 +1398,9 @@ proc genMagic(p: PProc, n: PNode, r: var TCompRes) =
|
|||
of mCopyStrLast: ternaryExpr(p, n, r, "", "($1.slice($2, ($3)+1).concat(0))")
|
||||
of mNewString: unaryExpr(p, n, r, "mnewString", "mnewString($1)")
|
||||
of mNewStringOfCap: unaryExpr(p, n, r, "mnewString", "mnewString(0)")
|
||||
of mDotDot:
|
||||
genProcForSymIfNeeded(p, n.sons[0].sym)
|
||||
genCall(p, n, r)
|
||||
else:
|
||||
genCall(p, n, r)
|
||||
#else internalError(e.info, 'genMagic: ' + magicToStr[op]);
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@
|
|||
|
||||
import
|
||||
hashes, options, msgs, strutils, platform, idents, nimlexbase, llstream,
|
||||
wordrecg
|
||||
wordrecg, etcpriv
|
||||
|
||||
const
|
||||
MaxLineLength* = 80 # lines longer than this lead to a warning
|
||||
|
|
@ -140,10 +140,12 @@ proc isKeyword*(kind: TTokType): bool =
|
|||
proc isNimIdentifier*(s: string): bool =
|
||||
if s[0] in SymStartChars:
|
||||
var i = 1
|
||||
while i < s.len:
|
||||
var sLen = s.len
|
||||
while i < sLen:
|
||||
if s[i] == '_':
|
||||
inc(i)
|
||||
if s[i] notin SymChars: return
|
||||
elif isMagicIdentSeparatorRune(cstring s, i):
|
||||
inc(i, magicIdentSeparatorRuneByteWidth)
|
||||
if s[i] notin SymChars: return
|
||||
inc(i)
|
||||
result = true
|
||||
|
|
@ -229,23 +231,6 @@ proc lexMessagePos(L: var TLexer, msg: TMsgKind, pos: int, arg = "") =
|
|||
var info = newLineInfo(L.fileIdx, L.lineNumber, pos - L.lineStart)
|
||||
L.dispMessage(info, msg, arg)
|
||||
|
||||
proc matchUnderscoreChars(L: var TLexer, tok: var TToken, chars: set[char]) =
|
||||
var pos = L.bufpos # use registers for pos, buf
|
||||
var buf = L.buf
|
||||
while true:
|
||||
if buf[pos] in chars:
|
||||
add(tok.literal, buf[pos])
|
||||
inc(pos)
|
||||
else:
|
||||
break
|
||||
if buf[pos] == '_':
|
||||
if buf[pos+1] notin chars:
|
||||
lexMessage(L, errInvalidToken, "_")
|
||||
break
|
||||
add(tok.literal, '_')
|
||||
inc(pos)
|
||||
L.bufpos = pos
|
||||
|
||||
proc matchTwoChars(L: TLexer, first: char, second: set[char]): bool =
|
||||
result = (L.buf[L.bufpos] == first) and (L.buf[L.bufpos + 1] in second)
|
||||
|
||||
|
|
@ -268,136 +253,195 @@ proc unsafeParseUInt(s: string, b: var BiggestInt, start = 0): int =
|
|||
result = i - start
|
||||
{.pop.} # overflowChecks
|
||||
|
||||
|
||||
template eatChar(L: var TLexer, t: var TToken, replacementChar: char) =
|
||||
add(t.literal, replacementChar)
|
||||
inc(L.bufpos)
|
||||
|
||||
template eatChar(L: var TLexer, t: var TToken) =
|
||||
add(t.literal, L.buf[L.bufpos])
|
||||
inc(L.bufpos)
|
||||
|
||||
proc getNumber(L: var TLexer): TToken =
|
||||
var
|
||||
pos, endpos: int
|
||||
startpos, endpos: int
|
||||
xi: BiggestInt
|
||||
# get the base:
|
||||
const literalishChars = { 'A'..'F', 'a'..'f', '0'..'9', 'X', 'x', 'o', 'c',
|
||||
'C', 'b', 'B', '_', '.', '\''}
|
||||
const literalishCharsNoDot = literalishChars - {'.'}
|
||||
|
||||
proc matchUnderscoreChars(L: var TLexer, tok: var TToken, chars: set[char]) =
|
||||
var pos = L.bufpos # use registers for pos, buf
|
||||
var buf = L.buf
|
||||
while true:
|
||||
if buf[pos] in chars:
|
||||
add(tok.literal, buf[pos])
|
||||
inc(pos)
|
||||
else:
|
||||
break
|
||||
if buf[pos] == '_':
|
||||
if buf[pos+1] notin chars:
|
||||
lexMessage(L, errInvalidToken, "_")
|
||||
break
|
||||
add(tok.literal, '_')
|
||||
inc(pos)
|
||||
L.bufpos = pos
|
||||
|
||||
proc matchChars(L: var TLexer, tok: var TToken, chars: set[char]) =
|
||||
var pos = L.bufpos # use registers for pos, buf
|
||||
var buf = L.buf
|
||||
while buf[pos] in chars:
|
||||
add(tok.literal, buf[pos])
|
||||
inc(pos)
|
||||
L.bufpos = pos
|
||||
|
||||
proc lexMessageLitNum(L: var TLexer, msg: TMsgKind, startpos: int) =
|
||||
# Used to get slightly human friendlier err messages.
|
||||
# Note: the erroneous 'O' char in the character set is intentional
|
||||
const literalishChars = {'A'..'F', 'a'..'f', '0'..'9', 'X', 'x', 'o', 'O',
|
||||
'c', 'C', 'b', 'B', '_', '.', '\'', 'd', 'i', 'u'}
|
||||
var msgPos = L.bufpos
|
||||
var t: TToken
|
||||
t.literal = ""
|
||||
L.bufpos = startpos # Use L.bufpos as pos because of matchChars
|
||||
matchChars(L, t, literalishChars)
|
||||
# We must verify +/- specifically so that we're not past the literal
|
||||
if L.buf[L.bufpos] in {'+', '-'} and
|
||||
L.buf[L.bufpos - 1] in {'e', 'E'}:
|
||||
add(t.literal, L.buf[L.bufpos])
|
||||
inc(L.bufpos)
|
||||
matchChars(L, t, literalishChars)
|
||||
if L.buf[L.bufpos] in {'\'', 'f', 'F', 'd', 'D', 'i', 'I', 'u', 'U'}:
|
||||
inc(L.bufpos)
|
||||
add(t.literal, L.buf[L.bufpos])
|
||||
matchChars(L, t, {'0'..'9'})
|
||||
L.bufpos = msgPos
|
||||
lexMessage(L, msg, t.literal)
|
||||
|
||||
result.tokType = tkIntLit # int literal until we know better
|
||||
result.literal = ""
|
||||
result.base = base10 # BUGFIX
|
||||
pos = L.bufpos # make sure the literal is correct for error messages:
|
||||
var eallowed = false
|
||||
if L.buf[pos] == '0' and L.buf[pos+1] in {'X', 'x'}:
|
||||
matchUnderscoreChars(L, result, {'A'..'F', 'a'..'f', '0'..'9', 'X', 'x'})
|
||||
result.base = base10
|
||||
startpos = L.bufpos
|
||||
var isAFloatLiteral = false
|
||||
# First stage: find out base, make verifications, build token literal string
|
||||
if L.buf[L.bufpos] == '0' and
|
||||
L.buf[L.bufpos + 1] in {'X', 'x', 'o', 'O', 'c', 'C', 'b', 'B'}:
|
||||
eatChar(L, result, '0')
|
||||
case L.buf[L.bufpos]
|
||||
of 'O':
|
||||
lexMessageLitNum(L, errInvalidNumberOctalCode, startpos)
|
||||
of 'x', 'X':
|
||||
eatChar(L, result, 'x')
|
||||
matchUnderscoreChars(L, result, {'0'..'9', 'a'..'f', 'A'..'F'})
|
||||
of 'o', 'c', 'C':
|
||||
eatChar(L, result, 'c')
|
||||
matchUnderscoreChars(L, result, {'0'..'7'})
|
||||
of 'b', 'B':
|
||||
eatChar(L, result, 'b')
|
||||
matchUnderscoreChars(L, result, {'0'..'1'})
|
||||
else:
|
||||
internalError(getLineInfo(L), "getNumber")
|
||||
else:
|
||||
matchUnderscoreChars(L, result, {'0'..'9', 'b', 'B', 'o', 'c', 'C'})
|
||||
eallowed = true
|
||||
if (L.buf[L.bufpos] == '.') and (L.buf[L.bufpos + 1] in {'0'..'9'}):
|
||||
add(result.literal, '.')
|
||||
inc(L.bufpos)
|
||||
matchUnderscoreChars(L, result, {'0'..'9'})
|
||||
eallowed = true
|
||||
if eallowed and L.buf[L.bufpos] in {'e', 'E'}:
|
||||
add(result.literal, 'e')
|
||||
inc(L.bufpos)
|
||||
if L.buf[L.bufpos] in {'+', '-'}:
|
||||
add(result.literal, L.buf[L.bufpos])
|
||||
inc(L.bufpos)
|
||||
matchUnderscoreChars(L, result, {'0'..'9'})
|
||||
if (L.buf[L.bufpos] == '.') and (L.buf[L.bufpos + 1] in {'0'..'9'}):
|
||||
isAFloatLiteral = true
|
||||
eatChar(L, result, '.')
|
||||
matchUnderscoreChars(L, result, {'0'..'9'})
|
||||
if L.buf[L.bufpos] in {'e', 'E'}:
|
||||
isAFloatLiteral = true
|
||||
eatChar(L, result, 'e')
|
||||
if L.buf[L.bufpos] in {'+', '-'}:
|
||||
eatChar(L, result)
|
||||
matchUnderscoreChars(L, result, {'0'..'9'})
|
||||
endpos = L.bufpos
|
||||
if L.buf[endpos] in {'\'', 'f', 'F', 'i', 'I', 'u', 'U'}:
|
||||
if L.buf[endpos] == '\'': inc(endpos)
|
||||
L.bufpos = pos # restore position
|
||||
case L.buf[endpos]
|
||||
# Second stage, find out if there's a datatype postfix and handle it
|
||||
var postPos = endpos
|
||||
if L.buf[postPos] in {'\'', 'f', 'F', 'd', 'D', 'i', 'I', 'u', 'U'}:
|
||||
if L.buf[postPos] == '\'':
|
||||
inc(postPos)
|
||||
case L.buf[postPos]
|
||||
of 'f', 'F':
|
||||
inc(endpos)
|
||||
if (L.buf[endpos] == '3') and (L.buf[endpos + 1] == '2'):
|
||||
inc(postPos)
|
||||
if (L.buf[postPos] == '3') and (L.buf[postPos + 1] == '2'):
|
||||
result.tokType = tkFloat32Lit
|
||||
inc(endpos, 2)
|
||||
elif (L.buf[endpos] == '6') and (L.buf[endpos + 1] == '4'):
|
||||
inc(postPos, 2)
|
||||
elif (L.buf[postPos] == '6') and (L.buf[postPos + 1] == '4'):
|
||||
result.tokType = tkFloat64Lit
|
||||
inc(endpos, 2)
|
||||
elif (L.buf[endpos] == '1') and
|
||||
(L.buf[endpos + 1] == '2') and
|
||||
(L.buf[endpos + 2] == '8'):
|
||||
inc(postPos, 2)
|
||||
elif (L.buf[postPos] == '1') and
|
||||
(L.buf[postPos + 1] == '2') and
|
||||
(L.buf[postPos + 2] == '8'):
|
||||
result.tokType = tkFloat128Lit
|
||||
inc(endpos, 3)
|
||||
else:
|
||||
lexMessage(L, errInvalidNumber, result.literal & "'f" & L.buf[endpos])
|
||||
inc(postPos, 3)
|
||||
else: # "f" alone defaults to float32
|
||||
result.tokType = tkFloat32Lit
|
||||
of 'd', 'D': # ad hoc convenience shortcut for f64
|
||||
inc(postPos)
|
||||
result.tokType = tkFloat64Lit
|
||||
of 'i', 'I':
|
||||
inc(endpos)
|
||||
if (L.buf[endpos] == '6') and (L.buf[endpos + 1] == '4'):
|
||||
inc(postPos)
|
||||
if (L.buf[postPos] == '6') and (L.buf[postPos + 1] == '4'):
|
||||
result.tokType = tkInt64Lit
|
||||
inc(endpos, 2)
|
||||
elif (L.buf[endpos] == '3') and (L.buf[endpos + 1] == '2'):
|
||||
inc(postPos, 2)
|
||||
elif (L.buf[postPos] == '3') and (L.buf[postPos + 1] == '2'):
|
||||
result.tokType = tkInt32Lit
|
||||
inc(endpos, 2)
|
||||
elif (L.buf[endpos] == '1') and (L.buf[endpos + 1] == '6'):
|
||||
inc(postPos, 2)
|
||||
elif (L.buf[postPos] == '1') and (L.buf[postPos + 1] == '6'):
|
||||
result.tokType = tkInt16Lit
|
||||
inc(endpos, 2)
|
||||
elif (L.buf[endpos] == '8'):
|
||||
inc(postPos, 2)
|
||||
elif (L.buf[postPos] == '8'):
|
||||
result.tokType = tkInt8Lit
|
||||
inc(endpos)
|
||||
inc(postPos)
|
||||
else:
|
||||
lexMessage(L, errInvalidNumber, result.literal & "'i" & L.buf[endpos])
|
||||
lexMessageLitNum(L, errInvalidNumber, startpos)
|
||||
of 'u', 'U':
|
||||
inc(endpos)
|
||||
if (L.buf[endpos] == '6') and (L.buf[endpos + 1] == '4'):
|
||||
inc(postPos)
|
||||
if (L.buf[postPos] == '6') and (L.buf[postPos + 1] == '4'):
|
||||
result.tokType = tkUInt64Lit
|
||||
inc(endpos, 2)
|
||||
elif (L.buf[endpos] == '3') and (L.buf[endpos + 1] == '2'):
|
||||
inc(postPos, 2)
|
||||
elif (L.buf[postPos] == '3') and (L.buf[postPos + 1] == '2'):
|
||||
result.tokType = tkUInt32Lit
|
||||
inc(endpos, 2)
|
||||
elif (L.buf[endpos] == '1') and (L.buf[endpos + 1] == '6'):
|
||||
inc(postPos, 2)
|
||||
elif (L.buf[postPos] == '1') and (L.buf[postPos + 1] == '6'):
|
||||
result.tokType = tkUInt16Lit
|
||||
inc(endpos, 2)
|
||||
elif (L.buf[endpos] == '8'):
|
||||
inc(postPos, 2)
|
||||
elif (L.buf[postPos] == '8'):
|
||||
result.tokType = tkUInt8Lit
|
||||
inc(endpos)
|
||||
inc(postPos)
|
||||
else:
|
||||
result.tokType = tkUIntLit
|
||||
else: lexMessage(L, errInvalidNumber, result.literal & "'" & L.buf[endpos])
|
||||
else:
|
||||
L.bufpos = pos # restore position
|
||||
else:
|
||||
lexMessageLitNum(L, errInvalidNumber, startpos)
|
||||
# Is there still a literalish char awaiting? Then it's an error!
|
||||
if L.buf[postPos] in literalishCharsNoDot or
|
||||
(L.buf[postPos] == '.' and L.buf[postPos + 1] in {'0'..'9'}):
|
||||
lexMessageLitNum(L, errInvalidNumber, startpos)
|
||||
# Third stage, extract actual number
|
||||
L.bufpos = startpos # restore position
|
||||
var pos: int = startpos
|
||||
try:
|
||||
if (L.buf[pos] == '0') and
|
||||
(L.buf[pos + 1] in {'x', 'X', 'b', 'B', 'o', 'O', 'c', 'C'}):
|
||||
inc(pos, 2)
|
||||
xi = 0 # it may be a base prefix
|
||||
xi = 0 # it is a base prefix
|
||||
case L.buf[pos - 1] # now look at the optional type suffix:
|
||||
of 'b', 'B':
|
||||
result.base = base2
|
||||
while true:
|
||||
case L.buf[pos]
|
||||
of '2'..'9', '.':
|
||||
lexMessage(L, errInvalidNumber, result.literal)
|
||||
inc(pos)
|
||||
of '_':
|
||||
if L.buf[pos+1] notin {'0'..'1'}:
|
||||
lexMessage(L, errInvalidToken, "_")
|
||||
break
|
||||
inc(pos)
|
||||
of '0', '1':
|
||||
while pos < endpos:
|
||||
if L.buf[pos] != '_':
|
||||
xi = `shl`(xi, 1) or (ord(L.buf[pos]) - ord('0'))
|
||||
inc(pos)
|
||||
else: break
|
||||
inc(pos)
|
||||
of 'o', 'c', 'C':
|
||||
result.base = base8
|
||||
while true:
|
||||
case L.buf[pos]
|
||||
of '8'..'9', '.':
|
||||
lexMessage(L, errInvalidNumber, result.literal)
|
||||
inc(pos)
|
||||
of '_':
|
||||
if L.buf[pos+1] notin {'0'..'7'}:
|
||||
lexMessage(L, errInvalidToken, "_")
|
||||
break
|
||||
inc(pos)
|
||||
of '0'..'7':
|
||||
while pos < endpos:
|
||||
if L.buf[pos] != '_':
|
||||
xi = `shl`(xi, 3) or (ord(L.buf[pos]) - ord('0'))
|
||||
inc(pos)
|
||||
else: break
|
||||
of 'O':
|
||||
lexMessage(L, errInvalidNumber, result.literal)
|
||||
inc(pos)
|
||||
of 'x', 'X':
|
||||
result.base = base16
|
||||
while true:
|
||||
while pos < endpos:
|
||||
case L.buf[pos]
|
||||
of '_':
|
||||
if L.buf[pos+1] notin {'0'..'9', 'a'..'f', 'A'..'F'}:
|
||||
lexMessage(L, errInvalidToken, "_")
|
||||
break
|
||||
inc(pos)
|
||||
of '0'..'9':
|
||||
xi = `shl`(xi, 4) or (ord(L.buf[pos]) - ord('0'))
|
||||
|
|
@ -408,8 +452,10 @@ proc getNumber(L: var TLexer): TToken =
|
|||
of 'A'..'F':
|
||||
xi = `shl`(xi, 4) or (ord(L.buf[pos]) - ord('A') + 10)
|
||||
inc(pos)
|
||||
else: break
|
||||
else: internalError(getLineInfo(L), "getNumber")
|
||||
else:
|
||||
break
|
||||
else:
|
||||
internalError(getLineInfo(L), "getNumber")
|
||||
case result.tokType
|
||||
of tkIntLit, tkInt64Lit: result.iNumber = xi
|
||||
of tkInt8Lit: result.iNumber = BiggestInt(int8(toU8(int(xi))))
|
||||
|
|
@ -425,7 +471,7 @@ proc getNumber(L: var TLexer): TToken =
|
|||
# XXX: Test this on big endian machine!
|
||||
of tkFloat64Lit: result.fNumber = (cast[PFloat64](addr(xi)))[]
|
||||
else: internalError(getLineInfo(L), "getNumber")
|
||||
elif isFloatLiteral(result.literal) or (result.tokType == tkFloat32Lit) or
|
||||
elif isAFloatLiteral or (result.tokType == tkFloat32Lit) or
|
||||
(result.tokType == tkFloat64Lit):
|
||||
result.fNumber = parseFloat(result.literal)
|
||||
if result.tokType == tkIntLit: result.tokType = tkFloatLit
|
||||
|
|
@ -441,18 +487,18 @@ proc getNumber(L: var TLexer): TToken =
|
|||
if result.tokType == tkIntLit:
|
||||
result.tokType = tkInt64Lit
|
||||
elif result.tokType in {tkInt8Lit, tkInt16Lit, tkInt32Lit}:
|
||||
lexMessage(L, errNumberOutOfRange, result.literal)
|
||||
lexMessageLitNum(L, errNumberOutOfRange, startpos)
|
||||
elif result.tokType == tkInt8Lit and
|
||||
(result.iNumber < int8.low or result.iNumber > int8.high):
|
||||
lexMessage(L, errNumberOutOfRange, result.literal)
|
||||
lexMessageLitNum(L, errNumberOutOfRange, startpos)
|
||||
elif result.tokType == tkInt16Lit and
|
||||
(result.iNumber < int16.low or result.iNumber > int16.high):
|
||||
lexMessage(L, errNumberOutOfRange, result.literal)
|
||||
lexMessageLitNum(L, errNumberOutOfRange, startpos)
|
||||
except ValueError:
|
||||
lexMessage(L, errInvalidNumber, result.literal)
|
||||
lexMessageLitNum(L, errInvalidNumber, startpos)
|
||||
except OverflowError, RangeError:
|
||||
lexMessage(L, errNumberOutOfRange, result.literal)
|
||||
L.bufpos = endpos
|
||||
lexMessageLitNum(L, errNumberOutOfRange, startpos)
|
||||
L.bufpos = postPos
|
||||
|
||||
proc handleHexChar(L: var TLexer, xi: var int) =
|
||||
case L.buf[L.bufpos]
|
||||
|
|
@ -625,23 +671,34 @@ proc getCharacter(L: var TLexer, tok: var TToken) =
|
|||
inc(L.bufpos) # skip '
|
||||
|
||||
proc getSymbol(L: var TLexer, tok: var TToken) =
|
||||
var h: THash = 0
|
||||
var h: Hash = 0
|
||||
var pos = L.bufpos
|
||||
var buf = L.buf
|
||||
while true:
|
||||
var c = buf[pos]
|
||||
case c
|
||||
of 'a'..'z', '0'..'9', '\x80'..'\xFF':
|
||||
h = h !& ord(c)
|
||||
if c == '\226' and
|
||||
buf[pos+1] == '\128' and
|
||||
buf[pos+2] == '\147': # It's a 'magic separator' en-dash Unicode
|
||||
if buf[pos + magicIdentSeparatorRuneByteWidth] notin SymChars:
|
||||
lexMessage(L, errInvalidToken, "–")
|
||||
break
|
||||
inc(pos, magicIdentSeparatorRuneByteWidth)
|
||||
else:
|
||||
h = h !& ord(c)
|
||||
inc(pos)
|
||||
of 'A'..'Z':
|
||||
c = chr(ord(c) + (ord('a') - ord('A'))) # toLower()
|
||||
h = h !& ord(c)
|
||||
inc(pos)
|
||||
of '_':
|
||||
if buf[pos+1] notin SymChars:
|
||||
lexMessage(L, errInvalidToken, "_")
|
||||
break
|
||||
inc(pos)
|
||||
|
||||
else: break
|
||||
inc(pos)
|
||||
h = !$h
|
||||
tok.ident = getIdent(addr(L.buf[L.bufpos]), pos - L.bufpos, h)
|
||||
L.bufpos = pos
|
||||
|
|
@ -652,7 +709,7 @@ proc getSymbol(L: var TLexer, tok: var TToken) =
|
|||
tok.tokType = TTokType(tok.ident.id + ord(tkSymbol))
|
||||
|
||||
proc endOperator(L: var TLexer, tok: var TToken, pos: int,
|
||||
hash: THash) {.inline.} =
|
||||
hash: Hash) {.inline.} =
|
||||
var h = !$hash
|
||||
tok.ident = getIdent(addr(L.buf[L.bufpos]), pos - L.bufpos, h)
|
||||
if (tok.ident.id < oprLow) or (tok.ident.id > oprHigh): tok.tokType = tkOpr
|
||||
|
|
@ -662,7 +719,7 @@ proc endOperator(L: var TLexer, tok: var TToken, pos: int,
|
|||
proc getOperator(L: var TLexer, tok: var TToken) =
|
||||
var pos = L.bufpos
|
||||
var buf = L.buf
|
||||
var h: THash = 0
|
||||
var h: Hash = 0
|
||||
while true:
|
||||
var c = buf[pos]
|
||||
if c notin OpChars: break
|
||||
|
|
|
|||
|
|
@ -17,10 +17,9 @@ type
|
|||
errIntLiteralExpected, errInvalidCharacterConstant,
|
||||
errClosingTripleQuoteExpected, errClosingQuoteExpected,
|
||||
errTabulatorsAreNotAllowed, errInvalidToken, errLineTooLong,
|
||||
errInvalidNumber, errNumberOutOfRange, errNnotAllowedInCharacter,
|
||||
errClosingBracketExpected, errMissingFinalQuote, errIdentifierExpected,
|
||||
errNewlineExpected,
|
||||
errInvalidModuleName,
|
||||
errInvalidNumber, errInvalidNumberOctalCode, errNumberOutOfRange,
|
||||
errNnotAllowedInCharacter, errClosingBracketExpected, errMissingFinalQuote,
|
||||
errIdentifierExpected, errNewlineExpected, errInvalidModuleName,
|
||||
errOperatorExpected, errTokenExpected, errStringAfterIncludeExpected,
|
||||
errRecursiveDependencyX, errOnOrOffExpected, errNoneSpeedOrSizeExpected,
|
||||
errInvalidPragma, errUnknownPragma, errInvalidDirectiveX,
|
||||
|
|
@ -143,6 +142,7 @@ const
|
|||
errInvalidToken: "invalid token: $1",
|
||||
errLineTooLong: "line too long",
|
||||
errInvalidNumber: "$1 is not a valid number",
|
||||
errInvalidNumberOctalCode: "$1 is not a valid number; did you mean octal? Then use one of '0o', '0c' or '0C'.",
|
||||
errNumberOutOfRange: "number $1 out of valid range",
|
||||
errNnotAllowedInCharacter: "\\n not allowed in character literal",
|
||||
errClosingBracketExpected: "closing ']' expected, but end of file reached",
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@
|
|||
import
|
||||
hashes, ast, astalgo, types
|
||||
|
||||
proc hashTree(n: PNode): THash =
|
||||
proc hashTree(n: PNode): Hash =
|
||||
if n == nil: return
|
||||
result = ord(n.kind)
|
||||
case n.kind
|
||||
|
|
@ -53,8 +53,8 @@ proc treesEquivalent(a, b: PNode): bool =
|
|||
result = true
|
||||
if result: result = sameTypeOrNil(a.typ, b.typ)
|
||||
|
||||
proc nodeTableRawGet(t: TNodeTable, k: THash, key: PNode): int =
|
||||
var h: THash = k and high(t.data)
|
||||
proc nodeTableRawGet(t: TNodeTable, k: Hash, key: PNode): int =
|
||||
var h: Hash = k and high(t.data)
|
||||
while t.data[h].key != nil:
|
||||
if (t.data[h].h == k) and treesEquivalent(t.data[h].key, key):
|
||||
return h
|
||||
|
|
@ -66,9 +66,9 @@ proc nodeTableGet*(t: TNodeTable, key: PNode): int =
|
|||
if index >= 0: result = t.data[index].val
|
||||
else: result = low(int)
|
||||
|
||||
proc nodeTableRawInsert(data: var TNodePairSeq, k: THash, key: PNode,
|
||||
proc nodeTableRawInsert(data: var TNodePairSeq, k: Hash, key: PNode,
|
||||
val: int) =
|
||||
var h: THash = k and high(data)
|
||||
var h: Hash = k and high(data)
|
||||
while data[h].key != nil: h = nextTry(h, high(data))
|
||||
assert(data[h].key == nil)
|
||||
data[h].h = k
|
||||
|
|
@ -77,7 +77,7 @@ proc nodeTableRawInsert(data: var TNodePairSeq, k: THash, key: PNode,
|
|||
|
||||
proc nodeTablePut*(t: var TNodeTable, key: PNode, val: int) =
|
||||
var n: TNodePairSeq
|
||||
var k: THash = hashTree(key)
|
||||
var k: Hash = hashTree(key)
|
||||
var index = nodeTableRawGet(t, k, key)
|
||||
if index >= 0:
|
||||
assert(t.data[index].key != nil)
|
||||
|
|
@ -94,7 +94,7 @@ proc nodeTablePut*(t: var TNodeTable, key: PNode, val: int) =
|
|||
|
||||
proc nodeTableTestOrSet*(t: var TNodeTable, key: PNode, val: int): int =
|
||||
var n: TNodePairSeq
|
||||
var k: THash = hashTree(key)
|
||||
var k: Hash = hashTree(key)
|
||||
var index = nodeTableRawGet(t, k, key)
|
||||
if index >= 0:
|
||||
assert(t.data[index].key != nil)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue