fixes the bug that keeps the template engine package from working

compiler/vmgen.nim

@@ -969,7 +969,7 @@ const
 
 proc fitsRegister*(t: PType): bool =
   t.skipTypes(abstractInst-{tyTypeDesc}).kind in {
-    tyRange, tyEnum, tyBool, tyInt..tyUInt64}
+    tyRange, tyEnum, tyBool, tyInt..tyUInt64, tyChar}
 
 proc requiresCopy(n: PNode): bool =
   if n.typ.skipTypes(abstractInst-{tyTypeDesc}).kind in atomicTypes:
@@ -1191,6 +1191,8 @@ proc genArrAccess2(c: PCtx; n: PNode; dest: var TDest; opc: TOpcode;
     c.gABC(n, opcNodeToReg, dest, cc)
     c.freeTemp(cc)
   else:
+    #message(n.info, warnUser, "argh")
+    #echo "FLAGS ", flags, " ", fitsRegister(n.typ), " ", typeToString(n.typ)
     c.gABC(n, opc, dest, a, b)
   c.freeTemp(a)
   c.freeTemp(b)
@@ -1623,7 +1625,7 @@ proc genProc(c: PCtx; s: PSym): int =
     c.gABC(body, opcEof, eofInstr.regA)
     c.optimizeJumps(result)
     s.offset = c.prc.maxSlots
-    #if s.name.s == "parse_until_symbol":
+    #if s.name.s == "find":
     #  echo renderTree(body)
     #  c.echoCode(result)
     c.prc = oldPrc

tests/template/annotate.nim

@@ -0,0 +1,113 @@
+import macros, parseutils
+
+# Generate tags
+macro make(names: openarray[expr]): stmt {.immediate.} =
+    result = newStmtList()
+
+    for i in 0 .. names.len-1:
+        result.add newProc(
+            name   = ident($names[i]).postfix("*"),
+            params = [
+                ident("string"),
+                newIdentDefs(
+                    ident("content"),
+                    ident("string")
+                )
+            ],
+            body = newStmtList(
+                parseStmt("reindent(content)")
+            )
+        )
+
+
+iterator lines(value: string): string =
+    var i = 0
+    while i < value.len:
+        var line: string
+        inc(i, value.parseUntil(line, 0x0A.char, i) + 1)
+        yield line
+
+
+proc reindent*(value: string, preset_indent = 0): string =
+    var indent = -1
+
+    # Detect indentation!
+    for ln in lines(value):
+        var read = ln.skipWhitespace()
+
+        # If the line is empty, ignore it for indentation
+        if read == ln.len: continue
+
+        indent = if indent < 0: read
+                 else: min(indent, read)
+
+    # Create a precursor indent as-needed
+    var precursor = newString(0)
+    for i in 1 .. preset_indent:
+        precursor.add(' ')
+
+    # Re-indent
+    result = newString(0)
+
+    for ln in lines(value):
+        var value = ln.substr(indent)
+
+        result.add(precursor)
+
+        if value.len > 0:
+            result.add(value)
+            result.add(0x0A.char)
+
+    return result
+
+
+#Define tags
+make([ html, xml, glsl, js, css, rst ])
+
+
+when isMainModule:
+    ## Test tags
+
+    const script = js"""
+        var x = 5;
+        console.log(x.toString());
+    """
+
+    const styles = css"""
+        .someRule {
+            width: 500px;
+        }
+    """
+
+    const body = html"""
+        <ul>
+            <li>1</li>
+            <li>2</li>
+            <li>
+                <a hef="#google">google</a>
+            </li>
+        </ul>
+    """
+
+    const info = xml"""
+        <item>
+            <i>1</i>
+            <i>2</i>
+        </item>
+    """
+
+    const shader = glsl"""
+        void main()
+        {
+            gl_Position = gl_ProjectionMatrix
+                        * gl_ModelViewMatrix
+                        * gl_Vertex;
+        }
+    """
+
+
+    echo script
+    echo styles
+    echo body
+    echo info
+    echo shader

tests/template/otests.nim

@@ -0,0 +1,219 @@
+# Fields
+const x = 5
+
+
+# Test substring
+static:
+    assert "test".substring(3)   == "t"
+    assert "test".substring(2,1) == "s"
+    assert "test".substring(3,2) == "t"
+    assert "test".substring(1,2) == "es"
+
+
+# Various parsing tests
+when true:
+
+    block: #no_substitution
+        proc actual: string = tmpli html"""
+            <p>Test!</p>
+        """
+        const expected = html"""
+            <p>Test!</p>
+        """
+        doAssert actual() == expected
+
+    block: #basic
+        proc actual: string = tmpli html"""
+            <p>Test $$x</p>
+            $x
+        """
+        const expected = html"""
+            <p>Test $x</p>
+            5
+        """
+        doAssert actual() == expected
+
+    block: #expression
+        proc actual: string = tmpli html"""
+            <p>Test $$(x * 5)</p>
+            $(x * 5)
+        """
+        const expected = html"""
+            <p>Test $(x * 5)</p>
+            25
+        """
+        doAssert actual() == expected
+
+    block: #escape
+        proc actual: string = tmpli js"""
+            [{
+                "hello world"
+            }]
+        """
+        const expected = js"""
+            [{
+                "hello world"
+            }]
+        """
+        doAssert actual() == expected
+
+    block: #forIn
+        proc actual: string = tmpli html"""
+            <p>Test for</p>
+            <ul>
+                $for y in 0..2 {
+                    <li>$y</li>
+                }
+            </ul>
+        """
+        const expected = html"""
+            <p>Test for</p>
+            <ul>
+                <li>0</li>
+                <li>1</li>
+                <li>2</li>
+            </ul>
+        """
+        doAssert actual() == expected
+
+    block: #while
+        proc actual: string = tmpli html"""
+            <p>Test while/stmt</p>
+            <ul>
+                ${ var y = 0 }
+                $while y < 4 {
+                    <li>$y</li>
+                    ${ inc(y) }
+                }
+            </ul>
+        """
+        const expected = html"""
+            <p>Test while/stmt</p>
+            <ul>
+                <li>0</li>
+                <li>1</li>
+                <li>2</li>
+                <li>3</li>
+            </ul>
+        """
+        doAssert actual() == expected
+
+    block: #ifElifElse
+        proc actual: string = tmpli html"""
+            <p>Test if/elif/else</p>
+            $if x == 8 {
+                <div>x is 8!</div>
+            }
+            $elif x == 7 {
+                <div>x is 7!</div>
+            }
+            $else {
+                <div>x is neither!</div>
+            }
+        """
+        const expected = html"""
+            <p>Test if/elif/else</p>
+            <div>x is neither!</div>
+        """
+        doAssert actual() == expected
+
+    block: #multiLineStatements
+        proc actual: string = tmpli html"""
+            <p>Test multiline statements</p>
+            ${
+                var x = 5
+                var y = 7
+            }
+            <span>$x</span><span>$y</span>
+        """
+        const expected = html"""
+            <p>Test multiline statements</p>
+            <span>5</span><span>7</span>
+        """
+        doAssert actual() == expected
+
+    block: #caseOfElse
+        proc actual: string = tmpli html"""
+            <p>Test case</p>
+            $case x
+            $of 5 {
+                <div>x == 5</div>
+            }
+            $of 6 {
+                <div>x == 6</div>
+            }
+            $else {}
+        """
+        const expected = html"""
+            <p>Test case</p>
+            <div>x == 5</div>
+        """
+        doAssert actual() == expected
+
+
+
+when true: #embeddingTest
+    proc no_substitution: string = tmpli html"""
+        <h1>Template test!</h1>
+    """
+
+    # # Single variable substitution
+    proc substitution(who = "nobody"): string = tmpli html"""
+        <div id="greeting">hello $who!</div>
+    """
+
+    # Expression template
+    proc test_expression(nums: openarray[int] = []): string =
+        var i = 2
+        tmpli html"""
+            $(no_substitution())
+            $(substitution("Billy"))
+            <div id="age">Age: $($nums[i] & "!!")</div>
+        """
+
+    proc test_statements(nums: openarray[int] = []): string =
+        tmpli html"""
+            $(test_expression(nums))
+            $if true {
+                <ul>
+                    $for i in nums {
+                        <li>$(i * 2)</li>
+                    }
+                </ul>
+            }
+        """
+
+    var actual = test_statements([0,1,2])
+    const expected = html"""
+        <h1>Template test!</h1>
+        <div id="greeting">hello Billy!</div>
+        <div id="age">Age: 2!!</div>
+        <ul>
+            <li>0</li>
+            <li>2</li>
+            <li>4</li>
+        </ul>
+    """
+    doAssert actual == expected
+
+
+when defined(future):
+    block: #tryCatch
+        proc actual: string = tmpli html"""
+            <p>Test try/catch</p>
+            <div>
+                $try {
+                    <div>Lets try this!</div>
+                }
+                $except {
+                    <div>Uh oh!</div>
+                }
+            </div>
+        """
+        const expected = html"""
+            <p>Test try/catch</p>
+            <div>
+                    <div>Lets try this!</div>
+            </div>
+        """
+        doAssert actual() == expected

tests/template/t_otemplates.nim

@@ -0,0 +1,345 @@
+discard """
+  output: "Success"
+"""
+
+# Ref:
+# http://nimrod-lang.org/macros.html
+# http://nimrod-lang.org/parseutils.html
+
+
+# Imports
+import tables, parseutils, macros, strutils
+import annotate
+export annotate
+
+
+# Fields
+const identChars = {'a'..'z', 'A'..'Z', '0'..'9', '_'}
+
+
+# Procedure Declarations
+proc parse_template(node: PNimrodNode, value: string) {.compiletime.}
+
+
+# Procedure Definitions
+proc substring(value: string, index: int, length = -1): string {.compiletime.} =
+    ## Returns a string at most `length` characters long, starting at `index`.
+    return if length < 0:    value.substr(index)
+           elif length == 0: ""
+           else:             value.substr(index, index + length-1)
+
+
+proc parse_thru_eol(value: string, index: int): int {.compiletime.} =
+    ## Reads until and past the end of the current line, unless
+    ## a non-whitespace character is encountered first
+    var remainder: string
+    var read = value.parseUntil(remainder, {0x0A.char}, index)
+    if remainder.skipWhitespace() == read:
+        return read + 1
+
+
+proc trim_after_eol(value: var string) {.compiletime.} =
+    ## Trims any whitespace at end after \n
+    var toTrim = 0
+    for i in countdown(value.len-1, 0):
+        # If \n, return
+        if value[i] in [' ', '\t']: inc(toTrim)
+        else: break
+
+    if toTrim > 0:
+        value = value.substring(0, value.len - toTrim)
+
+
+proc trim_eol(value: var string) {.compiletime.} =
+    ## Removes everything after the last line if it contains nothing but whitespace
+    for i in countdown(value.len - 1, 0):
+        # If \n, trim and return
+        if value[i] == 0x0A.char:
+            value = value.substr(0, i)
+            break
+
+        # This is the first character
+        if i == 0:
+            value = ""
+            break
+
+        # Skip change
+        if not (value[i] in [' ', '\t']): break
+
+
+proc detect_indent(value: string, index: int): int {.compiletime.} =
+    ## Detects how indented the line at `index` is.
+    # Seek to the beginning of the line.
+    var lastChar = index
+    for i in countdown(index, 0):
+        if value[i] == 0x0A.char:
+            # if \n, return the indentation level
+            return lastChar - i
+        elif not (value[i] in [' ', '\t']):
+            # if non-whitespace char, decrement lastChar
+            dec(lastChar)
+
+
+proc parse_thru_string(value: string, i: var int, strType = '"') {.compiletime.} =
+    ## Parses until ending " or ' is reached.
+    inc(i)
+    if i < value.len-1:
+        inc(i, value.skipUntil({'\\', strType}, i))
+
+
+proc parse_to_close(value: string, index: int, open='(', close=')', opened=0): int {.compiletime.} =
+    ## Reads until all opened braces are closed
+    ## ignoring any strings "" or ''
+    var remainder   = value.substring(index)
+    var open_braces = opened
+    result = 0
+
+    while result < remainder.len:
+        var c = remainder[result]
+
+        if   c == open:  inc(open_braces)
+        elif c == close: dec(open_braces)
+        elif c == '"':   remainder.parse_thru_string(result)
+        elif c == '\'':  remainder.parse_thru_string(result, '\'')
+
+        if open_braces == 0: break
+        else: inc(result)
+
+
+iterator parse_stmt_list(value: string, index: var int): string =
+    ## Parses unguided ${..} block
+    var read        = value.parse_to_close(index, open='{', close='}')
+    var expressions = value.substring(index + 1, read - 1).split({ ';', 0x0A.char })
+
+    for expression in expressions:
+        let value = expression.strip
+        if value.len > 0:
+            yield value
+
+    #Increment index & parse thru EOL
+    inc(index, read + 1)
+    inc(index, value.parse_thru_eol(index))
+
+
+iterator parse_compound_statements(value, identifier: string, index: int): string =
+    ## Parses through several statements, i.e. if {} elif {} else {}
+    ## and returns the initialization of each as an empty statement
+    ## i.e. if x == 5 { ... } becomes if x == 5: nil.
+
+    template get_next_ident(expected): stmt =
+        var nextIdent: string
+        discard value.parseWhile(nextIdent, {'$'} + identChars, i)
+
+        var next: string
+        var read: int
+
+        if nextIdent == "case":
+            # We have to handle case a bit differently
+            read = value.parseUntil(next, '$', i)
+            inc(i, read)
+            yield next.strip(leading=false) & "\n"
+
+        else:
+            read = value.parseUntil(next, '{', i)
+
+            if nextIdent in expected:
+                inc(i, read)
+                # Parse until closing }, then skip whitespace afterwards
+                read = value.parse_to_close(i, open='{', close='}')
+                inc(i, read + 1)
+                inc(i, value.skipWhitespace(i))
+                yield next & ": nil\n"
+
+            else: break
+
+
+    var i = index
+    while true:
+        # Check if next statement would be valid, given the identifier
+        if identifier in ["if", "when"]:
+            get_next_ident([identifier, "$elif", "$else"])
+
+        elif identifier == "case":
+            get_next_ident(["case", "$of", "$elif", "$else"])
+
+        elif identifier == "try":
+            get_next_ident(["try", "$except", "$finally"])
+
+
+proc parse_complex_stmt(value, identifier: string, index: var int): PNimrodNode {.compiletime.} =
+    ## Parses if/when/try /elif /else /except /finally statements
+
+    # Build up complex statement string
+    var stmtString = newString(0)
+    var numStatements = 0
+    for statement in value.parse_compound_statements(identifier, index):
+        if statement[0] == '$': stmtString.add(statement.substr(1))
+        else:                   stmtString.add(statement)
+        inc(numStatements)
+
+    # Parse stmt string
+    result = parseExpr(stmtString)
+
+    var resultIndex = 0
+
+    # Fast forward a bit if this is a case statement
+    if identifier == "case":
+        inc(resultIndex)
+
+    while resultIndex < numStatements:
+
+        # Detect indentation
+        let indent = detect_indent(value, index)
+
+        # Parse until an open brace `{`
+        var read = value.skipUntil('{', index)
+        inc(index, read + 1)
+
+        # Parse through EOL
+        inc(index, value.parse_thru_eol(index))
+
+        # Parse through { .. }
+        read = value.parse_to_close(index, open='{', close='}', opened=1)
+
+        # Add parsed sub-expression into body
+        var body       = newStmtList()
+        var stmtString = value.substring(index, read)
+        trim_after_eol(stmtString)
+        stmtString = reindent(stmtString, indent)
+        parse_template(body, stmtString)
+        inc(index, read + 1)
+
+        # Insert body into result
+        var stmtIndex = macros.high(result[resultIndex])
+        result[resultIndex][stmtIndex] = body
+
+        # Parse through EOL again & increment result index
+        inc(index, value.parse_thru_eol(index))
+        inc(resultIndex)
+
+
+proc parse_simple_statement(value: string, index: var int): PNimrodNode {.compiletime.} =
+    ## Parses for/while
+
+    # Detect indentation
+    let indent = detect_indent(value, index)
+
+    # Parse until an open brace `{`
+    var splitValue: string
+    var read = value.parseUntil(splitValue, '{', index)
+    result   = parseExpr(splitValue & ":nil")
+    inc(index, read + 1)
+
+    # Parse through EOL
+    inc(index, value.parse_thru_eol(index))
+
+    # Parse through { .. }
+    read = value.parse_to_close(index, open='{', close='}', opened=1)
+
+    # Add parsed sub-expression into body
+    var body       = newStmtList()
+    var stmtString = value.substring(index, read)
+    trim_after_eol(stmtString)
+    stmtString = reindent(stmtString, indent)
+    parse_template(body, stmtString)
+    inc(index, read + 1)
+
+    # Insert body into result
+    var stmtIndex = macros.high(result)
+    result[stmtIndex] = body
+
+    # Parse through EOL again
+    inc(index, value.parse_thru_eol(index))
+
+
+proc parse_until_symbol(node: PNimrodNode, value: string, index: var int): bool {.compiletime.} =
+    ## Parses a string until a $ symbol is encountered, if
+    ## two $$'s are encountered in a row, a split will happen
+    ## removing one of the $'s from the resulting output
+    var splitValue: string
+    var read = value.parseUntil(splitValue, '$', index)
+    var insertionPoint = node.len
+
+    inc(index, read + 1)
+    if index < value.len:
+
+        case value[index]
+        of '$':
+            # Check for duplicate `$`, meaning this is an escaped $
+            node.add newCall("add", ident("result"), newStrLitNode("$"))
+            inc(index)
+
+        of '(':
+            # Check for open `(`, which means parse as simple single-line expression.
+            trim_eol(splitValue)
+            read = value.parse_to_close(index) + 1
+            node.add newCall("add", ident("result"),
+                newCall(bindSym"strip", parseExpr("$" & value.substring(index, read)))
+            )
+            inc(index, read)
+
+        of '{':
+            # Check for open `{`, which means open statement list
+            trim_eol(splitValue)
+            for s in value.parse_stmt_list(index):
+                node.add parseExpr(s)
+
+        else:
+            # Otherwise parse while valid `identChars` and make expression w/ $
+            var identifier: string
+            read = value.parseWhile(identifier, identChars, index)
+
+            if identifier in ["for", "while"]:
+                ## for/while means open simple statement
+                trim_eol(splitValue)
+                node.add value.parse_simple_statement(index)
+
+            elif identifier in ["if", "when", "case", "try"]:
+                ## if/when/case/try means complex statement
+                trim_eol(splitValue)
+                node.add value.parse_complex_stmt(identifier, index)
+
+            elif identifier.len > 0:
+                ## Treat as simple variable
+                node.add newCall("add", ident("result"), newCall("$", ident(identifier)))
+                inc(index, read)
+
+        result = true
+
+    # Insert
+    if splitValue.len > 0:
+        node.insert insertionPoint, newCall("add", ident("result"), newStrLitNode(splitValue))
+
+
+proc parse_template(node: PNimrodNode, value: string) =
+    ## Parses through entire template, outputing valid
+    ## Nimrod code into the input `node` AST.
+    var index = 0
+    while index < value.len and
+          parse_until_symbol(node, value, index): nil
+
+
+macro tmpli*(body: expr): stmt =
+    result = newStmtList()
+
+    result.add parseExpr("result = \"\"")
+
+    var value = if body.kind in nnkStrLit..nnkTripleStrLit: body.strVal
+                else: body[1].strVal
+
+    parse_template(result, reindent(value))
+
+
+macro tmpl*(body: expr): stmt =
+    result = newStmtList()
+
+    var value = if body.kind in nnkStrLit..nnkTripleStrLit: body.strVal
+                else: body[1].strVal
+
+    parse_template(result, reindent(value))
+
+
+# Run tests
+when isMainModule:
+    include otests
+    echo "Success"