code generator supports constant sequences; more consistent compile time evaluation

compiler/ast.nim

@@ -559,9 +559,11 @@ const
     tyBool, tyChar, tyEnum, tyArray, tyObject, 
     tySet, tyTuple, tyRange, tyPtr, tyRef, tyVar, tySequence, tyProc,
     tyPointer, 
-    tyOpenArray, tyString, tyCString, tyInt..tyInt64, tyFloat..tyFloat128} 
+    tyOpenArray, tyString, tyCString, tyInt..tyInt64, tyFloat..tyFloat128,
+    tyUInt..tyUInt64} 
   
-  ConstantDataTypes*: TTypeKinds = {tyArray, tySet, tyTuple}
+  ConstantDataTypes*: TTypeKinds = {tyArrayConstr, tyArray, tySet, 
+                                    tyTuple, tySequence}
   ExportableSymKinds* = {skVar, skConst, skProc, skMethod, skType, skIterator, 
     skMacro, skTemplate, skConverter, skStub}
   PersistentNodeFlags*: TNodeFlags = {nfBase2, nfBase8, nfBase16, nfAllConst}

compiler/ccgexprs.nim

@@ -1831,6 +1831,7 @@ proc expr(p: BProc, e: PNode, d: var TLoc) =
     if sym.loc.r == nil or sym.loc.t == nil:
       InternalError(e.info, "expr: proc not init " & sym.name.s)
     putLocIntoDest(p, d, sym.loc)
+  of nkMetaNode: expr(p, e.sons[0], d)
   else: InternalError(e.info, "expr(" & $e.kind & "); unknown node kind")
 
 proc genNamedConstExpr(p: BProc, n: PNode): PRope =
@@ -1845,6 +1846,24 @@ proc genConstSimpleList(p: BProc, n: PNode): PRope =
   if length > 0: app(result, genNamedConstExpr(p, n.sons[length - 1]))
   appf(result, "}$n")
 
+proc genConstSeq(p: BProc, n: PNode, t: PType): PRope =
+  var data = ropef("{{$1, $1}", n.len.toRope)
+  for i in countup(0, n.len - 1):
+    appf(data, ",$1$n", [genConstExpr(p, n.sons[i])])
+  data.app("}")
+  
+  inc(p.labels)
+  result = con("CNSTSEQ", p.labels.toRope)
+  
+  appcg(p.module, cfsData,
+        "NIM_CONST struct {$n" & 
+        "  #TGenericSeq Sup;$n" &
+        "  $1 data[$2];$n" & 
+        "} $3 = $4;$n", [
+        getTypeDesc(p.module, t.sons[0]), n.len.toRope, result, data])
+
+  result = ropef("(($1)&$2)", [getTypeDesc(p.module, t), result])
+
 proc genConstExpr(p: BProc, n: PNode): PRope =
   case n.Kind
   of nkHiddenStdConv, nkHiddenSubConv:
@@ -1855,6 +1874,10 @@ proc genConstExpr(p: BProc, n: PNode): PRope =
     result = genRawSetData(cs, int(getSize(n.typ)))
   of nkBracket, nkPar:
     # XXX: tySequence!
+    var t = skipTypes(n.typ, abstractInst)
+    if t.kind == tySequence:
+      result = genConstSeq(p, n, t)
+    else:
       result = genConstSimpleList(p, n)
   else:
     var d: TLoc

compiler/ccgstmts.nim

@@ -67,7 +67,8 @@ proc genConstStmt(p: BProc, t: PNode) =
     var c = it.sons[0].sym 
     if sfFakeConst in c.flags:
       genSingleVar(p, it)
-    elif c.typ.kind in ConstantDataTypes and not (lfNoDecl in c.loc.flags): 
+    elif c.typ.kind in ConstantDataTypes and not (lfNoDecl in c.loc.flags) and
+        c.ast.len != 0: 
       # generate the data:
       fillLoc(c.loc, locData, c.typ, mangleName(c), OnUnknown)
       if sfImportc in c.flags: 

compiler/ecmasgen.nim

@@ -1426,6 +1426,7 @@ proc gen(p: var TProc, n: PNode, r: var TCompRes) =
   of nkCStringToString: convCStrToStr(p, n, r)
   of nkStmtListExpr: genStmtListExpr(p, n, r)
   of nkEmpty: nil
+  of nkMetaNode: gen(p, n.sons[0], r)
   else: InternalError(n.info, "gen: unknown node type: " & $n.kind)
   
 var globals: PGlobals

compiler/evals.nim

@@ -43,7 +43,7 @@ const
   evalMaxIterations = 500_000 # max iterations of all loops
   evalMaxRecDepth = 10_000    # max recursion depth for evaluation
 
-# Much better: use a timeout! -> Wether code compiles depends on the machine
+# other idea: use a timeout! -> Wether code compiles depends on the machine
 # the compiler runs on then! Bad idea!
 
 proc newStackFrame*(): PStackFrame = 
@@ -755,16 +755,14 @@ proc evalRepr(c: PEvalContext, n: PNode): PNode =
   result = newStrNodeT(renderTree(result, {renderNoComments}), n)
 
 proc isEmpty(n: PNode): bool =
-  result = (n != nil) and (n.kind == nkEmpty)
+  result = n != nil and n.kind == nkEmpty
 
 # The lexer marks multi-line strings as residing at the line where they 
 # are closed. This function returns the line where the string begins
 # Maybe the lexer should mark both the beginning and the end of expressions,
 # then this function could be removed.
 proc stringStartingLine(s: PNode): int =
-  var totalLines = 0
+  result = s.info.line - countLines(s.strVal)
-  for ln in splitLines(s.strVal): inc totalLines
-  result = s.info.line - totalLines
 
 proc evalParseExpr(c: PEvalContext, n: Pnode): Pnode =
   var code = evalAux(c, n.sons[1], {})
@@ -1069,9 +1067,6 @@ proc evalAux(c: PEvalContext, n: PNode, flags: TEvalFlags): PNode =
   dec(gNestedEvals)
   if gNestedEvals <= 0: stackTrace(c, n, errTooManyIterations)
   case n.kind                 # atoms:
-  of nkMetaNode:
-    result = copyTree(n.sons[0])
-    result.typ = n.typ
   of nkEmpty: result = n
   of nkSym: result = evalSym(c, n, flags)
   of nkType..nkNilLit: result = copyNode(n) # end of atoms
@@ -1131,6 +1126,9 @@ proc evalAux(c: PEvalContext, n: PNode, flags: TEvalFlags): PNode =
      nkTypeSection, nkTemplateDef, nkConstSection, nkIteratorDef,
      nkConverterDef, nkIncludeStmt, nkImportStmt, nkFromStmt: 
     nil
+  of nkMetaNode:
+    result = copyTree(n.sons[0])
+    result.typ = n.typ
   of nkIdentDefs, nkCast, nkYieldStmt, nkAsmStmt, nkForStmt, nkPragmaExpr, 
      nkLambda, nkContinueStmt, nkIdent: 
     stackTrace(c, n, errCannotInterpretNodeX, $n.kind)

compiler/sem.nim

@@ -55,6 +55,16 @@ proc ParamsTypeCheck(c: PContext, typ: PType) {.inline.} =
     GlobalError(typ.n.info, errXisNoType, typeToString(typ))
 
 proc semConstExpr(c: PContext, n: PNode): PNode = 
+  var e = semExprWithType(c, n)
+  if e == nil: 
+    GlobalError(n.info, errConstExprExpected)
+    return nil
+  result = getConstExpr(c.module, e)
+  if result == nil:
+    result = evalConstExpr(c.module, e)
+    if result == nil or result.kind == nkEmpty: 
+      GlobalError(n.info, errConstExprExpected)
+  when false:
     result = semExprWithType(c, n)
     if result == nil: 
       GlobalError(n.info, errConstExprExpected)
@@ -63,16 +73,7 @@ proc semConstExpr(c: PContext, n: PNode): PNode =
     if result == nil: GlobalError(n.info, errConstExprExpected)
   
 proc semAndEvalConstExpr(c: PContext, n: PNode): PNode = 
-  var e = semExprWithType(c, n)
+  result = semConstExpr(c, n)
-  if e == nil: 
-    GlobalError(n.info, errConstExprExpected)
-    return nil
-  result = getConstExpr(c.module, e)
-  if result == nil: 
-    #writeln(output, renderTree(n));
-    result = evalConstExpr(c.module, e)
-    if result == nil or result.kind == nkEmpty: 
-      GlobalError(n.info, errConstExprExpected)
   
 include seminst, semcall
   

compiler/semexprs.nim

@@ -9,10 +9,6 @@
 
 # this module does the semantic checking for expressions
 
-const 
-  ConstAbstractTypes = {tyNil, tyChar, tyInt..tyInt64, tyFloat..tyFloat128, 
-    tyArrayConstr, tyTuple, tySet}
-
 proc semTemplateExpr(c: PContext, n: PNode, s: PSym, semCheck = true): PNode = 
   markUsed(n, s)
   pushInfoContext(n.info)
@@ -49,6 +45,11 @@ proc semExprNoDeref(c: PContext, n: PNode, flags: TExprFlags = {}): PNode =
 proc semSymGenericInstantiation(c: PContext, n: PNode, s: PSym): PNode =
   result = symChoice(c, n, s)
   
+proc inlineConst(n: PNode, s: PSym): PNode {.inline.} =
+  result = copyTree(s.ast)
+  result.typ = s.typ
+  result.info = n.info
+
 proc semSym(c: PContext, n: PNode, s: PSym, flags: TExprFlags): PNode = 
   case s.kind
   of skProc, skMethod, skIterator, skConverter: 
@@ -57,7 +58,12 @@ proc semSym(c: PContext, n: PNode, s: PSym, flags: TExprFlags): PNode =
       LocalError(n.info, errXCannotBePassedToProcVar, s.name.s)
     result = symChoice(c, n, s)
   of skConst:
-    #
+    markUsed(n, s)
+    case skipTypes(s.typ, abstractInst).kind
+    of  tyNil, tyChar, tyInt..tyInt64, tyFloat..tyFloat128, 
+        tyTuple, tySet, tyUInt..tyUInt64:
+      result = inlineConst(n, s)
+    of tyArrayConstr, tySequence:
       # Consider::
       #     const x = []
       #     proc p(a: openarray[int])
@@ -68,12 +74,8 @@ proc semSym(c: PContext, n: PNode, s: PSym, flags: TExprFlags): PNode =
       # It is clear that ``[]`` means two totally different things. Thus, we
       # copy `x`'s AST into each context, so that the type fixup phase can
       # deal with two different ``[]``.
-    #
+      if s.ast.len == 0: result = inlineConst(n, s)
-    markUsed(n, s)
+      else: result = newSymNode(s, n.info)
-    if s.typ.kind in ConstAbstractTypes:
-      result = copyTree(s.ast)
-      result.typ = s.typ
-      result.info = n.info
     else:
       result = newSymNode(s, n.info)
   of skMacro: result = semMacroExpr(c, n, s)

compiler/semfold.nim

@@ -182,9 +182,7 @@ proc evalOp(m: TMagic, n, a, b, c: PNode): PNode =
   of mConStrStr: result = newStrNodeT(getStrOrChar(a) & getStrOrChar(b), n)
   of mInSet: result = newIntNodeT(Ord(inSet(a, b)), n)
   of mRepr:
-    # BUGFIX: we cannot eval mRepr here. But this means that it is not 
+    # BUGFIX: we cannot eval mRepr here for reasons that I forgot.
-    # available for interpretation. I don't know how to fix this.
-    #result := newStrNodeT(renderTree(a, {@set}[renderNoComments]), n);      
   of mIntToStr, mInt64ToStr: result = newStrNodeT($(getOrdValue(a)), n)
   of mBoolToStr: 
     if getOrdValue(a) == 0: result = newStrNodeT("false", n)
@@ -324,7 +322,7 @@ proc getConstExpr(m: PSym, n: PNode): PNode =
     var s = n.sym
     if s.kind == skEnumField: 
       result = newIntNodeT(s.position, n)
-    elif (s.kind == skConst): 
+    elif s.kind == skConst: 
       case s.magic
       of mIsMainModule: result = newIntNodeT(ord(sfMainModule in m.flags), n)
       of mCompileDate: result = newStrNodeT(times.getDateStr(), n)
@@ -370,10 +368,14 @@ proc getConstExpr(m: PSym, n: PNode): PNode =
       of mLow: 
         result = newIntNodeT(firstOrd(n.sons[1].typ), n)
       of mHigh: 
-        if not (skipTypes(n.sons[1].typ, abstractVar).kind in
+        if  skipTypes(n.sons[1].typ, abstractVar).kind notin
-            {tyOpenArray, tySequence, tyString}): 
+            {tyOpenArray, tySequence, tyString}: 
-          result = newIntNodeT(lastOrd(skipTypes(n.sons[1].typ, abstractVar)),
+          result = newIntNodeT(lastOrd(skipTypes(n[1].typ, abstractVar)), n)
-                               n)
+        else:
+          var a = n.sons[1]
+          if a.kind == nkBracket:
+            # we can optimize it away: 
+            result = newIntNodeT(sonsLen(a)-1, n)
       of mLengthOpenArray:
         var a = n.sons[1]
         if a.kind == nkBracket: 

compiler/semstmts.nim

@@ -280,6 +280,17 @@ proc semConst(c: PContext, n: PNode): PNode =
     var typ: PType = nil
     if a.sons[1].kind != nkEmpty: typ = semTypeNode(c, a.sons[1], nil)
 
+    when true:
+      var def = semConstExpr(c, a.sons[2])
+      if def == nil: GlobalError(a.sons[2].info, errConstExprExpected)
+      # check type compatibility between def.typ and typ:
+      if typ != nil:
+        def = fitRemoveHiddenConv(c, typ, def)
+      else:
+        typ = def.typ
+      if not typeAllowed(typ, skConst):
+        GlobalError(a.info, errXisNoType, typeToString(typ))
+    else:
       var e = semExprWithType(c, a.sons[2])
       if e == nil: GlobalError(a.sons[2].info, errConstExprExpected)
       var def = getConstExpr(c.module, e)

compiler/transf.nim

@@ -161,7 +161,6 @@ proc newAsgnStmt(c: PTransf, le: PNode, ri: PTransNode): PTransNode =
 
 proc transformSymAux(c: PTransf, n: PNode): PNode = 
   var b: PNode
-  if (n.kind != nkSym): internalError(n.info, "transformSym")
   var tc = c.transCon
   if sfBorrow in n.sym.flags: 
     # simply exchange the symbol:
@@ -176,6 +175,7 @@ proc transformSymAux(c: PTransf, n: PNode): PNode =
     if result != nil: return
     tc = tc.next
   result = b
+  when false:
     case b.sym.kind
     of skConst, skEnumField: 
       if sfFakeConst notin b.sym.flags:
@@ -360,7 +360,7 @@ proc transformConv(c: PTransf, n: PNode): PTransNode =
   var dest = skipTypes(n.typ, abstractVarRange)
   var source = skipTypes(n.sons[1].typ, abstractVarRange)
   case dest.kind
-  of tyInt..tyInt64, tyEnum, tyChar, tyBool: 
+  of tyInt..tyInt64, tyEnum, tyChar, tyBool, tyUInt..tyUInt64: 
     if not isOrdinalType(source):
       # XXX int64 -> float conversion?
       result = transformSons(c, n)
@@ -659,7 +659,7 @@ proc transformCall(c: PTransf, n: PNode): PTransNode =
 proc transform(c: PTransf, n: PNode): PTransNode = 
   case n.kind
   of nkSym: 
-    return transformSym(c, n)
+    result = transformSym(c, n)
   of nkEmpty..pred(nkSym), succ(nkSym)..nkNilLit: 
     # nothing to be done for leaves:
     result = PTransNode(n)
@@ -719,7 +719,7 @@ proc transform(c: PTransf, n: PNode): PTransNode =
   else:
     result = transformSons(c, n)
   var cnst = getConstExpr(c.module, PNode(result))
-  if cnst != nil: 
+  if cnst != nil and (cnst.kind != nkBracket or cnst.len == 0): 
     result = PTransNode(cnst) # do not miss an optimization  
  
 proc processTransf(context: PPassContext, n: PNode): PNode = 

compiler/types.nim

@@ -762,7 +762,7 @@ proc typeAllowedAux(marker: var TIntSet, typ: PType, kind: TSymKind): bool =
   of tyOpenArray, tyVarargs: 
     result = (kind == skParam) and typeAllowedAux(marker, t.sons[0], skVar)
   of tySequence: 
-    result = (kind != skConst) and typeAllowedAux(marker, t.sons[0], skVar) or
+    result = typeAllowedAux(marker, t.sons[0], skVar) or
         t.sons[0].kind == tyEmpty
   of tyArray:
     result = typeAllowedAux(marker, t.sons[1], skVar) or

doc/manual.txt

@@ -417,37 +417,6 @@ The grammar's start symbol is ``module``.
 Semantics
 =========
 
-Constants
----------
-
-`Constants`:idx: are symbols which are bound to a value. The constant's value
-cannot change. The compiler must be able to evaluate the expression in a
-constant declaration at compile time.
-
-Nimrod contains a sophisticated compile-time evaluator, so procedures which
-have no side-effect can be used in constant expressions too:
-
-.. code-block:: nimrod
-  import strutils
-  const
-    constEval = contains("abc", 'b') # computed at compile time!
-
-
-The rules for compile-time computability are: 
-
-1. Literals are compile-time computable.
-2. Type conversions are compile-time computable.
-3. Procedure calls of the form ``p(X)`` are compile-time computable if
-   ``p`` is a proc without side-effects (see the `noSideEffect pragma`_ 
-   for details) and if ``X`` is a (possibly empty) list of compile-time 
-   computable arguments.
-
-
-Constants cannot be of type ``var`` or ``object``, nor can 
-they contain such a type. For the types ``ptr`` and ``ref`` only the
-constant literal ``nil`` is possible.
-
-
 Types
 -----
 
@@ -671,8 +640,8 @@ and ``pred`` are not available for them either.
 
 
 The compiler supports the built-in stringify operator ``$`` for enumerations.
-The stringify's result can be controlled by specifying the string values to
+The stringify's result can be controlled by explicitely giving the string 
-use explicitely:
+values to use:
 
 .. code-block:: nimrod
 
@@ -1521,6 +1490,7 @@ variables of the same type:
 
 If an initializer is given the type can be omitted: the variable is then of the
 same type as the initializing expression. Variables are always initialized
+
 with a default value if there is no initializing expression. The default
 value depends on the type and is always a zero in binary.
 
@@ -1554,17 +1524,32 @@ Syntax::
               | COMMENT
   constSection ::= 'const' indPush constDecl (SAD constDecl)* DED indPop
 
+`Constants`:idx: are symbols which are bound to a value. The constant's value
+cannot change. The compiler must be able to evaluate the expression in a
+constant declaration at compile time.
 
-Example:
+Nimrod contains a sophisticated compile-time evaluator, so procedures which
+have no side-effect can be used in constant expressions too:
 
 .. code-block:: nimrod
-
+  import strutils
   const
-    MyFilename = "/home/my/file.txt"
+    constEval = contains("abc", 'b') # computed at compile time!
-    debugMode: bool = false
 
-The `const`:idx: section declares symbolic constants. A symbolic constant is
+
-a name for a constant expression. Symbolic constants only allow read-access.
+The rules for compile-time computability are: 
+
+1. Literals are compile-time computable.
+2. Type conversions are compile-time computable.
+3. Procedure calls of the form ``p(X)`` are compile-time computable if
+   ``p`` is a proc without side-effects (see the `noSideEffect pragma`_ 
+   for details) and if ``X`` is a (possibly empty) list of compile-time 
+   computable arguments.
+
+
+Constants cannot be of type ``var`` or ``object``, nor can 
+they contain such a type. For the types ``ptr`` and ``ref`` only the
+constant literal ``nil`` is possible.
 
 
 If statement
@@ -2398,7 +2383,6 @@ Example:
   add(root, newNode("hallo")) # instantiates generic procs ``newNode`` and
   add(root, newNode("world")) # ``add``
   for str in inorder(root):
-
     writeln(stdout, str)
 
 `Generics`:idx: are Nimrod's means to parametrize procs, iterators or types with
@@ -3338,6 +3322,12 @@ improve compile times.
 A thread proc is passed to ``createThread`` and invoked indirectly; so the
 ``thread`` pragma implies ``procvar``.
 
+If a global variable can also be marked with the ``thread`` pragma; it is 
+a `thead-local`:idx: variable then:
+
+.. code-block:: nimrod
+  var checkpoints* {.thread.}: seq[string] = @[]
+
 
 Actor model
 -----------

lib/nimbase.h

@@ -362,7 +362,7 @@ static N_INLINE(NI32, float32ToInt32)(float val) {
 #define STRING_LITERAL(name, str, length) \
   static const struct {                   \
     TGenericSeq Sup;                      \
-    NIM_CHAR data[length + 1];            \
+    NIM_CHAR data[(length) + 1];          \
   } name = {{length, length}, str}
 
 typedef struct TStringDesc* string;

lib/pure/strutils.nim

@@ -8,8 +8,8 @@
 #
 
 ## This module contains various string utility routines.
-## See the module `re` for regular expression support.
+## See the module `re <re.html>`_ for regular expression support.
-## See the module `pegs` for PEG support.
+## See the module `pegs <pegs.html>`_ for PEG support.
 
 import parseutils
 
@@ -369,6 +369,19 @@ proc splitLines*(s: string): seq[string] {.noSideEffect,
   ## sequence of substrings.
   accumulateResult(splitLines(s))
 
+proc countLines*(s: string): int {.noSideEffect,
+  rtl, extern: "nsuCountLines".} =
+  ## same as ``len(splitLines(s))``, but much more efficient.
+  var i = 0
+  while i < s.len:
+    case s[i]
+    of '\c':
+      if s[i+1] == '\l': inc i
+      inc result
+    of '\l': inc result
+    else: nil
+    inc i
+
 proc split*(s: string, seps: set[char] = Whitespace): seq[string] {.
   noSideEffect, rtl, extern: "nsuSplitCharSet".} =
   ## The same as the `split` iterator, but is a proc that returns a
@@ -958,7 +971,7 @@ proc c_sprintf(buf, frmt: CString) {.nodecl, importc: "sprintf", varargs,
                                      noSideEffect.}
 
 type
-  TFloatFormat* = enum
+  TFloatFormat* = enum ## the different modes of floating point formating
     ffDefault,    ## use the shorter floating point notation
     ffDecimal,    ## use decimal floating point notation
     ffScientific  ## use scientific notation (using ``e`` character)

tests/accept/compile/tconsteval.nim

@@ -1,5 +1,4 @@
 discard """
-  disabled: true
 """
 
 import strutils

todo.txt

@@ -1,10 +1,10 @@
 Version 0.8.14
 ==============
 
+- fix the 'const' issues
 - 'let x = y'; const ptr/ref
 - fix actors.nim
 - make threadvar efficient again on linux after testing
-- fix the 'const' issues
 - test the sort implementation again
 - optional indentation for 'case' statement
 

web/news.txt

@@ -76,7 +76,7 @@ Library Additions
   ``system.deallocShared``, ``system.reallocShared``.
 - Added explicit channels for thread communication.
 - Added ``matchers`` module for email address etc. matching.
-- Added ``strutils.unindent``.
+- Added ``strutils.unindent``, ``strutils.countLines``.
 - Added ``system.slurp`` for easy resource embedding.
 - Added ``system.running`` for threads.
 - Added ``xmltree.innerText``.