compilation cache: methods have a chance to work

compiler/ccgthreadvars.nim

@@ -8,7 +8,7 @@
 #
 
 ## Thread var support for crappy architectures that lack native support for 
-## thread local storage.
+## thread local storage. (**Thank you Mac OS X!**)
 
 proc emulatedThreadVars(): bool {.inline.} =
   result = optThreads in gGlobalOptions
@@ -23,9 +23,17 @@ proc AccessThreadLocalVar(p: BProc, s: PSym) =
     appcg(p, cpsInit, "NimTV=(NimThreadVars*)#GetThreadLocalVars();$n")
 
 var
-  nimtv: PRope # nimrod thread vars
+  nimtv: PRope                 # nimrod thread vars; the struct body
-  nimtvDeps: seq[PType] = @[]
+  nimtvDeps: seq[PType] = @[]  # type deps: every module needs whole struct
-  nimtvDeclared = initIntSet()
+  nimtvDeclared = initIntSet() # so that every var/field exists only once
+                               # in the struct
+
+# 'nimtv' is incredibly hard to modularize! Best effort is to store all thread
+# vars in a ROD section and with their type deps and load them
+# unconditionally...
+
+# nimtvDeps is VERY hard to cache because it's not a list of IDs nor can it be
+# made to be one.
 
 proc declareThreadVar(m: BModule, s: PSym, isExtern: bool) =
   if emulatedThreadVars():

compiler/rodread.nim

@@ -9,10 +9,11 @@
 
 # This module is responsible for loading of rod files.
 #
-#  Reading and writing binary files are really hard to debug. Therefore we use
+# Reading and writing binary files are really hard to debug. Therefore we use
-#  a special text format. ROD-files are more efficient to process because
+# a "creative" text/binary hybrid format. ROD-files are more efficient
-#  symbols are only loaded on demand.
+# to process because symbols are can be loaded on demand.
-#  It consists of:
+# 
+# A ROD file consists of:
 #
 #  - a header:
 #    NIM:$fileversion\n
@@ -28,7 +29,7 @@
 #    myfile.inc
 #    lib/mymodA
 #    )
-#  - a include file dependency section:
+#  - an include file dependency section:
 #    INCLUDES(
 #    <fileidx> <CRC of myfile.inc>\n # fileidx is the LINE in the file section!
 #    )
@@ -48,6 +49,11 @@
 #    id-diff idx-diff\n
 #    id-diff idx-diff\n
 #    )
+#
+#    Since the whole index has to be read in advance, we compress it by 
+#    storing the integer differences to the last entry instead of using the
+#    real numbers.
+#
 #  - an import index consisting of (ID, moduleID)-pairs:
 #    IMPORTS(
 #    id-diff moduleID-diff\n
@@ -55,7 +61,12 @@
 #    )
 #  - a list of all exported type converters because they are needed for correct
 #    semantic checking:
-#    CONVERTERS:id id\n   # position of the symbol in the DATA section
+#    CONVERTERS:id id\n   # symbol ID
+#
+#  - a list of all (private or exported) methods because they are needed for
+#    correct dispatcher generation:
+#    METHODS: id id\n   # symbol ID
+#
 #  - an AST section that contains the module's AST:
 #    INIT(
 #    idx\n  # position of the node in the DATA section
@@ -72,8 +83,6 @@
 #    stops immediately after ``DATA(`` and the rest is only loaded on demand
 #    by using mem'mapped a file.
 #
-#  We now also do index compression, because an index always needs to be read.
-#
 
 import 
   os, options, strutils, nversion, ast, astalgo, msgs, platform, condsyms, 
@@ -116,7 +125,7 @@ type
     files: TStringSeq
     dataIdx: int             # offset of start of data section
     convertersIdx: int       # offset of start of converters section
-    initIdx, interfIdx, compilerProcsIdx: int
+    initIdx, interfIdx, compilerProcsIdx, methodsIdx: int
     filename: string
     index, imports: TIndex
     readerIndex: int
@@ -125,6 +134,7 @@ type
     syms: TIdTable       # already processed symbols
     memfile: TMemFile    # unfortunately there is no point in time where we
                          # can close this! XXX
+    methods*: TSymSeq
   
   PRodReader* = ref TRodReader
 
@@ -547,6 +557,9 @@ proc processRodFile(r: PRodReader, crc: TCrc32) =
     of "CONVERTERS": 
       r.convertersIdx = r.pos + 1
       skipSection(r)
+    of "METHODS":
+      r.methodsIdx = r.pos + 1
+      skipSection(r)
     of "DATA": 
       r.dataIdx = r.pos + 2 # "(\10"
       # We do not read the DATA section here! We read the needed objects on
@@ -573,6 +586,7 @@ proc newRodReader(modfilename: string, crc: TCrc32,
   new(result)
   result.files = @[]
   result.modDeps = @[]
+  result.methods = @[]
   var r = result
   r.reason = rrNone
   r.pos = 0
@@ -681,7 +695,8 @@ proc loadInitSection(r: PRodReader): PNode =
   r.pos = oldPos
 
 proc loadConverters(r: PRodReader) = 
-  # We have to ensure that no exported converter is a stub anymore.
+  # We have to ensure that no exported converter is a stub anymore, and the
+  # import mechanism takes care of the rest.
   if r.convertersIdx == 0 or r.dataIdx == 0: 
     InternalError("importConverters")
   r.pos = r.convertersIdx
@@ -690,6 +705,15 @@ proc loadConverters(r: PRodReader) =
     discard rrGetSym(r, d, UnknownLineInfo())
     if r.s[r.pos] == ' ': inc(r.pos)
 
+proc loadMethods(r: PRodReader) =
+  if r.methodsIdx == 0 or r.dataIdx == 0:
+    InternalError("loadMethods")
+  r.pos = r.methodsIdx
+  while r.s[r.pos] > '\x0A':
+    var d = decodeVInt(r.s, r.pos)
+    r.methods.add(rrGetSym(r, d, UnknownLineInfo()))
+    if r.s[r.pos] == ' ': inc(r.pos)
+  
 proc getModuleIdx(filename: string): int = 
   for i in countup(0, high(gMods)): 
     if sameFile(gMods[i].filename, filename): return i
@@ -752,6 +776,7 @@ proc handleSymbolFile(module: PSym, filename: string): PRodReader =
     processInterf(result, module)
     processCompilerProcs(result, module)
     loadConverters(result)
+    loadMethods(result)
   else:
     module.id = getID()
   

compiler/rodwrite.nim

@@ -29,7 +29,7 @@ type
     interf: string
     compilerProcs: string
     index, imports: TIndex
-    converters: string
+    converters, methods: string
     init: string
     data: string
     filename: string
@@ -85,6 +85,7 @@ proc newRodWriter(modfilename: string, crc: TCrc32, module: PSym): PRodWriter =
   result.interf = newStringOfCap(2_000)
   result.compilerProcs = ""
   result.converters = ""
+  result.methods = ""
   result.init = ""
   result.data = newStringOfCap(12_000)
   
@@ -350,6 +351,9 @@ proc symStack(w: PRodWriter) =
         if s.kind == skConverter: 
           if w.converters.len != 0: add(w.converters, ' ')
           encodeVInt(s.id, w.converters)
+        elif s.kind == skMethod:
+          if w.methods.len != 0: add(w.methods, ' ')
+          encodeVInt(s.id, w.methods)
       elif IiTableGet(w.imports.tab, s.id) == invalidKey: 
         addToIndex(w.imports, s.id, m.id) #if not Contains(debugWritten, s.id):
                                           #  MessageOut(w.filename);
@@ -456,6 +460,10 @@ proc writeRod(w: PRodWriter) =
   f.write(w.converters)
   f.write(rodNL)
 
+  f.write("METHODS:")
+  f.write(w.methods)
+  f.write(rodNL)
+  
   f.write("INIT(" & rodNL)
   f.write(w.init)
   f.write(')' & rodNL)
@@ -486,8 +494,8 @@ proc process(c: PPassContext, n: PNode): PNode =
   of nkProcDef, nkMethodDef, nkIteratorDef, nkConverterDef: 
     var s = n.sons[namePos].sym
     if s == nil: InternalError(n.info, "rodwrite.process")
-    if (n.sons[codePos].kind != nkEmpty) or (s.magic != mNone) or
+    if n.sons[codePos].kind != nkEmpty or s.magic != mNone or
-        not (sfForward in s.flags): 
+        sfForward notin s.flags:
       addInterfaceSym(w, s)
   of nkVarSection:
     for i in countup(0, sonsLen(n) - 1): 

compiler/transf.nim

@@ -18,7 +18,7 @@
 
 import 
   intsets, strutils, lists, options, ast, astalgo, trees, treetab, msgs, os, 
-  idents, renderer, types, passes, semfold, magicsys, cgmeth
+  idents, renderer, types, passes, semfold, magicsys, cgmeth, rodread
 
 const 
   genPrefix* = ":tmp"         # prefix for generated names
@@ -726,7 +726,7 @@ proc processTransf(context: PPassContext, n: PNode): PNode =
   # Note: For interactive mode we cannot call 'passes.skipCodegen' and skip
   # this step! We have to rely that the semantic pass transforms too errornous
   # nodes into an empty node.
-  if passes.skipCodegen(n): return n
+  if passes.skipCodegen(n) or context.fromCache: return n
   var c = PTransf(context)
   pushTransCon(c, newTransCon(getCurrOwner(c)))
   result = PNode(transform(c, n))
@@ -739,9 +739,15 @@ proc openTransf(module: PSym, filename: string): PPassContext =
   n.module = module
   result = n
 
+proc openTransfCached(module: PSym, filename: string, 
+                      rd: PRodReader): PPassContext = 
+  result = openTransf(module, filename)
+  for m in items(rd.methods): methodDef(m)
+
 proc transfPass(): TPass = 
   initPass(result)
   result.open = openTransf
+  result.openCached = openTransfCached
   result.process = processTransf
   result.close = processTransf # we need to process generics too!
   

lib/system.nim

@@ -1951,7 +1951,9 @@ template spliceImpl(x, start, endp, spliced: expr): stmt =
     setLen(x, newLen)
 
 proc `[]=`*(s: var string, x: TSlice[int], b: string) = 
-  ## slice assignment for strings. Negative indexes are supported.
+  ## slice assignment for strings. Negative indexes are supported. If
+  ## ``b.len`` is not exactly the number of elements that are referred to
+  ## by `x`, a `splice`:idx: is performed. 
   var a = x.a-|s
   var L = x.b-|s - a + 1
   if L == b.len:
@@ -1960,15 +1962,15 @@ proc `[]=`*(s: var string, x: TSlice[int], b: string) =
     spliceImpl(s, x.a, x.b, b)
 
 proc `[]`*[Idx, T](a: array[Idx, T], x: TSlice[int]): seq[T] =
-  ## slice operation for arrays. Negative indexes are NOT supported because
+  ## slice operation for arrays. Negative indexes are **not** supported
-  ## the array might have negative bounds.
+  ## because the array might have negative bounds.
   var L = x.b - x.a + 1
   newSeq(result, L)
   for i in 0.. <L: result[i] = a[i + x.a]
 
 proc `[]=`*[Idx, T](a: var array[Idx, T], x: TSlice[int], b: openArray[T]) =
-  ## slice assignment for arrays. Negative indexes are NOT supported because
+  ## slice assignment for arrays. Negative indexes are **not** supported
-  ## the array might have negative bounds.
+  ## because the array might have negative bounds.
   var L = x.b - x.a + 1
   if L == b.len:
     for i in 0 .. <L: a[i+x.a] = b[i]
@@ -1976,8 +1978,8 @@ proc `[]=`*[Idx, T](a: var array[Idx, T], x: TSlice[int], b: openArray[T]) =
     raise newException(EOutOfRange, "differing lengths for slice assignment")
 
 proc `[]`*[Idx, T](a: array[Idx, T], x: TSlice[Idx]): seq[T] =
-  ## slice operation for arrays. Negative indexes are NOT supported because
+  ## slice operation for arrays. Negative indexes are **not** supported
-  ## the array might have negative bounds.
+  ## because the array might have negative bounds.
   var L = ord(x.b) - ord(x.a) + 1
   newSeq(result, L)
   var j = x.a
@@ -1986,8 +1988,8 @@ proc `[]`*[Idx, T](a: array[Idx, T], x: TSlice[Idx]): seq[T] =
     inc(j)
 
 proc `[]=`*[Idx, T](a: var array[Idx, T], x: TSlice[Idx], b: openArray[T]) =
-  ## slice assignment for arrays. Negative indexes are NOT supported because
+  ## slice assignment for arrays. Negative indexes are **not** supported
-  ## the array might have negative bounds.
+  ## because the array might have negative bounds.
   var L = ord(x.b) - ord(x.a) + 1
   if L == b.len:
     var j = x.a
@@ -2005,7 +2007,9 @@ proc `[]`*[T](s: seq[T], x: TSlice[int]): seq[T] =
   for i in 0.. <L: result[i] = s[i + a]
 
 proc `[]=`*[T](s: var seq[T], x: TSlice[int], b: openArray[T]) = 
-  ## slice assignment for sequences. Negative indexes are supported.
+  ## slice assignment for sequences. Negative indexes are supported. If
+  ## ``b.len`` is not exactly the number of elements that are referred to
+  ## by `x`, a `splice`:idx: is performed. 
   var a = x.a-|s
   var L = x.b-|s - a + 1
   if L == b.len:

todo.txt

@@ -1,25 +1,34 @@
 Version 0.8.14
 ==============
 
-- optimize unused constants away
+- optimize unused constants away (affected by HLO)
 - 'let x = y'
 - fix actors.nim
 - make threadvar efficient again on linux after testing
 - test the sort implementation again
 - document & test splicing; don't forget to test negative indexes
 - implement lib/pure/memfiles properly
+- eval context is per module; this way modularity is kept; global id generation
+  macro can still be done once macros support basic IO (store current id in
+  some file)
+
 
 incremental compilation
 -----------------------
 
 - adapt thread var implementation to care about the new merge operation
 - write test cases: needs test script support
-  - test type converters
+  - test G, A, B example from the documentation
   - test thread var
-  - test method generation
+  - test method generation; could work, needs ROD support though
   - test init sections
   - test DLL interfacing!
   - hallo.rod is missing initial statements: feature or bug?
+
+- automate tests:
+  - test basic recompilation scheme
+  - test type converters
+
 - fix remaining bugs
 
 
@@ -57,6 +66,9 @@ Bugs
     result = forward(x)
 
 - bug: DLL generation is broken
+- bug: stress testing basic method example (eval) without ``-d:relase`` leaks
+  memory; good way to figure out how a fixed amount of stack can hold
+  an arbitrary number of GC roots!
 
 
 version 0.9.XX

web/news.txt

@@ -61,6 +61,8 @@ Language Additions
   heart's content.
 - ``bind`` (used for symbol binding in templates and generics) is now a
   declarative statement.
+- The slice assignment ``a[i..j] = b`` where ``a`` is a sequence or string
+  now supports *splicing*.
 
 
 Compiler Additions