Merge branch 'master' of github.com:Araq/Nimrod

compiler/ast.nim

@@ -341,7 +341,7 @@ type
   TMagic* = enum # symbols that require compiler magic:
     mNone, mDefined, mDefinedInScope, mLow, mHigh, mSizeOf, mIs, mOf,
     mEcho, mShallowCopy, mSlurp,
-    mAstToYaml, mParseExprToAst, mParseStmtToAst, mExpandToAst,
+    mParseExprToAst, mParseStmtToAst, mExpandToAst,
     mUnaryLt, mSucc, 
     mPred, mInc, mDec, mOrd, mNew, mNewFinalize, mNewSeq, mLengthOpenArray, 
     mLengthStr, mLengthArray, mLengthSeq, mIncl, mExcl, mCard, mChr, mGCref, 

compiler/evals.nim

@@ -1118,9 +1118,6 @@ proc evalMagicOrCall(c: PEvalContext, n: PNode): PNode =
     result = evalAux(c, n.sons[1], {})
     if isSpecial(result): return
     result = newStrNodeT(result.info.toFileLineCol, n)
-  of mAstToYaml:
-    var ast = evalAux(c, n.sons[1], {efLValue})
-    result = newStrNode(nkStrLit, ast.treeToYaml.ropeToStr)
   of mNHint: 
     result = evalAux(c, n.sons[1], {})
     if isSpecial(result): return 

lib/core/macros.nim

@@ -189,40 +189,8 @@ proc toStrLit*(n: PNimrodNode): PNimrodNode {.compileTime.} =
   return newStrLitNode(repr(n))
 
 proc lineinfo*(n: PNimrodNode): string {.magic: "NLineInfo".}
-
+  ## returns the position the node appears in the original source file
-proc toLisp*(n: PNimrodNode): string {.compileTime.} =
+  ## in the form filename(line, col)
-  ## Convert the AST `n` to a human-readable string
-  ##
-  ## You can use this as a tool to explore the Nimrod's abstract syntax 
-  ## tree and to discover what kind of nodes must be created to represent
-  ## a certain expression/statement
-
-  if n == nil: return "nil"
-
-  result = $n.kind
-  add(result, "(")
-  
-  case n.kind
-  of nnkEmpty: nil # same as nil node in this representation
-  of nnkNilLit: add(result, "nil")
-  of nnkCharLit..nnkInt64Lit: add(result, $n.intVal)
-  of nnkFloatLit..nnkFloat64Lit: add(result, $n.floatVal)
-  of nnkStrLit..nnkTripleStrLit: add(result, $n.strVal)
-  of nnkIdent: add(result, $n.ident)
-  of nnkSym, nnkNone: assert false
-  else:
-    add(result, toLisp(n[0]))
-    for j in 1..n.len-1:
-      add(result, ", ")
-      add(result, toLisp(n[j]))
-
-  add(result, ")")
-
-proc toYaml*(n: PNimrodNode): string {.magic: "AstToYaml".}
-  ## Converts the AST `n` to an YAML string
-  ##
-  ## Provides more detailed, potentially harder to digest information
-  ## than `toLisp`
 
 proc parseExpr*(s: string): expr {.magic: "ParseExprToAst".}
   ## Compiles the passed string to its AST representation.
@@ -285,3 +253,72 @@ proc nestList*(theProc: TNimrodIdent,
   for i in countdown(L-3, 0):
     a = newCall(theProc, x[i], copyNimTree(a))
 
+proc treeRepr*(n: PNimrodNode): string {.compileTime.} =
+  ## Convert the AST `n` to a human-readable tree-like string
+  ##
+  ## see also `repr` and `lispRepr`
+
+  proc traverse(res: var string, level: int, n: PNimrodNode) =
+    for i in 0..level-1: res.add "  "
+    
+    if n == nil:
+      res.add "nil"
+    else:
+      res.add(($n.kind).substr(3))
+      
+      case n.kind
+      of nnkEmpty: nil # same as nil node in this representation
+      of nnkNilLit: res.add(" nil")
+      of nnkCharLit..nnkInt64Lit: res.add(" " & $n.intVal)
+      of nnkFloatLit..nnkFloat64Lit: res.add(" " & $n.floatVal)
+      of nnkStrLit..nnkTripleStrLit: res.add(" " & $n.strVal)
+      of nnkIdent: res.add(" " & $n.ident)
+      of nnkSym, nnkNone: assert false
+      else:
+        for j in 0..n.len-1:
+          res.add "\n"
+          traverse(res, level + 1, n[j])
+
+  result = ""
+  traverse(result, 0, n)
+
+proc lispRepr*(n: PNimrodNode): string {.compileTime.} =
+  ## Convert the AST `n` to a human-readable lisp-like string
+  ##
+  ## see also `repr` and `treeRepr`
+  
+  if n == nil: return "nil"
+
+  result = ($n.kind).substr(3)
+  add(result, "(")
+  
+  case n.kind
+  of nnkEmpty: nil # same as nil node in this representation
+  of nnkNilLit: add(result, "nil")
+  of nnkCharLit..nnkInt64Lit: add(result, $n.intVal)
+  of nnkFloatLit..nnkFloat64Lit: add(result, $n.floatVal)
+  of nnkStrLit..nnkTripleStrLit: add(result, $n.strVal)
+  of nnkIdent: add(result, $n.ident)
+  of nnkSym, nnkNone: assert false
+  else:
+    add(result, lispRepr(n[0]))
+    for j in 1..n.len-1:
+      add(result, ", ")
+      add(result, lispRepr(n[j]))
+
+  add(result, ")")
+
+macro dumpTree*(s: stmt): stmt = echo s[1].treeRepr
+  ## Accepts a block of nimrod code and prints the parsed abstract syntax
+  ## tree using the `toTree` function.
+  ##
+  ## You can use this as a tool to explore the Nimrod's abstract syntax 
+  ## tree and to discover what kind of nodes must be created to represent
+  ## a certain expression/statement
+
+macro dumpLisp*(s: stmt): stmt = echo s[1].lispRepr
+  ## Accepts a block of nimrod code and prints the parsed abstract syntax
+  ## tree using the `toLisp` function.
+  ##
+  ## see `dumpTree`
+

lib/pure/memfiles.nim

@@ -9,43 +9,186 @@
 
 ## This module provides support for `memory mapped files`:idx:
 ## (Posix's `mmap`:idx:) on the different operating systems.
-## XXX Currently it is implemented with Nimrod's
+
-## basic IO facilities and does not use any platform specific code!
+when defined(windows):
-## Oh and currently only ``fmRead`` is supported...
+  import windows
+elif defined(posix):
+  import posix
+else:
+  {.error: "the memfiles module is not supported yet on your operating system!".}
+
+## mem
+## a pointer to the memory mapped file `f`. The pointer can be
+## used directly to change the contents of the file, if `f` was opened
+## with write access.
+
+## size
+## size of the memory mapped file `f`.
+
+when defined(windows):
+  type Tsize = int64
+else:
+  type Tsize = int
 
 type
   TMemFile* = object {.pure.} ## represents a memory mapped file
-    file: TFile
+    mem*: pointer # XXX: The compiler won't let me add comments on the next line
-    buffer: pointer
+    size*: Tsize
-    fileLen: int
 
-proc open*(f: var TMemFile, filename: string, mode: TFileMode = fmRead): bool =
+    when defined(windows):
-  ## open a memory mapped file `f`. Returns true for success.
+      fHandle: int
-  assert mode == fmRead
+      mapHandle: int 
-  result = open(f.file, filename, mode)
-
-  var len = getFileSize(f.file)
-  if len < high(int):
-    f.fileLen = int(len)
-    f.buffer = alloc(f.fileLen)
-    if readBuffer(f.file, f.buffer, f.fileLen) != f.fileLen:
-      raise newException(EIO, "error while reading from file")
     else:
-    raise newException(EIO, "file too big to fit in memory")
+      handle: cint
+
+proc open*(
+  f           : var TMemFile,
+  filename    : string,
+  mode        : TFileMode = fmRead,
+  mappedSize  : int       = -1,
+  offset      : Tsize     = 0,
+  newFileSize : Tsize     = -1 ): bool =
+  ## open a memory mapped file `f`. Returns true for success.
+
+  # The file can be resized only when write mode is used
+  assert newFileSize == -1 or mode != fmRead
+  var readonly = mode == fmRead
+
+  template rollback =
+    f.mem = nil
+    f.size = 0
+
+  when defined(windows):
+    template fail(msg: expr) =
+      rollback()
+      if f.fHandle != 0: discard CloseHandle(f.fHandle)
+      if f.mapHandle != 0: discard CloseHandle(f.mapHandle)
+      # return false
+      raise newException(EIO, msg)
+      
+    f.fHandle = CreateFileA(
+      filename,
+      if readonly: GENERIC_READ else: GENERIC_ALL,
+      FILE_SHARE_READ,
+      nil,
+      if newFileSize != -1: CREATE_ALWAYS else: OPEN_EXISTING,
+      if readonly: FILE_ATTRIBUTE_READONLY else: FILE_ATTRIBUTE_TEMPORARY,
+      0)
+
+    if f.fHandle == INVALID_HANDLE_VALUE:
+      fail "error opening file"
+
+    if newFileSize != -1:
+      var 
+        sizeHigh = int32(newFileSize shr 32)
+        sizeLow  = int32(newFileSize and 0xffffffff)
+
+      var status = SetFilePointer(f.fHandle, sizeLow, addr(sizeHigh), FILE_BEGIN)
+      if (status == INVALID_SET_FILE_POINTER and GetLastError() != NO_ERROR) or
+         (SetEndOfFile(f.fHandle) == 0):
+        fail "error setting file size"
+
+    f.mapHandle = CreateFileMapping(
+      f.fHandle, nil,
+      if readonly: PAGE_READONLY else: PAGE_READWRITE,
+      0, 0, nil)
+
+    if f.mapHandle == 0:
+      fail "error creating mapping"
+
+    f.mem = MapViewOfFileEx(
+      f.mapHandle,
+      if readonly: FILE_MAP_READ else: FILE_MAP_WRITE,
+      int32(offset shr 32),
+      int32(offset and 0xffffffff),
+      if mappedSize == -1: 0 else: mappedSize,
+      nil)
+
+    if f.mem == nil:
+      fail "error mapping view"
+
+    var hi, low: int32
+    low = GetFileSize(f.fHandle, addr(hi))
+    if low == INVALID_FILE_SIZE:
+      fail "error getting file size"
+    else:
+      var fileSize = (int64(hi) shr 32) or low
+      f.size = if mappedSize != -1: min(fileSize, mappedSize) else: fileSize
+
+    result = true
+
+  else:
+    template fail(msg: expr) =
+      rollback()
+      if f.handle != 0:
+        discard close(f.handle)
+      # return false
+      raise newException(system.EIO, msg)
+  
+    var flags = if readonly: O_RDONLY else: O_RDWR
+
+    if newFileSize != -1:
+      flags = flags or O_CREAT or O_TRUNC
+
+    f.handle = open(filename, flags)
+    if f.handle == -1:
+      # XXX: errno is supposed to be set here
+      # Is there an exception that wraps it?
+      fail "error opening file"
+
+    if newFileSize != -1:
+      if ftruncate(f.handle, newFileSize) == -1:
+        fail "error setting file size"
+
+    if mappedSize != -1:
+      f.size = mappedSize
+    else:
+      var stat: Tstat
+      if fstat(f.handle, stat) != -1:
+        # XXX: Hmm, this could be unsafe
+        # Why is mmap taking int anyway?
+        f.size = int(stat.st_size)
+      else:
+        fail "error getting file size"
+
+    f.mem = mmap(
+      nil,
+      f.size,
+      if readonly: PROT_READ else: PROT_READ or PROT_WRITE,
+      if readonly: MAP_PRIVATE else: MAP_SHARED,
+      f.handle,
+      offset)
+
+    if f.mem == cast[pointer](MAP_FAILED):
+      fail "file mapping failed"
+
+    result = true
 
 proc close*(f: var TMemFile) =
   ## closes the memory mapped file `f`. All changes are written back to the
   ## file system, if `f` was opened with write access.
-  dealloc(f.buffer)
-  close(f.file)
   
-proc mem*(f: var TMemFile): pointer {.inline.} =
+  var error = false
-  ## retrives a pointer to the memory mapped file `f`. The pointer can be
-  ## used directly to change the contents of the file, if `f` was opened
-  ## with write access.
-  result = f.buffer
 
-proc size*(f: var TMemFile): int {.inline.} =
+  when defined(windows):
-  ## retrives the size of the memory mapped file `f`.
+    if f.fHandle != INVALID_HANDLE_VALUE:
-  result = f.fileLen
+      error = UnmapViewOfFile(f.mem) == 0
+      error = (CloseHandle(f.mapHandle) == 0) or error
+      error = (CloseHandle(f.fHandle) == 0) or error
+  else:
+    if f.handle != 0:
+      error = munmap(f.mem, f.size) != 0
+      error = (close(f.handle) != 0) or error
+
+  f.size = 0
+  f.mem = nil
+
+  when defined(windows):
+    f.fHandle = 0
+    f.mapHandle = 0
+  else:
+    f.handle = 0
+  
+  if error:
+    raise newException(system.EIO, "error closing file")
 

lib/windows/windows.nim

@@ -3104,6 +3104,9 @@ const
   STD_ERROR_HANDLE* =  DWORD(-12)
 
   INVALID_HANDLE_VALUE* = HANDLE(-1)
+  INVALID_SET_FILE_POINTER* = ULONG(-1)
+  INVALID_FILE_SIZE* = ULONG(-1)
+  INVALID_FILE_ATTRIBUTES* = ULONG(-1)
 
 const
   # GetStockObject

web/news.txt

@@ -101,8 +101,8 @@ Library Additions
 - Added ``xmltree.innerText``.
 - Added ``os.isAbsolute``.
 - Added ``parseutils.interpolatedFragments``.
-- Added ``macros.toLisp``, ``macros.toYaml``, ``macros.parseExpr``,
+- Added ``macros.treeRepr``, ``macros.lispRepr``, ``macros.dumpTree``, 
-  ``macros.parseStmt``, ``macros.getAst``.
+  ``macros.dumpLisp``, ``macros.parseExpr``, ``macros.parseStmt``, ``macros.getAst``.
 - Added ``locks`` core module for more flexible locking support.
 - Added ``irc`` module.
 - Added ``ftpclient`` module.