memfiles now uses winlean; changed the interface to raise EOS

compiler/rodread.nim

@@ -632,7 +632,10 @@ proc newRodReader(modfilename: string, crc: TCrc32,
   r.readerIndex = readerIndex
   r.filename = modfilename
   InitIdTable(r.syms)
-  if not open(r.memFile, modfilename): return nil
+  try:
+    r.memFile = memfiles.open(modfilename)
+  except EOS:
+    return nil
   # we terminate the file explicitely with ``\0``, so the cast to `cstring`
   # is save:
   r.s = cast[cstring](r.memFile.mem)

doc/lib.txt

@@ -141,6 +141,10 @@ Generic Operating System Services
   (also called *console*). The implementation simply uses ANSI escape
   sequences and does not depend on any other module.
 
+* `memfiles <memfiles.html>`_
+  This module provides support for memory mapped files (Posix's ``mmap``) 
+  on the different operating systems.
+
 
 Math libraries
 --------------

lib/pure/collections/sequtils.nim

@@ -11,6 +11,9 @@
 ##
 ## This module implements operations for the built-in `seq`:idx: type
 ## which were inspired by functional programming languages.
+##
+## **Note**: This interface will change as soon as the compiler supports
+## closures and proper coroutines.
 
 proc concat*[T](seqs: openarray[seq[T]]): seq[T] =
   ## Takes several sequences' items and returns them inside of one sequence.

lib/pure/memfiles.nim

@@ -1,39 +1,32 @@
 #
 #
 #            Nimrod's Runtime Library
-#        (c) Copyright 2011 Andreas Rumpf
+#        (c) Copyright 2011 Nimrod Contributors
 #
 #    See the file "copying.txt", included in this
 #    distribution, for details about the copyright.
 #
 
+## :Authors: Zahary Karadjov, Andreas Rumpf
+##
 ## This module provides support for `memory mapped files`:idx:
 ## (Posix's `mmap`:idx:) on the different operating systems.
 
 when defined(windows):
-  import windows
+  import winlean
 elif defined(posix):
   import posix
 else:
-  {.error: "the memfiles module is not supported yet on your operating system!".}
+  {.error: "the memfiles module is not supported on your operating system!".}
 
-## mem
+import os
-## a pointer to the memory mapped file `f`. The pointer can be
-## used directly to change the contents of the file, if `f` was opened
-## with write access.
-
-## size
-## size of the memory mapped file `f`.
-
-when defined(windows):
-  type Tsize = int64
-else:
-  type Tsize = int
 
 type
   TMemFile* = object {.pure.} ## represents a memory mapped file
-    mem*: pointer # XXX: The compiler won't let me add comments on the next line
+    mem*: pointer    ## a pointer to the memory mapped file. The pointer
-    size*: Tsize
+                     ## can be used directly to change the contents of the
+                     ## file, if it was opened with write access.
+    size*: int       ## size of the memory mapped file
 
     when defined(windows):
       fHandle: int
@@ -41,32 +34,31 @@ type
     else:
       handle: cint
 
-proc open*(
+proc open*(filename: string, mode: TFileMode = fmRead,
-  f           : var TMemFile,
+           mappedSize = -1, offset = 0, newFileSize = -1): TMemFile =
-  filename    : string,
+  ## opens a memory mapped file. If this fails, ``EOS`` is raised.
-  mode        : TFileMode = fmRead,
+  ## `newFileSize` can only be set if the file is not opened with ``fmRead``
-  mappedSize  : int       = -1,
+  ## access. `mappedSize` and `offset` can be used to map only a slice of
-  offset      : Tsize     = 0,
+  ## the file.
-  newFileSize : Tsize     = -1 ): bool =
-  ## open a memory mapped file `f`. Returns true for success.
 
-  # The file can be resized only when write mode is used
+  # The file can be resized only when write mode is used:
   assert newFileSize == -1 or mode != fmRead
   var readonly = mode == fmRead
 
   template rollback =
-    f.mem = nil
+    result.mem = nil
-    f.size = 0
+    result.size = 0
 
   when defined(windows):
     template fail(msg: expr) =
       rollback()
-      if f.fHandle != 0: discard CloseHandle(f.fHandle)
+      if result.fHandle != 0: discard CloseHandle(result.fHandle)
-      if f.mapHandle != 0: discard CloseHandle(f.mapHandle)
+      if result.mapHandle != 0: discard CloseHandle(result.mapHandle)
+      OSError()
       # return false
-      raise newException(EIO, msg)
+      #raise newException(EIO, msg)
       
-    f.fHandle = CreateFileA(
+    result.fHandle = CreateFileA(
       filename,
       if readonly: GENERIC_READ else: GENERIC_ALL,
       FILE_SHARE_READ,
@@ -75,7 +67,7 @@ proc open*(
       if readonly: FILE_ATTRIBUTE_READONLY else: FILE_ATTRIBUTE_TEMPORARY,
       0)
 
-    if f.fHandle == INVALID_HANDLE_VALUE:
+    if result.fHandle == INVALID_HANDLE_VALUE:
       fail "error opening file"
 
     if newFileSize != -1:
@@ -83,87 +75,83 @@ proc open*(
         sizeHigh = int32(newFileSize shr 32)
         sizeLow  = int32(newFileSize and 0xffffffff)
 
-      var status = SetFilePointer(f.fHandle, sizeLow, addr(sizeHigh), FILE_BEGIN)
+      var status = SetFilePointer(result.fHandle, sizeLow, addr(sizeHigh),
+                                  FILE_BEGIN)
       if (status == INVALID_SET_FILE_POINTER and GetLastError() != NO_ERROR) or
-         (SetEndOfFile(f.fHandle) == 0):
+         (SetEndOfFile(result.fHandle) == 0):
         fail "error setting file size"
 
-    f.mapHandle = CreateFileMapping(
+    result.mapHandle = CreateFileMapping(
-      f.fHandle, nil,
+      result.fHandle, nil,
       if readonly: PAGE_READONLY else: PAGE_READWRITE,
       0, 0, nil)
 
-    if f.mapHandle == 0:
+    if result.mapHandle == 0:
       fail "error creating mapping"
 
-    f.mem = MapViewOfFileEx(
+    result.mem = MapViewOfFileEx(
-      f.mapHandle,
+      result.mapHandle,
       if readonly: FILE_MAP_READ else: FILE_MAP_WRITE,
       int32(offset shr 32),
       int32(offset and 0xffffffff),
       if mappedSize == -1: 0 else: mappedSize,
       nil)
 
-    if f.mem == nil:
+    if result.mem == nil:
       fail "error mapping view"
 
     var hi, low: int32
-    low = GetFileSize(f.fHandle, addr(hi))
+    low = GetFileSize(result.fHandle, addr(hi))
     if low == INVALID_FILE_SIZE:
       fail "error getting file size"
     else:
       var fileSize = (int64(hi) shr 32) or low
-      f.size = if mappedSize != -1: min(fileSize, mappedSize) else: fileSize
+      if mappedSize != -1: result.size = min(fileSize, mappedSize).int
-
+      else: result.size = fileSize.int
-    result = true
 
   else:
     template fail(msg: expr) =
       rollback()
-      if f.handle != 0:
+      if result.handle != 0: discard close(result.handle)
-        discard close(f.handle)
+      OSError()
-      # return false
-      raise newException(system.EIO, msg)
   
     var flags = if readonly: O_RDONLY else: O_RDWR
 
     if newFileSize != -1:
       flags = flags or O_CREAT or O_TRUNC
 
-    f.handle = open(filename, flags)
+    result.handle = open(filename, flags)
-    if f.handle == -1:
+    if result.handle == -1:
       # XXX: errno is supposed to be set here
       # Is there an exception that wraps it?
       fail "error opening file"
 
     if newFileSize != -1:
-      if ftruncate(f.handle, newFileSize) == -1:
+      if ftruncate(result.handle, newFileSize) == -1:
         fail "error setting file size"
 
     if mappedSize != -1:
-      f.size = mappedSize
+      result.size = mappedSize
     else:
       var stat: Tstat
-      if fstat(f.handle, stat) != -1:
+      if fstat(result.handle, stat) != -1:
         # XXX: Hmm, this could be unsafe
         # Why is mmap taking int anyway?
-        f.size = int(stat.st_size)
+        result.size = int(stat.st_size)
       else:
         fail "error getting file size"
 
-    f.mem = mmap(
+    result.mem = mmap(
       nil,
-      f.size,
+      result.size,
       if readonly: PROT_READ else: PROT_READ or PROT_WRITE,
       if readonly: MAP_PRIVATE else: MAP_SHARED,
-      f.handle,
+      result.handle,
       offset)
 
-    if f.mem == cast[pointer](MAP_FAILED):
+    if result.mem == cast[pointer](MAP_FAILED):
       fail "file mapping failed"
 
-    result = true
-
 proc close*(f: var TMemFile) =
   ## closes the memory mapped file `f`. All changes are written back to the
   ## file system, if `f` was opened with write access.
@@ -189,6 +177,5 @@ proc close*(f: var TMemFile) =
   else:
     f.handle = 0
   
-  if error:
+  if error: OSError()
-    raise newException(system.EIO, "error closing file")
 

lib/pure/unittest.nim

@@ -7,15 +7,14 @@
 #    distribution, for details about the copyright.
 #
 
+## :Author: Zahary Karadjov (zah@github)
+##
 ## This module implements the standard unit testing facilities such as
 ## suites, fixtures and test cases as well as facilities for combinatorial 
 ## and randomzied test case generation (not yet available) 
 ## and object mocking (not yet available)
 ##
 ## It is loosely based on C++'s boost.test and Haskell's QuickTest
-##
-## Maintainer: Zahary Karadjov (zah@github)
-##
 
 import
   macros, terminal

lib/windows/winlean.nim

@@ -12,6 +12,7 @@
 
 type
   THandle* = int
+  LONG* = int
   WINBOOL* = int32
   DWORD* = int32
 
@@ -65,6 +66,7 @@ const
   DETACHED_PROCESS* = 8'i32
   
   SW_SHOWNORMAL* = 1'i32
+  INVALID_HANDLE_VALUE* = THANDLE(-1)
 
 proc CloseHandle*(hObject: THANDLE): WINBOOL {.stdcall, dynlib: "kernel32",
     importc: "CloseHandle".}
@@ -143,6 +145,7 @@ const
   FILE_ATTRIBUTE_HIDDEN* = 2'i32
   FILE_ATTRIBUTE_READONLY* = 1'i32
   FILE_ATTRIBUTE_SYSTEM* = 4'i32
+  FILE_ATTRIBUTE_TEMPORARY* = 256'i32
 
   MAX_PATH* = 260
 type
@@ -239,7 +242,8 @@ type
   TInAddr* {.pure, final, importc: "IN_ADDR", header: "Winsock2.h".} = object
     s_addr*: int32  # IP address
   
-  Tsockaddr_in* {.pure, final, importc: "SOCKADDR_IN", header: "Winsock2.h".} = object
+  Tsockaddr_in* {.pure, final, importc: "SOCKADDR_IN", 
+                  header: "Winsock2.h".} = object
     sin_family*: int16
     sin_port*: int16 # unsigned
     sin_addr*: TInAddr
@@ -248,7 +252,8 @@ type
   Tin6_addr* {.pure, final, importc: "IN6_ADDR", header: "Winsock2.h".} = object 
     bytes*: array[0..15, char]
 
-  Tsockaddr_in6* {.pure, final, importc: "SOCKADDR_IN6", header: "Winsock2.h".} = object
+  Tsockaddr_in6* {.pure, final, importc: "SOCKADDR_IN6", 
+                   header: "Winsock2.h".} = object
     sin6_family*: int16
     sin6_port*: int16 # unsigned
     sin6_flowinfo*: int32 # unsigned
@@ -397,3 +402,54 @@ type
 proc WaitForMultipleObjects*(nCount: DWORD, lpHandles: PWOHandleArray,
                              bWaitAll: WINBOOL, dwMilliseconds: DWORD): DWORD{.
     stdcall, dynlib: "kernel32", importc: "WaitForMultipleObjects".}
+    
+    
+# for memfiles.nim:
+
+const
+  GENERIC_READ* = 0x80000000'i32
+  GENERIC_ALL* = 0x10000000'i32
+  FILE_SHARE_READ* = 1'i32
+  CREATE_ALWAYS* = 2'i32
+  OPEN_EXISTING* = 3'i32
+  FILE_BEGIN* = 0'i32
+  INVALID_SET_FILE_POINTER* = -1'i32
+  NO_ERROR* = 0'i32
+  PAGE_READONLY* = 2'i32
+  PAGE_READWRITE* = 4'i32
+  FILE_MAP_READ* = 4'i32
+  FILE_MAP_WRITE* = 2'i32
+  INVALID_FILE_SIZE* = -1'i32
+
+proc CreateFileA*(lpFileName: cstring, dwDesiredAccess, dwShareMode: DWORD,
+                  lpSecurityAttributes: pointer,
+                  dwCreationDisposition, dwFlagsAndAttributes: DWORD,
+                  hTemplateFile: THANDLE): THANDLE {.
+    stdcall, dynlib: "kernel32", importc: "CreateFileA".}
+
+proc SetEndOfFile*(hFile: THANDLE): WINBOOL {.stdcall, dynlib: "kernel32",
+    importc: "SetEndOfFile".}
+
+proc SetFilePointer*(hFile: THANDLE, lDistanceToMove: LONG,
+                     lpDistanceToMoveHigh: ptr LONG, 
+                     dwMoveMethod: DWORD): DWORD {.
+    stdcall, dynlib: "kernel32", importc: "SetFilePointer".}
+
+proc GetFileSize*(hFile: THANDLE, lpFileSizeHigh: ptr DWORD): DWORD{.stdcall,
+    dynlib: "kernel32", importc: "GetFileSize".}
+
+proc MapViewOfFileEx*(hFileMappingObject: THANDLE, dwDesiredAccess: DWORD,
+                      dwFileOffsetHigh, dwFileOffsetLow: DWORD,
+                      dwNumberOfBytesToMap: DWORD, 
+                      lpBaseAddress: pointer): pointer{.
+    stdcall, dynlib: "kernel32", importc: "MapViewOfFileEx".}
+
+proc CreateFileMapping*(hFile: THANDLE,
+                        lpFileMappingAttributes: pointer,
+                        flProtect, dwMaximumSizeHigh: DWORD,
+                        dwMaximumSizeLow: DWORD, lpName: cstring): THANDLE {.
+    stdcall, dynlib: "kernel32", importc: "CreateFileMappingA".}
+
+proc UnmapViewOfFile*(lpBaseAddress: pointer): WINBOOL {.stdcall,
+    dynlib: "kernel32", importc: "UnmapViewOfFile".}
+

web/news.txt

@@ -106,6 +106,7 @@ Library Additions
 - Added ``locks`` core module for more flexible locking support.
 - Added ``irc`` module.
 - Added ``ftpclient`` module.
+- Added ``memfiles`` module.
 
 
 2011-07-10 Version 0.8.12 released

web/nimrod.ini

@@ -42,7 +42,7 @@ srcdoc: "impure/rdstdin;wrappers/zmq;wrappers/sphinx"
 srcdoc: "pure/collections/tables;pure/collections/sets;pure/collections/lists"
 srcdoc: "pure/collections/intsets;pure/collections/queues;pure/encodings"
 srcdoc: "pure/events;pure/collections/sequtils;pure/irc;ecmas/dom"
-srcdoc: "pure/ftpclient"
+srcdoc: "pure/ftpclient;pure/memfiles"
 
 webdoc: "wrappers/libcurl;pure/md5;wrappers/mysql;wrappers/iup"
 webdoc: "wrappers/sqlite3;wrappers/postgres;wrappers/tinyc"