Add MemMapFileStream. Fixes in memFiles. (#7944)
* Add MemMapFileStream * Added tests * Fixed bug in memfiles (zero index for string) * Added flush to changelog * Attempt to fix Win's nuances * Fix attempt to fix * Continue... * And again... * Reworked tests (all for win on Win) * Fixes in flush (Win) * Replace fn vars to consts * Added the attempts parameter to the flush * Replace while to for * Move to memfiles * Use Natural instead of uint * Better error messages for append mode. Handle specific cases.
This commit is contained in:
Dmitry Atamanov
Dominik Picheta
parent
f1d5e9090e
commit
bf5d619a52
5 changed files with 195 additions and 23 deletions
|
|
@ -22,7 +22,11 @@ elif defined(posix):
|
|||
else:
|
||||
{.error: "the memfiles module is not supported on your operating system!".}
|
||||
|
||||
import os
|
||||
import os, streams
|
||||
|
||||
proc newEIO(msg: string): ref IOError =
|
||||
new(result)
|
||||
result.msg = msg
|
||||
|
||||
type
|
||||
MemFile* = object ## represents a memory mapped file
|
||||
|
|
@ -44,11 +48,14 @@ proc mapMem*(m: var MemFile, mode: FileMode = fmRead,
|
|||
##
|
||||
## ``mappedSize`` of ``-1`` maps to the whole file, and
|
||||
## ``offset`` must be multiples of the PAGE SIZE of your OS
|
||||
if mode == fmAppend:
|
||||
raise newEIO("The append mode is not supported.")
|
||||
|
||||
var readonly = mode == fmRead
|
||||
when defined(windows):
|
||||
result = mapViewOfFileEx(
|
||||
m.mapHandle,
|
||||
if readonly: FILE_MAP_READ else: FILE_MAP_WRITE,
|
||||
if readonly: FILE_MAP_READ else: FILE_MAP_READ or FILE_MAP_WRITE,
|
||||
int32(offset shr 32),
|
||||
int32(offset and 0xffffffff),
|
||||
if mappedSize == -1: 0 else: mappedSize,
|
||||
|
|
@ -113,6 +120,9 @@ proc open*(filename: string, mode: FileMode = fmRead,
|
|||
## mm_half = memfiles.open("/tmp/test.mmap", mode = fmReadWrite, mappedSize = 512)
|
||||
|
||||
# The file can be resized only when write mode is used:
|
||||
if mode == fmAppend:
|
||||
raise newEIO("The append mode is not supported.")
|
||||
|
||||
assert newFileSize == -1 or mode != fmRead
|
||||
var readonly = mode == fmRead
|
||||
|
||||
|
|
@ -121,6 +131,10 @@ proc open*(filename: string, mode: FileMode = fmRead,
|
|||
result.size = 0
|
||||
|
||||
when defined(windows):
|
||||
let desiredAccess = GENERIC_READ
|
||||
let shareMode = FILE_SHARE_READ
|
||||
let flags = FILE_FLAG_RANDOM_ACCESS
|
||||
|
||||
template fail(errCode: OSErrorCode, msg: untyped) =
|
||||
rollback()
|
||||
if result.fHandle != 0: discard closeHandle(result.fHandle)
|
||||
|
|
@ -133,11 +147,11 @@ proc open*(filename: string, mode: FileMode = fmRead,
|
|||
winApiProc(
|
||||
filename,
|
||||
# GENERIC_ALL != (GENERIC_READ or GENERIC_WRITE)
|
||||
if readonly: GENERIC_READ else: GENERIC_READ or GENERIC_WRITE,
|
||||
FILE_SHARE_READ,
|
||||
if readonly: desiredAccess else: desiredAccess or GENERIC_WRITE,
|
||||
if readonly: shareMode else: shareMode or FILE_SHARE_WRITE,
|
||||
nil,
|
||||
if newFileSize != -1: CREATE_ALWAYS else: OPEN_EXISTING,
|
||||
if readonly: FILE_ATTRIBUTE_READONLY else: FILE_ATTRIBUTE_TEMPORARY,
|
||||
if readonly: FILE_ATTRIBUTE_READONLY or flags else: FILE_ATTRIBUTE_NORMAL or flags,
|
||||
0)
|
||||
|
||||
when useWinUnicode:
|
||||
|
|
@ -172,7 +186,7 @@ proc open*(filename: string, mode: FileMode = fmRead,
|
|||
|
||||
result.mem = mapViewOfFileEx(
|
||||
result.mapHandle,
|
||||
if readonly: FILE_MAP_READ else: FILE_MAP_WRITE,
|
||||
if readonly: FILE_MAP_READ else: FILE_MAP_READ or FILE_MAP_WRITE,
|
||||
int32(offset shr 32),
|
||||
int32(offset and 0xffffffff),
|
||||
if mappedSize == -1: 0 else: mappedSize,
|
||||
|
|
@ -245,6 +259,28 @@ proc open*(filename: string, mode: FileMode = fmRead,
|
|||
if close(result.handle) == 0:
|
||||
result.handle = -1
|
||||
|
||||
proc flush*(f: var MemFile; attempts: Natural = 3) =
|
||||
## Flushes `f`'s buffer for the number of attempts equal to `attempts`.
|
||||
## If were errors an exception `OSError` will be raised.
|
||||
var res = false
|
||||
var lastErr: OSErrorCode
|
||||
when defined(windows):
|
||||
for i in 1..attempts:
|
||||
res = flushViewOfFile(f.mem, 0) != 0
|
||||
if res:
|
||||
break
|
||||
lastErr = osLastError()
|
||||
if lastErr != ERROR_LOCK_VIOLATION.OSErrorCode:
|
||||
raiseOSError(lastErr)
|
||||
else:
|
||||
for i in 1..attempts:
|
||||
res = msync(f.mem, f.size, MS_SYNC or MS_INVALIDATE) == 0
|
||||
if res:
|
||||
break
|
||||
lastErr = osLastError()
|
||||
if lastErr != EBUSY.OSErrorCode:
|
||||
raiseOSError(lastErr, "error flushing mapping")
|
||||
|
||||
proc close*(f: var MemFile) =
|
||||
## closes the memory mapped file `f`. All changes are written back to the
|
||||
## file system, if `f` was opened with write access.
|
||||
|
|
@ -362,9 +398,8 @@ iterator lines*(mfile: MemFile, buf: var TaintedString, delim='\l', eat='\r'): T
|
|||
## echo line
|
||||
|
||||
for ms in memSlices(mfile, delim, eat):
|
||||
buf.setLen(ms.size)
|
||||
copyMem(addr(buf[0]), ms.data, ms.size)
|
||||
buf[ms.size] = '\0'
|
||||
setLen(buf.string, ms.size)
|
||||
copyMem(buf.cstring, ms.data, ms.size)
|
||||
yield buf
|
||||
|
||||
iterator lines*(mfile: MemFile, delim='\l', eat='\r'): TaintedString {.inline.} =
|
||||
|
|
@ -382,3 +417,68 @@ iterator lines*(mfile: MemFile, delim='\l', eat='\r'): TaintedString {.inline.}
|
|||
var buf = TaintedString(newStringOfCap(80))
|
||||
for line in lines(mfile, buf, delim, eat):
|
||||
yield buf
|
||||
|
||||
type
|
||||
MemMapFileStream* = ref MemMapFileStreamObj ## a stream that encapsulates a `MemFile`
|
||||
MemMapFileStreamObj* = object of Stream
|
||||
mf: MemFile
|
||||
mode: FileMode
|
||||
pos: ByteAddress
|
||||
|
||||
proc mmsClose(s: Stream) =
|
||||
MemMapFileStream(s).pos = -1
|
||||
close(MemMapFileStream(s).mf)
|
||||
|
||||
proc mmsFlush(s: Stream) = flush(MemMapFileStream(s).mf)
|
||||
|
||||
proc mmsAtEnd(s: Stream): bool = (MemMapFileStream(s).pos >= MemMapFileStream(s).mf.size) or
|
||||
(MemMapFileStream(s).pos < 0)
|
||||
|
||||
proc mmsSetPosition(s: Stream, pos: int) =
|
||||
if pos > MemMapFileStream(s).mf.size or pos < 0:
|
||||
raise newEIO("cannot set pos in stream")
|
||||
MemMapFileStream(s).pos = pos
|
||||
|
||||
proc mmsGetPosition(s: Stream): int = MemMapFileStream(s).pos
|
||||
|
||||
proc mmsPeekData(s: Stream, buffer: pointer, bufLen: int): int =
|
||||
let startAddress = cast[ByteAddress](MemMapFileStream(s).mf.mem)
|
||||
let p = cast[ByteAddress](MemMapFileStream(s).pos)
|
||||
let l = min(bufLen, MemMapFileStream(s).mf.size - p)
|
||||
moveMem(buffer, cast[pointer](startAddress + p), l)
|
||||
result = l
|
||||
|
||||
proc mmsReadData(s: Stream, buffer: pointer, bufLen: int): int =
|
||||
result = mmsPeekData(s, buffer, bufLen)
|
||||
inc(MemMapFileStream(s).pos, result)
|
||||
|
||||
proc mmsWriteData(s: Stream, buffer: pointer, bufLen: int) =
|
||||
if MemMapFileStream(s).mode == fmRead:
|
||||
raise newEIO("cannot write to read-only stream")
|
||||
let size = MemMapFileStream(s).mf.size
|
||||
if MemMapFileStream(s).pos + bufLen > size:
|
||||
raise newEIO("cannot write to stream")
|
||||
let p = cast[ByteAddress](MemMapFileStream(s).mf.mem) +
|
||||
cast[ByteAddress](MemMapFileStream(s).pos)
|
||||
moveMem(cast[pointer](p), buffer, bufLen)
|
||||
inc(MemMapFileStream(s).pos, bufLen)
|
||||
|
||||
proc newMemMapFileStream*(filename: string, mode: FileMode = fmRead, fileSize: int = -1):
|
||||
MemMapFileStream =
|
||||
## creates a new stream from the file named `filename` with the mode `mode`.
|
||||
## Raises ## `EOS` if the file cannot be opened. See the `system
|
||||
## <system.html>`_ module for a list of available FileMode enums.
|
||||
## ``fileSize`` can only be set if the file does not exist and is opened
|
||||
## with write access (e.g., with fmReadWrite).
|
||||
var mf: MemFile = open(filename, mode, newFileSize = fileSize)
|
||||
new(result)
|
||||
result.mode = mode
|
||||
result.mf = mf
|
||||
result.closeImpl = mmsClose
|
||||
result.atEndImpl = mmsAtEnd
|
||||
result.setPositionImpl = mmsSetPosition
|
||||
result.getPositionImpl = mmsGetPosition
|
||||
result.readDataImpl = mmsReadData
|
||||
result.peekDataImpl = mmsPeekData
|
||||
result.writeDataImpl = mmsWriteData
|
||||
result.flushImpl = mmsFlush
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue