Added fsmonitor module.

doc/lib.txt

@@ -160,6 +160,9 @@ Generic Operating System Services
   This module provides support for memory mapped files (Posix's ``mmap``) 
   on the different operating systems.
 
+* `fsmonitor <fsmonitor.html>`_
+  This module implements the ability to monitor a directory/file for changes
+  using Posix's inotify API.
 
 Math libraries
 --------------

lib/posix/inotify.nim

@@ -0,0 +1,70 @@
+#
+#
+#            Nimrod's Runtime Library
+#        (c) Copyright 2012 Dominik Picheta
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+# Get the platform-dependent flags.  
+# Structure describing an inotify event.  
+type 
+  Tinotify_event*{.pure, final, importc: "struct inotify_event", 
+                   header: "<sys/inotify.h>".} = object 
+    wd*{.importc: "wd".}: cint # Watch descriptor.  
+    mask*{.importc: "mask".}: uint32 # Watch mask.  
+    cookie*{.importc: "cookie".}: uint32 # Cookie to synchronize two events.  
+    len*{.importc: "len".}: uint32 # Length (including NULs) of name.  
+    name*{.importc: "name".}: char # Name.  
+    
+# Supported events suitable for MASK parameter of INOTIFY_ADD_WATCH.  
+const 
+  IN_ACCESS* = 0x00000001   # File was accessed.  
+  IN_MODIFY* = 0x00000002   # File was modified.  
+  IN_ATTRIB* = 0x00000004   # Metadata changed.  
+  IN_CLOSE_WRITE* = 0x00000008 # Writtable file was closed.  
+  IN_CLOSE_NOWRITE* = 0x00000010 # Unwrittable file closed.  
+  IN_CLOSE* = (IN_CLOSE_WRITE or IN_CLOSE_NOWRITE) # Close.  
+  IN_OPEN* = 0x00000020     # File was opened.  
+  IN_MOVED_FROM* = 0x00000040 # File was moved from X.  
+  IN_MOVED_TO* = 0x00000080 # File was moved to Y.  
+  IN_MOVE* = (IN_MOVED_FROM or IN_MOVED_TO) # Moves.  
+  IN_CREATE* = 0x00000100   # Subfile was created.  
+  IN_DELETE* = 0x00000200   # Subfile was deleted.  
+  IN_DELETE_SELF* = 0x00000400 # Self was deleted.  
+  IN_MOVE_SELF* = 0x00000800 # Self was moved.  
+# Events sent by the kernel.  
+const 
+  IN_UNMOUNT* = 0x00002000  # Backing fs was unmounted.  
+  IN_Q_OVERFLOW* = 0x00004000 # Event queued overflowed.  
+  IN_IGNORED* = 0x00008000  # File was ignored.   
+# Special flags.  
+const 
+  IN_ONLYDIR* = 0x01000000  # Only watch the path if it is a
+                            #        directory.  
+  IN_DONT_FOLLOW* = 0x02000000 # Do not follow a sym link.  
+  IN_EXCL_UNLINK* = 0x04000000 # Exclude events on unlinked
+                               #        objects.  
+  IN_MASK_ADD* = 0x20000000 # Add to the mask of an already
+                            #        existing watch.  
+  IN_ISDIR* = 0x40000000    # Event occurred against dir.  
+  IN_ONESHOT* = 0x80000000  # Only send event once.  
+# All events which a program can wait on.  
+const 
+  IN_ALL_EVENTS* = (IN_ACCESS or IN_MODIFY or IN_ATTRIB or IN_CLOSE_WRITE or
+      IN_CLOSE_NOWRITE or IN_OPEN or IN_MOVED_FROM or IN_MOVED_TO or
+      IN_CREATE or IN_DELETE or IN_DELETE_SELF or IN_MOVE_SELF)
+# Create and initialize inotify instance.
+proc inotify_init*(): cint{.cdecl, importc: "inotify_init", 
+                            header: "<sys/inotify.h>".}
+# Create and initialize inotify instance.  
+proc inotify_init1*(flags: cint): cint{.cdecl, importc: "inotify_init1", 
+    header: "<sys/inotify.h>".}
+# Add watch of object NAME to inotify instance FD.  Notify about
+#   events specified by MASK.  
+proc inotify_add_watch*(fd: cint; name: cstring; mask: uint32): cint{.
+    cdecl, importc: "inotify_add_watch", header: "<sys/inotify.h>".}
+# Remove the watch specified by WD from the inotify instance FD.  
+proc inotify_rm_watch*(fd: cint; wd: cint): cint{.cdecl, 
+    importc: "inotify_rm_watch", header: "<sys/inotify.h>".}

lib/pure/asyncio.nim

@@ -76,8 +76,8 @@ else:
   from posix import TTimeVal, TFdSet, FD_ZERO, FD_SET, FD_ISSET, select
 
 type
-  TDelegate = object
+  TDelegate* = object
-    fd: cint
+    fd*: cint
     deleVal*: PObject
 
     handleRead*: proc (h: PObject) {.nimcall.}
@@ -398,21 +398,17 @@ proc select(readfds, writefds, exceptfds: var seq[PDelegate],
   pruneSocketSet(exceptfds, (ex))
 
 proc poll*(d: PDispatcher, timeout: int = 500): bool =
-  ## This function checks for events on all the sockets in the `PDispatcher`.
+  ## This function checks for events on all the delegates in the `PDispatcher`.
   ## It then proceeds to call the correct event handler.
   ##
-  ## **Note:** There is no event which signifes when you have been disconnected,
+  ## This function returns ``True`` if there are file descriptors that are still 
-  ## it is your job to check whether what you get from ``recv`` is ``""``.
+  ## open, otherwise ``False``. File descriptors that have been
-  ## If you have been disconnected, `d`'s ``getSocket`` function should report
-  ## this appropriately.
-  ##
-  ## This function returns ``True`` if there are sockets that are still 
-  ## connected (or connecting), otherwise ``False``. Sockets that have been
   ## closed are immediately removed from the dispatcher automatically.
   ##
   ## **Note:** Each delegate has a task associated with it. This gets called
-  ## after each select() call, if you make timeout ``-1`` the tasks will
+  ## after each select() call, if you set timeout to ``-1`` the tasks will
-  ## only be executed after one or more sockets becomes readable or writeable.
+  ## only be executed after one or more file descriptors becomes readable or
+  ## writeable.
   result = true
   var readDg, writeDg, errorDg: seq[PDelegate] = @[]
   var len = d.delegates.len
@@ -433,10 +429,17 @@ proc poll*(d: PDispatcher, timeout: int = 500): bool =
       dec len
   d.delegates.setLen(len)
   
+  var hasDataBufferedCount = 0
+  for d in d.delegates:
+    if d.hasDataBuffered(d.deleVal):
+      hasDataBufferedCount.inc()
+      d.handleRead(d.deleVal)
+  if hasDataBufferedCount > 0: return True
+  
   if readDg.len() == 0 and writeDg.len() == 0:
     ## TODO: Perhaps this shouldn't return if errorDg has something?
     return False
-  # TODO: Buffering hasDataBuffered!!
+  
   if select(readDg, writeDg, errorDg, timeout) != 0:
     for i in 0..len(d.delegates)-1:
       if i > len(d.delegates)-1: break # One delegate might've been removed.
@@ -453,63 +456,6 @@ proc poll*(d: PDispatcher, timeout: int = 500): bool =
   for i in items(d.delegates):
     i.task(i.deleVal)
 
-  discard """result = true
-  var readSocks, writeSocks: seq[TSocket] = @[]
-  
-  var L = d.delegates.len
-  var dc = 0
-  while dc < L:
-    template deleg: expr = d.delegates[dc]
-    let aSock = deleg.getSocket(deleg.deleVal)
-    if (deleg.mode != MWriteable and aSock.info == SockConnected) or
-          aSock.info == SockListening or aSock.info == SockUDPBound:
-      readSocks.add(aSock.sock)
-    if aSock.info == SockConnecting or
-        (aSock.info == SockConnected and deleg.mode != MReadable):
-      writeSocks.add(aSock.sock)
-    if aSock.info == SockClosed:
-      # Socket has been closed remove it from the dispatcher.
-      d.delegates[dc] = d.delegates[L-1]
-      
-      dec L
-    else: inc dc
-  d.delegates.setLen(L)
-  
-  if readSocks.len() == 0 and writeSocks.len() == 0:
-    return False
-
-  if select(readSocks, writeSocks, timeout) != 0:
-    for i in 0..len(d.delegates)-1:
-      if i > len(d.delegates)-1: break # One delegate might've been removed.
-      let deleg = d.delegates[i]
-      let sock = deleg.getSocket(deleg.deleVal)
-      if sock.info == SockConnected or 
-         sock.info == SockUDPBound:
-        if deleg.mode != MWriteable and sock.sock notin readSocks:
-          if not (sock.info == SockConnecting):
-            assert(not (sock.info == SockListening))
-            deleg.handleRead(deleg.deleVal)
-          else:
-            assert(false)
-        if deleg.mode != MReadable and sock.sock notin writeSocks:
-          deleg.handleWrite(deleg.deleVal)
-      
-      if sock.info == SockListening:
-        if sock.sock notin readSocks:
-          # This is a server socket, that had listen() called on it.
-          # This socket should have a client waiting now.
-          deleg.handleAccept(deleg.deleVal)
-      
-      if sock.info == SockConnecting:
-        # Checking whether the socket has connected this way should work on
-        # Windows and Posix. I've checked. 
-        if sock.sock notin writeSocks:
-          deleg.handleConnect(deleg.deleVal)
-  
-  # Execute tasks
-  for i in items(d.delegates):
-    i.task(i.deleVal)"""
-
 proc len*(disp: PDispatcher): int =
   ## Retrieves the amount of delegates in ``disp``.
   return disp.delegates.len

lib/pure/fsmonitor.nim

@@ -0,0 +1,216 @@
+#
+#
+#            Nimrod's Runtime Library
+#        (c) Copyright 2012 Dominik Picheta
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+## This module allows you to monitor files or directories for changes using
+## asyncio.
+##
+## Windows support is not yet implemented.
+##
+## **Note:** This module uses ``inotify`` on Linux (Other Unixes are not yet
+## supported). ``inotify`` was merged into the 2.6.13 Linux kernel, this
+## module will therefore not work with any Linux kernel prior to that, unless
+## it has been patched to support inotify.
+
+when defined(windows):
+  {.error: "Windows is not yet supported by this module.".}
+elif defined(linux):
+  from posix import read
+else:
+  {.error: "Your platform is not supported.".}
+
+import inotify, os, asyncio, tables
+
+type
+  PFSMonitor* = ref TFSMonitor
+  TFSMonitor = object of TObject
+    fd: cint
+    handleEvent: proc (m: PFSMonitor, ev: TMonitorEvent) {.closure.}
+    targets: TTable[cint, string]
+  
+  TMonitorEventType* = enum ## Monitor event type
+    MonitorAccess,       ## File was accessed.
+    MonitorAttrib,       ## Metadata changed.
+    MonitorCloseWrite,   ## Writtable file was closed.
+    MonitorCloseNoWrite, ## Unwrittable file closed.
+    MonitorCreate,       ## Subfile was created.
+    MonitorDelete,       ## Subfile was deleted.
+    MonitorDeleteSelf,   ## Watched file/directory was itself deleted.
+    MonitorModify,       ## File was modified.
+    MonitorMoveSelf,     ## Self was moved.
+    MonitorMoved,        ## File was moved.
+    MonitorOpen,         ## File was opened.
+    MonitorAll           ## Filter for all event types.
+  
+  TMonitorEvent* = object
+    case kind*: TMonitorEventType  ## Type of the event.
+    of MonitorMoveSelf, MonitorMoved:
+      oldPath*: string          ## Old absolute location
+      newPath*: string          ## New absolute location
+    else:
+      fullname*: string         ## Absolute filename of the file/directory affected.
+    name*: string             ## Non absolute filepath of the file/directory
+                              ## affected relative to the directory watched.
+                              ## "" if this event refers to the file/directory
+                              ## watched.
+    wd*: cint                 ## Watch descriptor.
+
+const
+  MaxEvents = 100
+
+proc newMonitor*(): PFSMonitor =
+  ## Creates a new file system monitor.
+  new(result)
+  result.fd = inotifyInit()
+  result.targets = initTable[cint, string]()
+  if result.fd < 0:
+    OSError()
+
+proc add*(monitor: PFSMonitor, target: string,
+               filters = {MonitorAll}): cint {.discardable.} =
+  ## Adds ``target`` which may be a directory or a file to the list of
+  ## watched paths of ``monitor``.
+  ## You can specify the events to report using the ``filters`` parameter.
+  
+  var INFilter = -1
+  for f in filters:
+    case f
+    of MonitorAccess: INFilter = INFilter and IN_ACCESS
+    of MonitorAttrib: INFilter = INFilter and IN_ATTRIB
+    of MonitorCloseWrite: INFilter = INFilter and IN_CLOSE_WRITE
+    of MonitorCloseNoWrite: INFilter = INFilter and IN_CLOSE_NO_WRITE
+    of MonitorCreate: INFilter = INFilter and IN_CREATE
+    of MonitorDelete: INFilter = INFilter and IN_DELETE
+    of MonitorDeleteSelf: INFilter = INFilter and IN_DELETE_SELF
+    of MonitorModify: INFilter = INFilter and IN_MODIFY
+    of MonitorMoveSelf: INFilter = INFilter and IN_MOVE_SELF
+    of MonitorMoved: INFilter = INFilter and IN_MOVED_FROM and IN_MOVED_TO
+    of MonitorOpen: INFilter = INFilter and IN_OPEN
+    of MonitorAll: INFilter = INFilter and IN_ALL_EVENTS
+  
+  result = inotifyAddWatch(monitor.fd, target, INFilter.uint32)
+  if result < 0:
+    OSError()
+  monitor.targets.add(result, target)
+
+proc del*(monitor: PFSMonitor, wd: cint) =
+  ## Removes watched directory or file as specified by ``wd`` from ``monitor``.
+  ##
+  ## If ``wd`` is not a part of ``monitor`` an EOS error is raised.
+  if inotifyRmWatch(monitor.fd, wd) < 0:
+    OSError()
+
+proc getEvent(m: PFSMonitor, fd: cint): seq[TMonitorEvent] =
+  result = @[]
+  let size = (sizeof(TINotifyEvent)+2000)*MaxEvents
+  var buffer = newString(size)
+
+  let le = read(fd, addr(buffer[0]), size)
+
+  var movedFrom: TTable[cint, tuple[wd: cint, old: string]] = 
+            initTable[cint, tuple[wd: cint, old: string]]()
+
+  var i = 0
+  while i < le:
+    var event = cast[ptr TINotifyEvent](addr(buffer[i]))
+    var mev: TMonitorEvent
+    mev.wd = event.wd
+    if event.len.int != 0:
+      mev.name = newString(event.len.int)
+      copyMem(addr(mev.name[0]), addr event.name, event.len.int-1)
+    else:
+      mev.name = ""
+    
+    if (event.mask.int and IN_MOVED_FROM) != 0: 
+      # Moved from event, add to m's collection
+      movedFrom.add(event.cookie.cint, (mev.wd, mev.name))
+      inc(i, sizeof(TINotifyEvent) + event.len.int)
+      continue
+    elif (event.mask.int and IN_MOVED_TO) != 0: 
+      mev.kind = MonitorMoved
+      assert movedFrom.hasKey(event.cookie.cint)
+      # Find the MovedFrom event.
+      mev.oldPath = movedFrom[event.cookie.cint].old
+      mev.newPath = "" # Set later
+      # Delete it from the TTable
+      movedFrom.del(event.cookie.cint)
+    elif (event.mask.int and IN_ACCESS) != 0: mev.kind = MonitorAccess
+    elif (event.mask.int and IN_ATTRIB) != 0: mev.kind = MonitorAttrib
+    elif (event.mask.int and IN_CLOSE_WRITE) != 0: 
+      mev.kind = MonitorCloseWrite
+    elif (event.mask.int and IN_CLOSE_NOWRITE) != 0: 
+      mev.kind = MonitorCloseNoWrite
+    elif (event.mask.int and IN_CREATE) != 0: mev.kind = MonitorCreate
+    elif (event.mask.int and IN_DELETE) != 0: 
+      mev.kind = MonitorDelete
+    elif (event.mask.int and IN_DELETE_SELF) != 0: 
+      mev.kind = MonitorDeleteSelf
+    elif (event.mask.int and IN_MODIFY) != 0: mev.kind = MonitorModify
+    elif (event.mask.int and IN_MOVE_SELF) != 0: 
+      mev.kind = MonitorMoveSelf
+    elif (event.mask.int and IN_OPEN) != 0: mev.kind = MonitorOpen
+    
+    if mev.kind != MonitorMoved:
+      mev.fullname = ""
+    
+    result.add(mev)
+    inc(i, sizeof(TINotifyEvent) + event.len.int)
+
+  # If movedFrom events have not been matched with a moveTo. File has
+  # been moved to an unwatched location, emit a MonitorDelete.
+  for cookie, t in pairs(movedFrom):
+    var mev: TMonitorEvent
+    mev.kind = MonitorDelete
+    mev.wd = t.wd
+    mev.name = t.old
+    result.add(mev)
+
+proc FSMonitorRead(h: PObject) =
+  var events = PFSMonitor(h).getEvent(PFSMonitor(h).fd)
+  #var newEv: TMonitorEvent
+  for ev in events:
+    var target = PFSMonitor(h).targets[ev.wd]
+    var newEv = ev
+    if newEv.kind == MonitorMoved:
+      newEv.oldPath = target / newEv.oldPath
+      newEv.newPath = target / newEv.name
+    else:
+      newEv.fullName = target / newEv.name
+    PFSMonitor(h).handleEvent(PFSMonitor(h), newEv)
+
+proc toDelegate(m: PFSMonitor): PDelegate =
+  result = newDelegate()
+  result.deleVal = m
+  result.fd = m.fd
+  result.mode = fmRead
+  result.handleRead = FSMonitorRead
+  result.open = true
+
+proc register*(d: PDispatcher, monitor: PFSMonitor,
+               handleEvent: proc (m: PFSMonitor, ev: TMonitorEvent) {.closure.}) =
+  ## Registers ``monitor`` with dispatcher ``d``.
+  monitor.handleEvent = handleEvent
+  var deleg = toDelegate(monitor)
+  d.register(deleg)
+
+when isMainModule:
+  var disp = newDispatcher()
+  var monitor = newMonitor()
+  echo monitor.add("/home/dom/inotifytests/")
+  disp.register(monitor,
+    proc (m: PFSMonitor, ev: TMonitorEvent) =
+      echo("Got event: ", ev.kind)
+      if ev.kind == MonitorMoved:
+        echo("From ", ev.oldPath, " to ", ev.newPath)
+        echo("Name is ", ev.name)
+      else:
+        echo("Name ", ev.name, " fullname ", ev.fullName))
+      
+  while true:
+    if not disp.poll(): break
+  

web/news.txt

@@ -65,7 +65,7 @@ Library Additions
   some operation.
 - Added ``strutils.format``, ``subexes.format`` which use the 
   new ``varargs`` type.
-
+- Added module ``fsmonitor``.
 
 Changes affecting backwards compatibility
 -----------------------------------------

web/nimrod.ini

@@ -34,7 +34,7 @@ srcdoc: "pure/parsecfg;pure/parsexml;pure/parsecsv;pure/parsesql"
 srcdoc: "pure/streams;pure/terminal;pure/cgi;impure/web;pure/unicode"
 srcdoc: "impure/zipfiles;pure/htmlgen;pure/parseutils;pure/browsers"
 srcdoc: "impure/db_postgres;impure/db_mysql;impure/db_sqlite;impure/db_mongo"
-srcdoc: "pure/httpserver;pure/httpclient;pure/smtp;impure/ssl"
+srcdoc: "pure/httpserver;pure/httpclient;pure/smtp;impure/ssl;pure/fsmonitor"
 srcdoc: "pure/ropes;pure/unidecode/unidecode;pure/xmldom;pure/xmldomparser"
 srcdoc: "pure/xmlparser;pure/htmlparser;pure/xmltree;pure/colors;pure/mimetypes"
 srcdoc: "pure/json;pure/base64;pure/scgi;pure/redis;impure/graphics"