Implement Unix file regularity check (#20448) (#20628)

* Implement Unix file regularity check

* update std/dirs also

lib/pure/os.nim

@@ -1095,6 +1095,8 @@ type
     creationTime*: times.Time         ## Time file was created. Not supported on all systems!
     blockSize*: int                   ## Preferred I/O block size for this object.
                                       ## In some filesystems, this may vary from file to file.
+    isRegular*: bool                  ## Is file regular? (on Unix some "files"
+                                      ## can be non-regular like FIFOs, devices)
 
 template rawToFormalFileInfo(rawInfo, path, formalInfo): untyped =
   ## Transforms the native file info structure into the one nim uses.
@@ -1155,14 +1157,14 @@ template rawToFormalFileInfo(rawInfo, path, formalInfo): untyped =
     checkAndIncludeMode(S_IWOTH, fpOthersWrite)
     checkAndIncludeMode(S_IXOTH, fpOthersExec)
 
-    formalInfo.kind =
+    (formalInfo.kind, formalInfo.isRegular) =
       if S_ISDIR(rawInfo.st_mode):
-        pcDir
+        (pcDir, true)
       elif S_ISLNK(rawInfo.st_mode):
         assert(path != "") # symlinks can't occur for file handles
         getSymlinkFileKind(path)
       else:
-        pcFile
+        (pcFile, S_ISREG(rawInfo.st_mode))
 
 when defined(js):
   when not declared(FileHandle):
@@ -1215,7 +1217,8 @@ proc getFileInfo*(path: string, followSymlink = true): FileInfo {.noWeirdTarget.
   ##
   ## When `followSymlink` is true (default), symlinks are followed and the
   ## information retrieved is information related to the symlink's target.
-  ## Otherwise, information on the symlink itself is retrieved.
+  ## Otherwise, information on the symlink itself is retrieved (however,
+  ## field `isRegular` is still determined from the target on Unix).
   ##
   ## If the information cannot be retrieved, such as when the path doesn't
   ## exist, or when permission restrictions prevent the program from retrieving

lib/std/dirs.nim

@@ -121,30 +121,33 @@ iterator walkDirs*(pattern: Path): Path {.tags: [ReadDirEffect].} =
   for p in walkDirs(pattern.string):
     yield Path(p)
 
-iterator walkDir*(dir: Path; relative = false, checkDir = false):
+iterator walkDir*(dir: Path; relative = false, checkDir = false,
+                 onlyRegular = false):
     tuple[kind: PathComponent, path: Path] {.tags: [ReadDirEffect].} =
   ## Walks over the directory `dir` and yields for each directory or file in
   ## `dir`. The component type and full path for each item are returned.
   ##
-  ## Walking is not recursive. If ``relative`` is true (default: false)
+  ## Walking is not recursive.
-  ## the resulting path is shortened to be relative to ``dir``.
+  ## * If `relative` is true (default: false)
-  ##
+  ##   the resulting path is shortened to be relative to ``dir``,
-  ## If `checkDir` is true, `OSError` is raised when `dir`
+  ##   otherwise the full path is returned.
+  ## * If `checkDir` is true, `OSError` is raised when `dir`
   ##   doesn't exist.
-  for (k, p) in walkDir(dir.string, relative, checkDir):
+  ## * If `onlyRegular` is true, then (besides all directories) only *regular*
+  ##   files (**without** special "file" objects like FIFOs, device files,
+  ##   etc) will be yielded on Unix.
+  for (k, p) in walkDir(dir.string, relative, checkDir, onlyRegular):
     yield (k, Path(p))
 
 iterator walkDirRec*(dir: Path,
                      yieldFilter = {pcFile}, followFilter = {pcDir},
-                     relative = false, checkDir = false): Path {.tags: [ReadDirEffect].} =
+                     relative = false, checkDir = false, onlyRegular = false):
+                    Path {.tags: [ReadDirEffect].} =
   ## Recursively walks over the directory `dir` and yields for each file
   ## or directory in `dir`.
   ##
-  ## If ``relative`` is true (default: false) the resulting path is
+  ## Options `relative`, `checkdir`, `onlyRegular` are explained in
-  ## shortened to be relative to ``dir``, otherwise the full path is returned.
+  ## [walkDir iterator] description.
-  ##
-  ## If `checkDir` is true, `OSError` is raised when `dir`
-  ## doesn't exist.
   ##
   ## .. warning:: Modifying the directory structure while the iterator
   ##   is traversing may result in undefined behavior!
@@ -173,5 +176,6 @@ iterator walkDirRec*(dir: Path,
   ## * `walkFiles iterator`_
   ## * `walkDirs iterator`_
   ## * `walkDir iterator`_
-  for p in walkDirRec(dir.string, yieldFilter, followFilter, relative, checkDir):
+  for p in walkDirRec(dir.string, yieldFilter, followFilter, relative,
+                      checkDir, onlyRegular):
     yield Path(p)

lib/std/private/oscommon.nim

@@ -77,14 +77,17 @@ type
 
 
 when defined(posix) and not weirdTarget:
-  proc getSymlinkFileKind*(path: string): PathComponent =
+  proc getSymlinkFileKind*(path: string):
+      tuple[pc: PathComponent, isRegular: bool] =
     # Helper function.
     var s: Stat
     assert(path != "")
-    if stat(path, s) == 0'i32 and S_ISDIR(s.st_mode):
+    result = (pcLinkToFile, true)
-      result = pcLinkToDir
+    if stat(path, s) == 0'i32:
-    else:
+      if S_ISDIR(s.st_mode):
-      result = pcLinkToFile
+        result = (pcLinkToDir, true)
+      elif not S_ISREG(s.st_mode):
+        result = (pcLinkToFile, false)
 
 proc tryMoveFSObject*(source, dest: string, isDir: bool): bool {.noWeirdTarget.} =
   ## Moves a file (or directory if `isDir` is true) from `source` to `dest`.

lib/std/private/osdirs.nim

@@ -154,16 +154,21 @@ proc staticWalkDir(dir: string; relative: bool): seq[
                   tuple[kind: PathComponent, path: string]] =
   discard
 
-iterator walkDir*(dir: string; relative = false, checkDir = false):
+iterator walkDir*(dir: string; relative = false, checkDir = false,
+                  onlyRegular = false):
   tuple[kind: PathComponent, path: string] {.tags: [ReadDirEffect].} =
   ## Walks over the directory `dir` and yields for each directory or file in
   ## `dir`. The component type and full path for each item are returned.
   ##
-  ## Walking is not recursive. If ``relative`` is true (default: false)
+  ## Walking is not recursive.
-  ## the resulting path is shortened to be relative to ``dir``.
+  ## * If `relative` is true (default: false)
-  ##
+  ##   the resulting path is shortened to be relative to ``dir``,
-  ## If `checkDir` is true, `OSError` is raised when `dir`
+  ##   otherwise the full path is returned.
+  ## * If `checkDir` is true, `OSError` is raised when `dir`
   ##   doesn't exist.
+  ## * If `onlyRegular` is true, then (besides all directories) only *regular*
+  ##   files (**without** special "file" objects like FIFOs, device files,
+  ##   etc) will be yielded on Unix.
   ##
   ## **Example:**
   ##
@@ -234,24 +239,30 @@ iterator walkDir*(dir: string; relative = false, checkDir = false):
               y = path
             var k = pcFile
 
+            template resolveSymlink() =
+              var isRegular: bool
+              (k, isRegular) = getSymlinkFileKind(path)
+              if onlyRegular and not isRegular: continue
+
             template kSetGeneric() =  # pure Posix component `k` resolution
               if lstat(path.cstring, s) < 0'i32: continue  # don't yield
               elif S_ISDIR(s.st_mode):
                 k = pcDir
               elif S_ISLNK(s.st_mode):
-                k = getSymlinkFileKind(path)
+                resolveSymlink()
+              elif onlyRegular and not S_ISREG(s.st_mode): continue
 
             when defined(linux) or defined(macosx) or
                  defined(bsd) or defined(genode) or defined(nintendoswitch):
               case x.d_type
               of DT_DIR: k = pcDir
               of DT_LNK:
-                if dirExists(path): k = pcLinkToDir
+                resolveSymlink()
-                else: k = pcLinkToFile
               of DT_UNKNOWN:
                 kSetGeneric()
-              else: # e.g. DT_REG etc
+              else: # DT_REG or special "files" like FIFOs
-                discard # leave it as pcFile
+                if onlyRegular and x.d_type != DT_REG: continue
+                else: discard # leave it as pcFile
             else:  # assuming that field `d_type` is not present
               kSetGeneric()
 
@@ -259,15 +270,13 @@ iterator walkDir*(dir: string; relative = false, checkDir = false):
 
 iterator walkDirRec*(dir: string,
                      yieldFilter = {pcFile}, followFilter = {pcDir},
-                     relative = false, checkDir = false): string {.tags: [ReadDirEffect].} =
+                     relative = false, checkDir = false, onlyRegular = false):
+                    string {.tags: [ReadDirEffect].} =
   ## Recursively walks over the directory `dir` and yields for each file
   ## or directory in `dir`.
   ##
-  ## If ``relative`` is true (default: false) the resulting path is
+  ## Options `relative`, `checkdir`, `onlyRegular` are explained in
-  ## shortened to be relative to ``dir``, otherwise the full path is returned.
+  ## [walkDir iterator] description.
-  ##
-  ## If `checkDir` is true, `OSError` is raised when `dir`
-  ## doesn't exist.
   ##
   ## .. warning:: Modifying the directory structure while the iterator
   ##   is traversing may result in undefined behavior!
@@ -301,7 +310,8 @@ iterator walkDirRec*(dir: string,
   var checkDir = checkDir
   while stack.len > 0:
     let d = stack.pop()
-    for k, p in walkDir(dir / d, relative = true, checkDir = checkDir):
+    for k, p in walkDir(dir / d, relative = true, checkDir = checkDir,
+                        onlyRegular = onlyRegular):
       let rel = d / p
       if k in {pcDir, pcLinkToDir} and k in followFilter:
         stack.add rel

tests/stdlib/tgetfileinfo.nim

@@ -4,7 +4,7 @@ discard """
 """
 
 import os, strutils
-import std/syncio
+import std/[syncio, assertions]
 # Cases
 #  1 - String : Existing File : Symlink true
 #  2 - String : Existing File : Symlink false
@@ -127,10 +127,37 @@ proc testGetFileInfo =
       echo pcLinkToDir
       echo pcLinkToFile
 
+    doAssert dirInfo.isRegular == true
+    doAssert fileInfo.isRegular == true
+    when defined(posix):
+      doAssert linkDirInfo.isRegular == true
+      doAssert linkFileInfo.isRegular == true
+
     removeDir(dirPath)
     removeFile(filePath)
     when defined(posix):
       removeFile(linkDirPath)
       removeFile(linkFilePath)
 
+  # Test that `isRegular` is set correctly
+  block:
+    when defined(posix):
+      let
+        tmp = getTempDir()
+        fifoPath     = tmp / "test-fifo"
+        linkFifoPath = tmp / "test-link-fifo"
+
+      doAssert execShellCmd("mkfifo " & fifoPath) == 0
+      createSymlink(fifoPath, linkFifoPath)
+
+      let
+        fifoInfo = getFileInfo(fifoPath)
+        linkFifoInfo = getFileInfo(linkFifoPath)
+
+      doAssert fifoInfo.isRegular == false
+      doAssert linkFifoInfo.isRegular == false
+
+      removeFile(fifoPath)
+      removeFile(linkFifoPath)
+
 testGetFileInfo()

tests/stdlib/tos.nim

@@ -344,6 +344,8 @@ block walkDirRec:
 
   removeDir("walkdir_test")
 
+import std/sequtils
+
 block: # walkDir
   doAssertRaises(OSError):
     for a in walkDir("nonexistent", checkDir = true): discard
@@ -358,6 +360,21 @@ block: # walkDir
         doAssert k == pcLinkToDir
       removeDir("walkdir_test")
 
+  when defined(posix):
+    block walkDirRegular:
+      createDir("walkdir_test")
+      doAssert execShellCmd("mkfifo walkdir_test/fifo") == 0
+      createSymlink("fifo", "walkdir_test/fifo_link")
+      let withSpecialFiles = toSeq(walkDir("walkdir_test", relative = true))
+      doAssert (withSpecialFiles.len == 2 and
+                (pcFile, "fifo") in withSpecialFiles and
+                (pcLinkToFile, "fifo_link") in withSpecialFiles)
+      # now Unix special files are excluded from walkdir output:
+      let onlyRegularFiles = toSeq(walkDir("walkdir_test", relative = true,
+                                           onlyRegular = true))
+      doAssert onlyRegularFiles.len == 0
+      removeDir("walkdir_test")
+
 block normalizedPath:
   doAssert normalizedPath("") == ""
   block relative:
@@ -708,8 +725,6 @@ block: # isAdmin
   # In Azure on POSIX tests run as a normal user
   if isAzure and defined(posix): doAssert not isAdmin()
 
-import std/sequtils
-
 when doslikeFileSystem:
   import std/private/ntpath
 

tools/nimgrep.nim

@@ -989,7 +989,7 @@ iterator walkDirBasic(dir: string, walkOptC: WalkOptComp[Pattern]): string
     let rightDirForFiles = d.isRightDirectory(walkOptC)
     var files = newSeq[string]()
     var dirs = newSeq[string]()
-    for kind, path in walkDir(d):
+    for kind, path in walkDir(d, onlyRegular = true):
       case kind
       of pcFile:
         if path.hasRightPath(walkOptC) and rightDirForFiles: