Fix paths, git for Windows build

appveyor.yml

@@ -1,8 +1,8 @@
 version: '{build}'
 
 image:
+  - Visual Studio 2015
   - Ubuntu
-  - Visual Studio 2017
 
 matrix:
   fast_finish: true
@@ -16,13 +16,12 @@ for:
 -
   matrix:
     only:
-      - image: Visual Studio 2017
+      - image: Visual Studio 2015
 
   environment:
-    ARCH: 32
+    ARCH: 64
-    MINGW_URL: https://sourceforge.net/projects/mingw-w64/files/Toolchains%20targetting%20Win32/Personal%20Builds/mingw-builds/8.1.0/threads-posix/dwarf
+    GIT_URL: https://github.com/git-for-windows/git/releases/download/v2.23.0.windows.1/
-    MINGW_ARCHIVE: i686-8.1.0-release-posix-dwarf-rt_v6-rev0.7z
+    GIT_ARCHIVE: PortableGit-2.23.0-64-bit.7z.exe
-    SFNET_URL: https://sourceforge.net/projects/msys2/files/REPOS/MINGW/i686
 
   install:
     - CD c:\
@@ -30,20 +29,23 @@ for:
       echo %NIM_VERSION% &&
       MKDIR binaries &&
       CD binaries &&
-      appveyor DownloadFile "%MINGW_URL%/%MINGW_ARCHIVE%/download" -FileName "%MINGW_ARCHIVE%" &&
+      MKDIR git &&
-      7z x -y "%MINGW_ARCHIVE%"> nul &&
+      CD git &&
-      del "%MINGW_ARCHIVE%" &&
+      appveyor DownloadFile "%GIT_URL%/%GIT_ARCHIVE%" -FileName "%GIT_ARCHIVE%" &&
+      7z x -y -bd "%GIT_ARCHIVE%"> nul &&
+      del "%GIT_ARCHIVE%" &&
+      CD .. &&
       appveyor DownloadFile "https://nim-lang.org/download/nim-%NIM_VERSION%_x%ARCH%.zip" -FileName "nim-%NIM_VERSION%_x%ARCH%.zip" &&
       7z x -y "nim-%NIM_VERSION%_x%ARCH%.zip"> nul &&
       del "nim-%NIM_VERSION%_x%ARCH%.zip")
-    - SET PATH=c:\binaries\mingw%ARCH%\bin;c:\binaries\nim-%NIM_VERSION%\bin;%USERPROFILE%\.nimble\bin;%PATH%
+    - SET PATH=c:\binaries\git\bin;C:\mingw-w64\x86_64-8.1.0-posix-seh-rt_v6-rev0\mingw64\bin;c:\binaries\nim-%NIM_VERSION%\bin;%USERPROFILE%\.nimble\bin;%PATH%
     - CD %APPVEYOR_BUILD_FOLDER%
 
-  on_finish:
+# on_finish:
-    - 7z a -r buildlogs-win-pkgs.zip %USERPROFILE%\.nimble\pkgs
+#   - 7z a -r buildlogs-win-pkgs.zip %USERPROFILE%\.nimble\pkgs
-    - appveyor PushArtifact buildlogs-win-pkgs.zip
+#   - appveyor PushArtifact buildlogs-win-pkgs.zip
-    - 7z a -r buildlogs-win-projects.zip c:\projects\*
+#   - 7z a -r buildlogs-win-projects.zip c:\projects\*
-    - appveyor PushArtifact buildlogs-win-projects.zip
+#   - appveyor PushArtifact buildlogs-win-projects.zip
 
   cache:
     - c:\binaries
@@ -71,11 +73,11 @@ for:
     - export PATH=/home/appveyor/binaries/nim-$NIM_VERSION/bin:~/.nimble/bin:$PATH
     - cd $APPVEYOR_BUILD_FOLDER
 
-  on_finish:
+# on_finish:
-    - zip -r -q buildlogs-lin-pkgs.zip ~/.nimble/pkgs
+#   - zip -r -q buildlogs-lin-pkgs.zip ~/.nimble/pkgs
-    - appveyor PushArtifact buildlogs-lin-pkgs.zip
+#   - appveyor PushArtifact buildlogs-lin-pkgs.zip
-    - zip -r -q buildlogs-lin-projects.zip /home/appveyor/projects
+#   - zip -r -q buildlogs-lin-projects.zip /home/appveyor/projects
-    - appveyor PushArtifact buildlogs-lin-projects.zip
+#   - appveyor PushArtifact buildlogs-lin-projects.zip
 
   cache:
   - /home/appveyor/binaries

nimterop/build.nim

@@ -4,6 +4,11 @@ import os except findExe
 
 import "."/[compat]
 
+proc sanitizePath*(path: string, noQuote = false): string =
+  result = path.multiReplace([("\\\\", $DirSep), ("\\", $DirSep), ("/", $DirSep)])
+  if not noQuote:
+    result = result.quoteShell
+
 proc execAction*(cmd: string, nostderr=false): string =
   ## Execute an external command - supported at compile time
   ##
@@ -39,7 +44,7 @@ proc findExe*(exe: string): string =
     (oup, code) = gorgeEx(cmd)
 
   if code == 0:
-    return oup.strip()
+    return oup.splitLines()[0].strip()
 
 proc mkDir*(dir: string) =
   ## Create a directory at cmopile time
@@ -49,7 +54,7 @@ proc mkDir*(dir: string) =
   if not dirExists(dir):
     let
       flag = when not defined(Windows): "-p" else: ""
-    discard execAction(&"mkdir {flag} {dir.quoteShell}")
+    discard execAction(&"mkdir {flag} {dir.sanitizePath}")
 
 proc cpFile*(source, dest: string, move=false) =
   ## Copy a file from source to destination at compile time
@@ -68,7 +73,7 @@ proc cpFile*(source, dest: string, move=false) =
         else:
           "cp -f"
 
-  discard execAction(&"{cmd} {source.quoteShell} {dest.quoteShell}")
+  discard execAction(&"{cmd} {source.sanitizePath} {dest.sanitizePath}")
 
 proc mvFile*(source, dest: string) =
   ## Move a file from source to destination at compile time
@@ -87,7 +92,7 @@ proc rmFile*(source: string, dir = false) =
       else:
         "rm -rf"
 
-  discard execAction(&"{cmd} {source.quoteShell}")
+  discard execAction(&"{cmd} {source.sanitizePath}")
 
 proc rmDir*(source: string) =
   ## Remove a directory or pattern at compile time
@@ -103,7 +108,7 @@ proc extractZip*(zipfile, outdir: string) =
           "[IO.Compression.ZipFile]::ExtractToDirectory('$#', '.'); }\""
 
   echo "# Extracting " & zipfile
-  discard execAction(&"cd {outdir.quoteShell} && {cmd % zipfile}")
+  discard execAction(&"cd {outdir.sanitizePath} && {cmd % zipfile}")
 
 proc extractTar*(tarfile, outdir: string) =
   ## Extract a tar file using tar, 7z or 7za to the specified output directory
@@ -121,20 +126,22 @@ proc extractTar*(tarfile, outdir: string) =
         of ".bz2": "j"
         else: ""
 
-    cmd = "tar xvf" & typ & " " & tarfile.quoteShell
+    cmd = "tar xvf" & typ & " " & tarfile.sanitizePath
   else:
     for i in ["7z", "7za"]:
       if findExe(i).len != 0:
-        cmd = i & " x $#" % tarfile.quoteShell
+        cmd = i & " x $#" % tarfile.sanitizePath
 
         name = tarfile.splitFile().name
         if ".tar" in name.toLowerAscii():
-          cmd &= " && " & i & " x $#" % name.quoteShell
+          cmd &= " && " & i & " x $#" % name.sanitizePath
 
         break
 
+  doAssert cmd.len != 0, "No extraction tool - tar, 7z, 7za - available for " & tarfile.sanitizePath
+
   echo "# Extracting " & tarfile
-  discard execAction(&"cd {outdir.quoteShell} && {cmd}")
+  discard execAction(&"cd {outdir.sanitizePath} && {cmd}")
   if name.len != 0:
     rmFile(outdir / name)
 
@@ -152,7 +159,7 @@ proc downloadUrl*(url, outdir: string) =
     mkDir(outdir)
     var cmd = findExe("curl")
     if cmd.len != 0:
-      cmd &= " -L $# -o $#"
+      cmd &= " -Lk $# -o $#"
     else:
       cmd = findExe("wget")
       if cmd.len != 0:
@@ -161,7 +168,7 @@ proc downloadUrl*(url, outdir: string) =
         cmd = "powershell [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12; wget $# -OutFile $#"
       else:
         doAssert false, "No download tool available - curl, wget"
-    discard execAction(cmd % [url, (outdir/file).quoteShell])
+    discard execAction(cmd % [url, (outdir/file).sanitizePath])
 
     if ext == ".zip":
       extractZip(file, outdir)
@@ -172,7 +179,7 @@ proc gitReset*(outdir: string) =
   ## Hard reset the git repository at the specified directory
   echo "# Resetting " & outdir
 
-  let cmd = &"cd {outdir.quoteShell} && git reset --hard"
+  let cmd = &"cd {outdir.sanitizePath} && git reset --hard"
   while execAction(cmd).contains("Permission denied"):
     sleep(1000)
     echo "#   Retrying ..."
@@ -185,7 +192,7 @@ proc gitCheckout*(file, outdir: string) =
   ## successful wrapping with `cImport()` or `c2nImport()`.
   echo "# Resetting " & file
   let file2 = file.relativePath outdir
-  let cmd = &"cd {outdir.quoteShell} && git checkout {file2.quoteShell}"
+  let cmd = &"cd {outdir.sanitizePath} && git checkout {file2.sanitizePath}"
   while execAction(cmd).contains("Permission denied"):
     sleep(500)
     echo "#   Retrying ..."
@@ -206,7 +213,7 @@ proc gitPull*(url: string, outdir = "", plist = "", checkout = "") =
     return
 
   let
-    outdirQ = outdir.quoteShell
+    outdirQ = outdir.sanitizePath
 
   mkDir(outdir)
 
@@ -279,7 +286,7 @@ proc configure*(path, check: string, flags = "") =
       if fileExists(path / i):
         echo "#   Running autogen.sh"
 
-        echo execAction(&"cd {(path / i).parentDir().quoteShell} && bash autogen.sh")
+        echo execAction(&"cd {(path / i).parentDir().sanitizePath} && bash autogen.sh")
 
         break
 
@@ -288,7 +295,7 @@ proc configure*(path, check: string, flags = "") =
       if fileExists(path / i):
         echo "#   Running autoreconf"
 
-        echo execAction(&"cd {path.quoteShell} && autoreconf -fi")
+        echo execAction(&"cd {path.sanitizePath} && autoreconf -fi")
 
         break
 
@@ -296,7 +303,7 @@ proc configure*(path, check: string, flags = "") =
     echo "#   Running configure " & flags
 
     var
-      cmd = &"cd {path.quoteShell} && bash configure"
+      cmd = &"cd {path.sanitizePath} && bash configure"
     if flags.len != 0:
       cmd &= &" {flags}"
 
@@ -327,7 +334,7 @@ proc cmake*(path, check, flags: string) =
   mkDir(path)
 
   var
-    cmd = &"cd {path.quoteShell} && cmake {flags}"
+    cmd = &"cd {path.sanitizePath} && cmake {flags}"
 
   echo execAction(cmd)
 
@@ -359,7 +366,7 @@ proc make*(path, check: string|Regex, flags = "") =
       cpFile(cmd, cmd.replace("mingw32-make", "make"))
   doAssert cmd.len != 0, "Make not found"
 
-  cmd = &"cd {path.quoteShell} && make"
+  cmd = &"cd {path.sanitizePath} && make"
   if flags.len != 0:
     cmd &= &" {flags}"
 
@@ -383,8 +390,7 @@ proc getGccPaths*(mode = "c"): seq[string] =
       break
     if inc:
       var
-        path = line.strip()
+        path = line.strip().myNormalizedPath()
-      path.normalizePath()
       if path notin result:
         result.add path
 
@@ -400,15 +406,13 @@ proc getGccLibPaths*(mode = "c"): seq[string] =
     if "LIBRARY_PATH=" in line:
       for path in line[13 .. ^1].split(PathSep):
         var
-          path = path.strip()
+          path = path.strip().myNormalizedPath()
-        path.normalizePath()
         if path notin result:
           result.add path
       break
     elif '\t' in line:
       var
-        path = line.strip()
+        path = line.strip().myNormalizedPath()
-      path.normalizePath()
       if path notin result:
         result.add path
 
@@ -504,7 +508,7 @@ proc buildLibrary(lname, outdir, conFlags, cmakeFlags, makeFlags: string): strin
             gen = "MinGW Makefiles"
         else:
           gen = "Unix Makefiles"
-        cmake(outdir / "build", "Makefile", &".. -G {gen.quoteShell} {cmakeFlags}")
+        cmake(outdir / "build", "Makefile", &".. -G {gen.sanitizePath} {cmakeFlags}")
         cmakeDeps = true
         makePath = outdir / "build"
       else:

nimterop/cimport.nim

@@ -79,11 +79,11 @@ proc getFileDate(fullpath: string): string =
     ret = 0
     cmd =
       when defined(Windows):
-        &"cmd /c for %a in ({fullpath.quoteShell}) do echo %~ta"
+        &"cmd /c for %a in ({fullpath.sanitizePath}) do echo %~ta"
       elif defined(Linux):
-        &"stat -c %y {fullpath.quoteShell}"
+        &"stat -c %y {fullpath.sanitizePath}"
       elif defined(OSX):
-        &"stat -f %m {fullpath.quoteShell}"
+        &"stat -f %m {fullpath.sanitizePath}"
 
   (result, ret) = gorgeEx(cmd)
 
@@ -120,7 +120,7 @@ proc getNimCheckError(output: string): tuple[tmpFile, errors: string] =
         getCurrentCompilerExe()
       else:
         "nim"
-    (check, _) = gorgeEx(&"{nim} check {result.tmpFile.quoteShell}")
+    (check, _) = gorgeEx(&"{nim} check {result.tmpFile.sanitizePath}")
 
   result.errors = "\n\n" & check
 
@@ -131,7 +131,7 @@ proc getToast(fullpath: string, recurse: bool = false, dynlib: string = "",
     cmd = when defined(Windows): "cmd /c " else: ""
 
   let toastExe = toastExePath()
-  doAssert fileExists(toastExe), "toast not compiled: " & toastExe.quoteShell &
+  doAssert fileExists(toastExe), "toast not compiled: " & toastExe.sanitizePath &
     " make sure 'nimble build' or 'nimble install' built it"
   cmd &= &"{toastExe} --preprocess"
 
@@ -145,7 +145,7 @@ proc getToast(fullpath: string, recurse: bool = false, dynlib: string = "",
     cmd.add &" --defines+={i.quoteShell}"
 
   for i in gStateCT.includeDirs:
-    cmd.add &" --includeDirs+={i.quoteShell}"
+    cmd.add &" --includeDirs+={i.sanitizePath}"
 
   if not noNimout:
     cmd.add &" --pnim"
@@ -157,12 +157,12 @@ proc getToast(fullpath: string, recurse: bool = false, dynlib: string = "",
       cmd.add &" --symOverride={gStateCT.symOverride.join(\",\")}"
 
     when (NimMajor, NimMinor, NimPatch) >= (0, 19, 9):
-      cmd.add &" --nim:{getCurrentCompilerExe().quoteShell}"
+      cmd.add &" --nim:{getCurrentCompilerExe().sanitizePath}"
 
     if gStateCT.pluginSourcePath.nBl:
-      cmd.add &" --pluginSourcePath={gStateCT.pluginSourcePath.quoteShell}"
+      cmd.add &" --pluginSourcePath={gStateCT.pluginSourcePath.sanitizePath}"
 
-  cmd.add &" {fullpath.quoteShell}"
+  cmd.add &" {fullpath.sanitizePath}"
 
   # see https://github.com/nimterop/nimterop/issues/69
   (result, ret) = gorgeEx(cmd, cache=getCacheValue(fullpath))
@@ -527,7 +527,7 @@ macro cImport*(filename: static string, recurse: static bool = false, dynlib: st
   let
     fullpath = findPath(filename)
 
-  echo "# Importing " & fullpath
+  echo "# Importing " & fullpath.sanitizePath
 
   let
     output = getToast(fullpath, recurse, dynlib, mode, flags)

nimterop/compat.nim

@@ -6,10 +6,18 @@ put everything that requires `when (NimMajor, NimMinor, NimPatch)` here
 import os
 
 when (NimMajor, NimMinor, NimPatch) >= (0, 19, 9):
+  proc myNormalizedPath*(path: string): string = path.normalizedPath()
+
   export relativePath
+
 else:
   import std/[ospaths,strutils]
 
+  proc myNormalizedPath*(path: string): string =
+    result = path.normalizedPath()
+    when defined(windows):
+      result = result.strip(trailing = false, chars = {'\\'})
+
   proc relativePath*(file, base: string): string =
     ## naive version of `os.relativePath` ; remove after nim >= 0.19.9
     runnableExamples:
@@ -17,8 +25,8 @@ else:
       check:
         "/foo/bar/baz/log.txt".unixToNativePath.relativePath("/foo/bar".unixToNativePath) == "baz/log.txt".unixToNativePath
         "foo/bar/baz/log.txt".unixToNativePath.relativePath("foo/bar".unixToNativePath) == "baz/log.txt".unixToNativePath
-    var base = base.normalizedPath
+    var base = base.myNormalizedPath
-    var file = file.normalizedPath
+    var file = file.myNormalizedPath
     if not base.endsWith DirSep: base.add DirSep
     doAssert file.startsWith base
     result = file[base.len .. ^1]

nimterop/getters.nim

@@ -73,9 +73,6 @@ const gTypeMap = {
   "long double": "clongdouble"
 }.toTable()
 
-proc sanitizePath*(path: string): string =
-  path.multiReplace([("\\\\", $DirSep), ("\\", $DirSep), ("/", $DirSep)])
-
 proc getType*(str: string): string =
   if str == "void":
     return "object"
@@ -205,15 +202,15 @@ proc getPreprocessor*(gState: State, fullpath: string, mode = "cpp"): string =
 
     rdata: seq[string] = @[]
     start = false
-    sfile = fullpath.sanitizePath
+    sfile = fullpath.sanitizePath(noQuote = true)
 
   for inc in gState.includeDirs:
-    cmd &= &"-I{inc.quoteShell} "
+    cmd &= &"-I{inc.sanitizePath} "
 
   for def in gState.defines:
     cmd &= &"-D{def} "
 
-  cmd &= &"{fullpath.quoteShell}"
+  cmd &= &"{fullpath.sanitizePath}"
 
   # Include content only from file
   for line in execAction(cmd).splitLines():
@@ -221,19 +218,19 @@ proc getPreprocessor*(gState: State, fullpath: string, mode = "cpp"): string =
       if line.len > 1 and line[0 .. 1] == "# ":
         start = false
         let
-          saniLine = line.sanitizePath
+          saniLine = line.sanitizePath(noQuote = true)
         if sfile in saniLine:
           start = true
         elif not ("\\" in line) and not ("/" in line) and extractFilename(sfile) in line:
           start = true
         elif gState.recurse:
           let
-            pDir = sfile.expandFilename().parentDir().sanitizePath()
+            pDir = sfile.expandFilename().parentDir().sanitizePath(noQuote = true)
           if pDir.len == 0 or pDir in saniLine:
             start = true
           else:
             for inc in gState.includeDirs:
-              if inc.absolutePath().sanitizePath in saniLine:
+              if inc.absolutePath().sanitizePath(noQuote = true) in saniLine:
                 start = true
                 break
       else:
@@ -395,7 +392,7 @@ proc loadPlugin*(gState: State, sourcePath: string) =
     pdll = sourcePath.dll
   if not fileExists(pdll) or
     sourcePath.getLastModificationTime() > pdll.getLastModificationTime():
-    discard execAction(&"{gState.nim.quoteShell} c --app:lib {sourcePath.quoteShell}")
+    discard execAction(&"{gState.nim.sanitizePath} c --app:lib {sourcePath.sanitizePath}")
   doAssert fileExists(pdll), "No plugin binary generated for " & sourcePath
 
   let lib = loadLib(pdll)

nimterop/treesitter/api.nim

@@ -7,17 +7,17 @@ import ".."/[setup, paths, types]
 static:
   treesitterSetup()
 
-const sourcePath = incDir() / "treesitter/lib"
+const sourcePath = incDir() / "treesitter" / "lib"
 
 when defined(Linux):
   {.passC: "-std=c11".}
 
 {.passC: "-DUTF8PROC_STATIC".}
-{.passC: "-I$1/include" % sourcePath.}
+{.passC: "-I$1" % (sourcePath / "include").}
-{.passC: "-I$1/src" % sourcePath.}
+{.passC: "-I$1" % (sourcePath / "src").}
-{.passC: "-I$1/../../utf8proc" % sourcePath.}
+{.passC: "-I$1" % (sourcePath / ".." / ".." / "utf8proc").}
 
-{.compile: sourcePath / "src/lib.c".}
+{.compile: sourcePath / "src" / "lib.c".}
 
 ### Generated below
 
@@ -27,7 +27,7 @@ defineEnum(TSInputEncoding)
 defineEnum(TSSymbolType)
 defineEnum(TSLogType)
 const
-  headerapi {.used.} = sourcePath / "include/tree_sitter/api.h"
+  headerapi {.used.} = sourcePath / "include" / "tree_sitter" / "api.h"
   TREE_SITTER_LANGUAGE_VERSION* = 9
   TSInputEncodingUTF8* = 0.TSInputEncoding
   TSInputEncodingUTF16* = 1.TSInputEncoding

nimterop/treesitter/c.nim

@@ -5,7 +5,7 @@ import ".."/[setup, paths]
 static:
   treesitterCSetup()
 
-const srcDir = incDir() / "treesitter_c/src"
+const srcDir = incDir() / "treesitter_c" / "src"
 
 import "."/api
 

nimterop/treesitter/cpp.nim

@@ -5,7 +5,7 @@ import ".."/[setup, paths]
 static:
   treesitterCppSetup()
 
-const srcDir = incDir() / "treesitter_cpp/src"
+const srcDir = incDir() / "treesitter_cpp" / "src"
 
 {.passC: "-I$1" % srcDir.}