path canonicalization for imported modules, relative paths written in rod files

compiler/ast.nim

@@ -11,7 +11,7 @@
 
 import 
   msgs, hashes, nversion, options, strutils, crc, ropes, idents, lists, 
-  intsets, idgen
+  intsets, idgen, os
 
 const
   ImportTablePos* = 0         # imported symbols are at level 0
@@ -1030,3 +1030,18 @@ proc getStrOrChar*(a: PNode): string =
     internalError(a.info, "getStrOrChar")
     result = ""
 
+proc getModuleName*(n: PNode): string =
+  # This returns a short relative module name without the nim extension
+  # e.g. like "system", "importer" or "somepath/module"
+  # The proc won't perform any checks that the path is actually valid
+  case n.kind
+  of nkStrLit, nkRStrLit, nkTripleStrLit:
+    result = UnixToNativePath(n.strVal)
+  of nkIdent:
+    result = n.ident.s
+  of nkSym:
+    result = n.sym.name.s
+  else:
+    internalError(n.info, "getModuleName")
+    result = ""
+

compiler/depends.nim

@@ -33,10 +33,10 @@ proc addDotDependency(c: PPassContext, n: PNode): PNode =
   case n.kind
   of nkImportStmt: 
     for i in countup(0, sonsLen(n) - 1): 
-      var imported = splitFile(getModuleFile(n.sons[i])).name
+      var imported = getModuleName(n.sons[i])
       addDependencyAux(g.module.name.s, imported)
   of nkFromStmt: 
-    var imported = splitFile(getModuleFile(n.sons[0])).name
+    var imported = getModuleName(n.sons[0])
     addDependencyAux(g.module.name.s, imported)
   of nkStmtList, nkBlockStmt, nkStmtListExpr, nkBlockExpr: 
     for i in countup(0, sonsLen(n) - 1): discard addDotDependency(c, n.sons[i])

compiler/docgen.nim

@@ -760,21 +760,12 @@ proc renderRstToOut(d: PDoc, n: PRstNode): PRope =
 proc checkForFalse(n: PNode): bool = 
   result = n.kind == nkIdent and IdentEq(n.ident, "false")
   
-proc getModuleFile(n: PNode): string = 
-  case n.kind
-  of nkStrLit, nkRStrLit, nkTripleStrLit: result = n.strVal
-  of nkIdent: result = n.ident.s
-  of nkSym: result = n.sym.name.s
-  else: 
-    internalError(n.info, "getModuleFile()")
-    result = ""
-  
 proc traceDeps(d: PDoc, n: PNode) = 
   const k = skModule
   if d.section[k] != nil: app(d.section[k], ", ")
   dispA(d.section[k], 
         "<a class=\"reference external\" href=\"$1.html\">$1</a>", 
-        "$1", [toRope(getModuleFile(n))])
+        "$1", [toRope(getModuleName(n))])
 
 proc generateDoc(d: PDoc, n: PNode) = 
   case n.kind

compiler/importer.nim

@@ -16,25 +16,13 @@ import
 proc evalImport*(c: PContext, n: PNode): PNode
 proc evalFrom*(c: PContext, n: PNode): PNode
 proc importAllSymbols*(c: PContext, fromMod: PSym)
-proc getModuleFile*(n: PNode): string
-# implementation
 
-proc findModule(info: TLineInfo, modulename: string): string = 
+proc checkModuleName*(n: PNode): string =
-  # returns path to module
+  # This returns the full canonical path for a given module import
-  result = options.FindFile(AddFileExt(modulename, nimExt))
+  var modulename = n.getModuleName
-  if result == "": Fatal(info, errCannotOpenFile, modulename)
+  result = findModule(modulename)
-  
+  if result.len == 0:
-proc getModuleFile(n: PNode): string = 
+    Fatal(n.info, errCannotOpenFile, modulename)
-  case n.kind
-  of nkStrLit, nkRStrLit, nkTripleStrLit: 
-    result = findModule(n.info, UnixToNativePath(n.strVal))
-  of nkIdent: 
-    result = findModule(n.info, n.ident.s)
-  of nkSym: 
-    result = findModule(n.info, n.sym.name.s)
-  else: 
-    internalError(n.info, "getModuleFile()")
-    result = ""
 
 proc rawImportSymbol(c: PContext, s: PSym) = 
   # This does not handle stubs, because otherwise loading on demand would be
@@ -103,7 +91,7 @@ proc importAllSymbols(c: PContext, fromMod: PSym) =
 proc evalImport(c: PContext, n: PNode): PNode = 
   result = n
   for i in countup(0, sonsLen(n) - 1): 
-    var f = getModuleFile(n.sons[i])
+    var f = checkModuleName(n.sons[i])
     var m = gImportModule(f)
     if sfDeprecated in m.flags: 
       Message(n.sons[i].info, warnDeprecated, m.name.s) 
@@ -114,7 +102,7 @@ proc evalImport(c: PContext, n: PNode): PNode =
 proc evalFrom(c: PContext, n: PNode): PNode = 
   result = n
   checkMinSonsLen(n, 2)
-  var f = getModuleFile(n.sons[0])
+  var f = checkModuleName(n.sons[0])
   var m = gImportModule(f)
   n.sons[0] = newSymNode(m)
   addDecl(c, m)               # add symbol to symbol table of module

compiler/main.nim

@@ -69,7 +69,7 @@ proc importModule(filename: string): PSym =
 proc CompileModule(filename: string, flags: TSymFlags): PSym =
   var rd: PRodReader = nil
   var f = addFileExt(filename, nimExt)
-  result = newModule(filename)
+  result = newModule(f)
   result.flags = result.flags + flags
   if gCmd in {cmdCompileToC, cmdCompileToCpp, cmdCheck, cmdIdeTools}: 
     rd = handleSymbolFile(result, f)

compiler/msgs.nim

@@ -409,26 +409,34 @@ proc newFileInfo(fullPath, projPath: string): TFileInfo =
   result.projPath = projPath
 
 proc fileInfoIdx*(filename: string): int32 =
-  var canonical = canonicalizePath(filename)
+  var
+    canonical: string
+    pseudoPath = false
+
+  try:
+    canonical = canonicalizePath(filename)
+  except:
+    canonical = filename
+    # The compiler uses "filenames" such as `command line` or `stdin`
+    # This flag indicates that we are working with such a path here
+    pseudoPath = true
 
   if filenameToIndexTbl.hasKey(canonical):
     result = filenameToIndexTbl[canonical]
   else:
     result = fileInfos.len.int32
-    fileInfos.add(newFileInfo(canonical, canonical.shortenDir))
+    fileInfos.add(newFileInfo(canonical, if pseudoPath: "" else: canonical.shortenDir))
     filenameToIndexTbl[canonical] = result
 
-proc newLineInfo*(filename: string, line, col: int): TLineInfo =
-  result.fileIndex = filename.fileInfoIdx
-  result.line = int16(line)
-  result.col = int16(col)
-
 proc newLineInfo*(fileInfoIdx: int32, line, col: int): TLineInfo =
   result.fileIndex = fileInfoIdx
   result.line = int16(line)
   result.col = int16(col)
 
-fileInfos.add(newFileInfo("command line", ""))
+proc newLineInfo*(filename: string, line, col: int): TLineInfo {.inline.} =
+  result = newLineInfo(filename.fileInfoIdx, line, col)
+
+fileInfos.add(newFileInfo("", "command line"))
 var gCmdLineInfo* = newLineInfo(int32(0), 1, 1)
 
 proc raiseRecoverableError*() {.noinline, noreturn.} =

compiler/options.nim

@@ -188,19 +188,28 @@ iterator iterSearchPath*(): string =
     yield it.data
     it = PStrEntry(it.Next)  
 
-proc rawFindFile(f: string): string = 
+proc rawFindFile(f: string): string =
+  template ret(e: expr) =
+    result = e.canonicalizePath
+    return
+
   if ExistsFile(f): 
-    result = f
+    ret f
   else: 
     for it in iterSearchPath():
       result = JoinPath(it, f)
-      if ExistsFile(result): return
+      if ExistsFile(result):
+        ret result
     result = ""
 
 proc FindFile*(f: string): string = 
   result = rawFindFile(f)
   if len(result) == 0: result = rawFindFile(toLower(f))
-  
+
+proc findModule*(modulename: string): string {.inline.} =
+  # returns path to module
+  result = FindFile(AddFileExt(modulename, nimExt))
+
 proc binaryStrSearch*(x: openarray[string], y: string): int = 
   var a = 0
   var b = len(x) - 1

compiler/rodread.nim

@@ -576,7 +576,9 @@ proc processRodFile(r: PRodReader, crc: TCrc32) =
       L = 0
       while r.s[r.pos] > '\x0A' and r.s[r.pos] != ')': 
         setlen(r.files, L + 1)
-        r.files[L] = decodeStr(r.s, r.pos)
+        var relativePath = decodeStr(r.s, r.pos)
+        var resolvedPath = relativePath.findModule
+        r.files[L] = if resolvedPath.len > 0: resolvedPath else: relativePath
         inc(r.pos)            # skip #10
         inc(r.line)
         inc(L)
@@ -628,7 +630,8 @@ proc processRodFile(r: PRodReader, crc: TCrc32) =
       r.initIdx = r.pos + 2   # "(\10"
       skipSection(r)
     else: 
-      MsgWriteln("skipping section: " & $r.pos)
+      MsgWriteln("skipping section: " & section &
+                 " at " & $r.pos & " in " & r.filename)
       skipSection(r)
     if r.s[r.pos] == '\x0A': 
       inc(r.pos)
@@ -792,7 +795,7 @@ proc getModuleIdx(filename: string): int =
   result = len(gMods)
   setlen(gMods, result + 1)
 
-proc checkDep(filename: string): TReasonForRecompile = 
+proc checkDep(filename: string): TReasonForRecompile =
   assert(not isNil(filename)) 
   var idx = getModuleIdx(filename)
   if gMods[idx].reason != rrEmpty: 

compiler/rodwrite.nim

@@ -87,17 +87,18 @@ proc newRodWriter(modfilename: string, crc: TCrc32, module: PSym): PRodWriter =
   result.init = ""
   result.data = newStringOfCap(12_000)
   
-proc addModDep(w: PRodWriter, dep: string) = 
+proc addModDep(w: PRodWriter, dep: string) =
   if w.modDeps.len != 0: add(w.modDeps, ' ')
   encodeVInt(fileIdx(w, dep), w.modDeps)
 
 const 
   rodNL = "\x0A"
 
-proc addInclDep(w: PRodWriter, dep: string) = 
+proc addInclDep(w: PRodWriter, dep: string) =
+  var resolved = dep.findModule
   encodeVInt(fileIdx(w, dep), w.inclDeps)
   add(w.inclDeps, " ")
-  encodeVInt(crcFromFile(dep), w.inclDeps)
+  encodeVInt(crcFromFile(resolved), w.inclDeps)
   add(w.inclDeps, rodNL)
 
 proc pushType(w: PRodWriter, t: PType) =
@@ -555,21 +556,21 @@ proc process(c: PPassContext, n: PNode): PNode =
       #            addInterfaceSym(w, a.sons[j].sym);        
       #        end 
   of nkImportStmt: 
-    for i in countup(0, sonsLen(n) - 1): addModDep(w, getModuleFile(n.sons[i]))
+    for i in countup(0, sonsLen(n) - 1): addModDep(w, getModuleName(n.sons[i]))
     addStmt(w, n)
   of nkFromStmt: 
-    addModDep(w, getModuleFile(n.sons[0]))
+    addModDep(w, getModuleName(n.sons[0]))
     addStmt(w, n)
   of nkIncludeStmt: 
-    for i in countup(0, sonsLen(n) - 1): addInclDep(w, getModuleFile(n.sons[i]))
+    for i in countup(0, sonsLen(n) - 1): addInclDep(w, getModuleName(n.sons[i]))
   of nkPragma: 
     addStmt(w, n)
   else: 
     nil
 
-proc myOpen(module: PSym, filename: string): PPassContext = 
+proc myOpen(module: PSym, filename: string): PPassContext =
   if module.id < 0: InternalError("rodwrite: module ID not set")
-  var w = newRodWriter(filename, rodread.GetCRC(filename), module)
+  var w = newRodWriter(filename, rodread.GetCRC(module.info.toFullPath), module)
   rawAddInterfaceSym(w, module)
   result = w
 

compiler/semstmts.nim

@@ -797,7 +797,7 @@ proc evalInclude(c: PContext, n: PNode): PNode =
   result = newNodeI(nkStmtList, n.info)
   addSon(result, n)
   for i in countup(0, sonsLen(n) - 1): 
-    var f = getModuleFile(n.sons[i])
+    var f = checkModuleName(n.sons[i])
     var fileIndex = f.fileInfoIdx
     if ContainsOrIncl(c.includedFiles, fileIndex): 
       GlobalError(n.info, errRecursiveDependencyX, f)