implemented new experimental scriptable import mechanism

compiler/gorgeimpl.nim

@@ -0,0 +1,83 @@
+#
+#
+#           The Nim Compiler
+#        (c) Copyright 2017 Andreas Rumpf
+#
+#    See the file "copying.txt", included in this
+#    distribution, for details about the copyright.
+#
+
+## Module that implements ``gorge`` for the compiler as well as
+## the scriptable import mechanism.
+
+import msgs, securehash, os, osproc, streams, strutils, options
+
+proc readOutput(p: Process): (string, int) =
+  result[0] = ""
+  var output = p.outputStream
+  while not output.atEnd:
+    result[0].add(output.readLine)
+    result[0].add("\n")
+  if result[0].len > 0:
+    result[0].setLen(result[0].len - "\n".len)
+  result[1] = p.waitForExit
+
+proc opGorge*(cmd, input, cache: string, info: TLineInfo): (string, int) =
+  let workingDir = parentDir(info.toFullPath)
+  if cache.len > 0:# and optForceFullMake notin gGlobalOptions:
+    let h = secureHash(cmd & "\t" & input & "\t" & cache)
+    let filename = options.toGeneratedFile("gorge_" & $h, "txt")
+    var f: File
+    if open(f, filename):
+      result = (f.readAll, 0)
+      f.close
+      return
+    var readSuccessful = false
+    try:
+      var p = startProcess(cmd, workingDir,
+                           options={poEvalCommand, poStderrToStdout})
+      if input.len != 0:
+        p.inputStream.write(input)
+        p.inputStream.close()
+      result = p.readOutput
+      readSuccessful = true
+      # only cache successful runs:
+      if result[1] == 0:
+        writeFile(filename, result[0])
+    except IOError, OSError:
+      if not readSuccessful: result = ("", -1)
+  else:
+    try:
+      var p = startProcess(cmd, workingDir,
+                           options={poEvalCommand, poStderrToStdout})
+      if input.len != 0:
+        p.inputStream.write(input)
+        p.inputStream.close()
+      result = p.readOutput
+    except IOError, OSError:
+      result = ("", -1)
+
+proc scriptableImport*(pkg, subdir: string; info: TLineInfo): string =
+  var cmd = getConfigVar("resolver.exe")
+  if cmd.len == 0: cmd = "nimresolve"
+  else: cmd = quoteShell(cmd)
+  cmd.add " --source:"
+  cmd.add quoteShell(info.toFullPath())
+  cmd.add " --stdlib:"
+  cmd.add quoteShell(options.libpath)
+  cmd.add "  --project:"
+  cmd.add quoteShell(gProjectFull)
+  if subdir.len != 0:
+    cmd.add " --subdir:"
+    cmd.add quoteShell(subdir)
+  if options.gNoNimblePath:
+    cmd.add " --nonimblepath"
+  cmd.add ' '
+  cmd.add quoteShell(pkg)
+  let (res, exitCode) = opGorge(cmd, "", cmd, info)
+  if exitCode == 0:
+    result = res.strip()
+  elif res.len > 0:
+    localError(info, res)
+  else:
+    localError(info, "cannot resolve: " & (pkg / subdir))

compiler/importer.nim

@@ -11,11 +11,22 @@
 
 import
   intsets, strutils, os, ast, astalgo, msgs, options, idents, rodread, lookups,
-  semdata, passes, renderer
+  semdata, passes, renderer, gorgeimpl
 
 proc evalImport*(c: PContext, n: PNode): PNode
 proc evalFrom*(c: PContext, n: PNode): PNode
 
+proc lookupPackage(pkg, subdir: PNode): string =
+  let sub = if subdir != nil: renderTree(subdir, {renderNoComments}).replace(" ") else: ""
+  case pkg.kind
+  of nkStrLit, nkRStrLit, nkTripleStrLit:
+    result = scriptableImport(pkg.strVal, sub, pkg.info)
+  of nkIdent:
+    result = scriptableImport(pkg.ident.s, sub, pkg.info)
+  else:
+    localError(pkg.info, "package name must be an identifier or string literal")
+    result = ""
+
 proc getModuleName*(n: PNode): string =
   # This returns a short relative module name without the nim extension
   # e.g. like "system", "importer" or "somepath/module"
@@ -31,16 +42,33 @@ proc getModuleName*(n: PNode): string =
     result = n.ident.s
   of nkSym:
     result = n.sym.name.s
-  of nkInfix, nkPrefix:
-    if n.sons[0].kind == nkIdent and n.sons[0].ident.id == getIdent("as").id:
+  of nkInfix:
+    let n0 = n[0]
+    let n1 = n[1]
+    if n0.kind == nkIdent and n0.ident.id == getIdent("as").id:
       # XXX hack ahead:
       n.kind = nkImportAs
       n.sons[0] = n.sons[1]
       n.sons[1] = n.sons[2]
       n.sons.setLen(2)
       return getModuleName(n.sons[0])
-    # hacky way to implement 'x / y /../ z':
-    result = renderTree(n, {renderNoComments}).replace(" ")
+    if n1.kind == nkPrefix and n1[0].kind == nkIdent and n1[0].ident.s == "$":
+      if n0.kind == nkIdent and n0.ident.s == "/":
+        result = lookupPackage(n1[1], n[2])
+      else:
+        localError(n.info, "only '/' supported with $package notation")
+        result = ""
+    else:
+      # hacky way to implement 'x / y /../ z':
+      result = getModuleName(n1)
+      result.add renderTree(n0, {renderNoComments})
+      result.add getModuleName(n[2])
+  of nkPrefix:
+    if n.sons[0].kind == nkIdent and n.sons[0].ident.s == "$":
+      result = lookupPackage(n[1], nil)
+    else:
+      # hacky way to implement 'x / y /../ z':
+      result = renderTree(n, {renderNoComments}).replace(" ")
   of nkDotExpr:
     result = renderTree(n, {renderNoComments}).replace(".", "/")
   of nkImportAs:
@@ -209,12 +237,14 @@ proc evalImport(c: PContext, n: PNode): PNode =
   for i in countup(0, sonsLen(n) - 1):
     let it = n.sons[i]
     if it.kind == nkInfix and it.len == 3 and it[2].kind == nkBracket:
-      let sep = renderTree(it.sons[0], {renderNoComments})
-      let dir = renderTree(it.sons[1], {renderNoComments})
+      let sep = it[0]
+      let dir = it[1]
+      let a = newNodeI(nkInfix, it.info)
+      a.add sep
+      a.add dir
+      a.add sep # dummy entry, replaced in the loop
       for x in it[2]:
-        let f = renderTree(x, {renderNoComments})
-        let a = newStrNode(nkStrLit, (dir & sep & f).replace(" "))
-        a.info = it.info
+        a.sons[2] = x
         impMod(c, a)
     else:
       impMod(c, it)

compiler/nimblecmd.nim

@@ -16,11 +16,11 @@ proc addPath*(path: string, info: TLineInfo) =
     options.searchPaths.insert(path, 0)
 
 type
-  Version = distinct string
+  Version* = distinct string
 
-proc `$`(ver: Version): string {.borrow.}
+proc `$`*(ver: Version): string {.borrow.}
 
-proc newVersion(ver: string): Version =
+proc newVersion*(ver: string): Version =
   doAssert(ver.len == 0 or ver[0] in {'#', '\0'} + Digits,
            "Wrong version: " & ver)
   return Version(ver)
@@ -28,7 +28,7 @@ proc newVersion(ver: string): Version =
 proc isSpecial(ver: Version): bool =
   return ($ver).len > 0 and ($ver)[0] == '#'
 
-proc `<`(ver: Version, ver2: Version): bool =
+proc `<`*(ver: Version, ver2: Version): bool =
   ## This is synced from Nimble's version module.
 
   # Handling for special versions such as "#head" or "#branch".

compiler/sem.nim

@@ -495,13 +495,15 @@ proc isImportSystemStmt(n: PNode): bool =
   case n.kind
   of nkImportStmt:
     for x in n:
-      let f = checkModuleName(x, false)
+      if x.kind == nkIdent:
+        let f = checkModuleName(x, false)
+        if f == magicsys.systemModule.info.fileIndex:
+          return true
+  of nkImportExceptStmt, nkFromStmt:
+    if n[0].kind == nkIdent:
+      let f = checkModuleName(n[0], false)
       if f == magicsys.systemModule.info.fileIndex:
         return true
-  of nkImportExceptStmt, nkFromStmt:
-    let f = checkModuleName(n[0], false)
-    if f == magicsys.systemModule.info.fileIndex:
-      return true
   else: discard
 
 proc semStmtAndGenerateGenerics(c: PContext, n: PNode): PNode =

compiler/vm.nim

@@ -19,7 +19,7 @@ import ast except getstr
 import
   strutils, astalgo, msgs, vmdef, vmgen, nimsets, types, passes,
   parser, vmdeps, idents, trees, renderer, options, transf, parseutils,
-  vmmarshal
+  vmmarshal, gorgeimpl
 
 from semfold import leValueConv, ordinalValToString
 from evaltempl import evalTemplate

compiler/vmdeps.nim

@@ -7,50 +7,7 @@
 #    distribution, for details about the copyright.
 #
 
-import ast, types, msgs, os, osproc, streams, options, idents, securehash
-
-proc readOutput(p: Process): (string, int) =
-  result[0] = ""
-  var output = p.outputStream
-  while not output.atEnd:
-    result[0].add(output.readLine)
-    result[0].add("\n")
-  if result[0].len > 0:
-    result[0].setLen(result[0].len - "\n".len)
-  result[1] = p.waitForExit
-
-proc opGorge*(cmd, input, cache: string, info: TLineInfo): (string, int) =
-  let workingDir = parentDir(info.toFullPath)
-  if cache.len > 0:# and optForceFullMake notin gGlobalOptions:
-    let h = secureHash(cmd & "\t" & input & "\t" & cache)
-    let filename = options.toGeneratedFile("gorge_" & $h, "txt")
-    var f: File
-    if open(f, filename):
-      result = (f.readAll, 0)
-      f.close
-      return
-    var readSuccessful = false
-    try:
-      var p = startProcess(cmd, workingDir,
-                           options={poEvalCommand, poStderrToStdout})
-      if input.len != 0:
-        p.inputStream.write(input)
-        p.inputStream.close()
-      result = p.readOutput
-      readSuccessful = true
-      writeFile(filename, result[0])
-    except IOError, OSError:
-      if not readSuccessful: result = ("", -1)
-  else:
-    try:
-      var p = startProcess(cmd, workingDir,
-                           options={poEvalCommand, poStderrToStdout})
-      if input.len != 0:
-        p.inputStream.write(input)
-        p.inputStream.close()
-      result = p.readOutput
-    except IOError, OSError:
-      result = ("", -1)
+import ast, types, msgs, os, streams, options, idents
 
 proc opSlurp*(file: string, info: TLineInfo, module: PSym): string =
   try: