next steps for FFI support

compiler/commands.nim

@@ -398,13 +398,13 @@ proc processSwitch(switch, arg: string, pass: TCmdlinePass, info: TLineInfo) =
     if pass in {passCmd2, passPP}: extccomp.addLinkOption(arg)
   of "cincludes":
     expectArg(switch, arg, pass, info)
-    if pass in {passCmd2, passPP}: cIncludes.add arg
+    if pass in {passCmd2, passPP}: cIncludes.add arg.processPath
   of "clibdir":
     expectArg(switch, arg, pass, info)
-    if pass in {passCmd2, passPP}: cLibs.add arg
+    if pass in {passCmd2, passPP}: cLibs.add arg.processPath
   of "clib":
     expectArg(switch, arg, pass, info)
-    if pass in {passCmd2, passPP}: cLinkedLibs.add arg
+    if pass in {passCmd2, passPP}: cLinkedLibs.add arg.processPath
   of "header":
     headerFile = arg
     incl(gGlobalOptions, optGenIndex)

compiler/evalffi.nim

@@ -441,3 +441,46 @@ proc callForeignFunction*(call: PNode): PNode =
   for i in 1 .. call.len-1:
     call.sons[i] = unpack(args[i-1], typ.sons[i], call[i])
     dealloc args[i-1]
+
+proc callForeignFunction*(fn: PNode, fntyp: PType,
+                          args: var TNodeSeq, start, len: int,
+                          info: TLineInfo): PNode =
+  internalAssert fn.kind == nkPtrLit
+  
+  var cif: TCif
+  var sig: TParamList
+  for i in 0..len-1:
+    var aTyp = args[i+start].typ
+    if aTyp.isNil:
+      internalAssert i+1 < fntyp.len
+      aTyp = fntyp.sons[i+1]
+      args[i+start].typ = aTyp
+    sig[i] = mapType(aTyp)
+    if sig[i].isNil: globalError(info, "cannot map FFI type")
+  
+  if prep_cif(cif, mapCallConv(fntyp.callConv, info), cuint(len),
+              mapType(fntyp.sons[0]), sig) != OK:
+    globalError(info, "error in FFI call")
+  
+  var cargs: TArgList
+  let fn = cast[pointer](fn.intVal)
+  for i in 0 .. len-1:
+    let t = args[i+start].typ
+    cargs[i] = alloc0(packSize(args[i+start], t))
+    pack(args[i+start], t, cargs[i])
+  let retVal = if isEmptyType(fntyp.sons[0]): pointer(nil)
+               else: alloc(fntyp.sons[0].getSize.int)
+
+  libffi.call(cif, fn, retVal, cargs)
+  
+  if retVal.isNil: 
+    result = emptyNode
+  else:
+    result = unpack(retVal, fntyp.sons[0], nil)
+    result.info = info
+
+  if retVal != nil: dealloc retVal
+  for i in 0 .. len-1:
+    let t = args[i+start].typ
+    args[i+start] = unpack(cargs[i], t, args[i+start])
+    dealloc cargs[i]

compiler/nimrod.cfg

@@ -12,3 +12,6 @@ path:"$lib/packages/docutils"
 
 define:booting
 
+@if windows:
+  cincludes: "$lib/wrappers/libffi/common"
+@end

compiler/vm.nim

@@ -14,10 +14,13 @@ import ast except getstr
 
 import
   strutils, astalgo, msgs, vmdef, vmgen, nimsets, types, passes, unsigned,
-  parser, vmdeps, idents, trees, renderer
+  parser, vmdeps, idents, trees, renderer, options
 
 from semfold import leValueConv, ordinalValToString
 
+when hasFFI:
+  import evalffi
+
 type
   PStackFrame* = ref TStackFrame
   TStackFrame* = object
@@ -608,10 +611,25 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): PNode =
       let rc = instr.regC
       let isClosure = regs[rb].kind == nkPar
       let prc = if not isClosure: regs[rb].sym else: regs[rb].sons[0].sym
+      if sfImportc in prc.flags:
+        if allowFFI notin c.features:
+          globalError(c.debug[pc], errGenerated, "VM not allowed to do FFI")
+        # we pass 'tos.slots' instead of 'regs' so that the compiler can keep
+        # 'regs' in a register:
+        when hasFFI:
+          let newValue = callForeignFunction(c.globals.sons[prc.position-1],
+                                             prc.typ, tos.slots,
+                                             rb+1, rc-1, c.debug[pc])
+          if newValue.kind != nkEmpty:
+            assert instr.opcode == opcIndCallAsgn
+            regs[ra] = newValue
+        else:
+          globalError(c.debug[pc], errGenerated, "VM not built with FFI support")
+      else:
         let newPc = compile(c, prc)
         #echo "new pc ", newPc, " calling: ", prc.name.s
         var newFrame = PStackFrame(prc: prc, comesFrom: pc, next: tos)
-      newSeq(newFrame.slots, prc.position)
+        newSeq(newFrame.slots, prc.offset)
         if not isEmptyType(prc.typ.sons[0]):
           newFrame.slots[0] = getNullValue(prc.typ.sons[0], prc.info)
         # pass every parameter by var (the language definition allows this):
@@ -620,7 +638,7 @@ proc rawExecute(c: PCtx, start: int, tos: PStackFrame): PNode =
         if isClosure:
           newFrame.slots[rc] = regs[rb].sons[1]
         # allocate the temporaries:
-      for i in rc+ord(isClosure) .. <prc.position:
+        for i in rc+ord(isClosure) .. <prc.offset:
           newFrame.slots[i] = newNode(nkEmpty)
         tos = newFrame
         move(regs, newFrame.slots)
@@ -1054,7 +1072,7 @@ proc evalMacroCall*(module: PSym, n, nOrig: PNode, sym: PSym): PNode =
   let start = genProc(c, sym)
 
   var tos = PStackFrame(prc: sym, comesFrom: 0, next: nil)
-  let maxSlots = sym.position
+  let maxSlots = sym.offset
   newSeq(tos.slots, maxSlots)
   # setup arguments:
   var L = n.safeLen

compiler/vmdef.nim

@@ -175,6 +175,7 @@ type
     module*: PSym
     callsite*: PNode
     mode*: TEvalMode
+    features*: TSandboxFlags
 
   TPosition* = distinct int
 

compiler/vmgen.nim

@@ -11,7 +11,10 @@
 
 import
   unsigned, strutils, ast, astalgo, types, msgs, renderer, vmdef, 
-  trees, intsets, rodread, magicsys
+  trees, intsets, rodread, magicsys, options
+
+when hasFFI:
+  import evalffi
 
 proc codeListing(c: PCtx, result: var string) =
   # first iteration: compute all necessary labels:
@@ -395,9 +398,14 @@ proc genReturn(c: PCtx; n: PNode) =
 proc genCall(c: PCtx; n: PNode; dest: var TDest) =
   if dest < 0 and not isEmptyType(n.typ): dest = getTemp(c, n.typ)
   let x = c.getTempRange(n.len, slotTempUnknown)
+  # varargs need 'opcSetType' for the FFI support:
+  let fntyp = n.sons[0].typ
   for i in 0.. <n.len:
     var r: TRegister = x+i
     c.gen(n.sons[i], r)
+    if i >= fntyp.len:
+      internalAssert tfVarargs in fntyp.flags
+      c.gABx(n, opcSetType, r, c.genType(n.sons[i].typ))
   if dest < 0:
     c.gABC(n, opcIndCall, 0, x, n.len)
   else:
@@ -884,11 +892,24 @@ proc genLit(c: PCtx; n: PNode; dest: var TDest) =
   let lit = genLiteral(c, n)
   c.gABx(n, opc, dest, lit)
 
+proc importcSym(c: PCtx; info: TLineInfo; s: PSym) =
+  when hasFFI:
+    if allowFFI in c.features:
+      c.globals.add(importcSymbol(s))
+      s.position = c.globals.len
+    else:
+      localError(info, errGenerated, "VM is not allowed to 'importc'")
+  else:
+    localError(info, errGenerated,
+               "cannot 'importc' variable at compile time")
+
 proc genRdVar(c: PCtx; n: PNode; dest: var TDest) =
   let s = n.sym
   if sfGlobal in s.flags:
     if dest < 0: dest = c.getTemp(s.typ)
     if s.position == 0:
+      if sfImportc in s.flags: c.importcSym(n.info, s)
+      else:
         c.globals.add(s.ast)
         s.position = c.globals.len
       # XXX var g = codeHere() ?
@@ -1003,6 +1024,8 @@ proc genVarSection(c: PCtx; n: PNode) =
       let s = a.sons[0].sym
       if sfGlobal in s.flags:
         if s.position == 0:
+          if sfImportc in s.flags: c.importcSym(a.info, s)
+          else:
             let sa = if s.ast.isNil: getNullValue(s.typ, a.info) else: s.ast
             c.globals.add(sa)
             s.position = c.globals.len
@@ -1103,6 +1126,7 @@ proc gen(c: PCtx; n: PNode; dest: var TDest) =
     of skVar, skForVar, skTemp, skLet, skParam, skResult:
       genRdVar(c, n, dest)
     of skProc, skConverter, skMacro, skMethod, skIterator:
+      if sfImportc in s.flags: c.importcSym(n.info, s)
       genLit(c, n, dest)
     of skConst:
       gen(c, s.ast, dest)
@@ -1317,11 +1341,11 @@ proc genProc(c: PCtx; s: PSym): int =
     c.patch(procStart)
     c.gABC(body, opcEof, eofInstr.regA)
     c.optimizeJumps(result)
-    s.position = c.prc.maxSlots
+    s.offset = c.prc.maxSlots
     #if s.name.s == "innerProc":
     #  c.echoCode
     #  echo renderTree(body)
     c.prc = oldPrc
   else:
-    c.prc.maxSlots = s.position
+    c.prc.maxSlots = s.offset
     result = x.intVal.int

config/nimrod.cfg

@@ -27,6 +27,7 @@ path="$lib/wrappers/readline"
 path="$lib/wrappers/sdl"
 path="$lib/wrappers/x11"
 path="$lib/wrappers/zip"
+path="$lib/wrappers/libffi"
 path="$lib/windows"
 path="$lib/posix"
 path="$lib/js"

lib/pure/strutils.nim

@@ -1020,8 +1020,8 @@ proc editDistance*(a, b: string): int {.noSideEffect,
 
 # floating point formating:
 
-proc c_sprintf(buf, frmt: CString) {.nodecl, importc: "sprintf", varargs,
+proc c_sprintf(buf, frmt: CString) {.header: "<stdio.h>", importc: "sprintf",
-                                     noSideEffect.}
+                                     varargs, noSideEffect.}
 
 type
   TFloatFormat* = enum ## the different modes of floating point formating

lib/wrappers/libffi/libffi.nim

@@ -27,11 +27,39 @@
 {.deadCodeElim: on.}
 
 when defined(windows):
-  const libffidll* = "libffi.dll"
+  # on Windows we don't use a DLL but instead embed libffi directly:
-elif defined(macosx): 
+  {.pragma: mylib, header: r"ffi.h".}
-  const libffidll* = "libffi.dylib"
+
+  {.compile: r"common\callproc.c".}
+  {.compile: r"common\malloc_closure.c".}
+  {.compile: r"common\raw_api.c".}
+  when defined(vcc):
+    #{.compile: "libffi_msvc\ffi.h".}
+    #<ClInclude: "..\Modules\_ctypes\libffi_msvc\ffi_common.h".}
+    #<ClInclude: "..\Modules\_ctypes\libffi_msvc\fficonfig.h".}
+    #<ClInclude: "..\Modules\_ctypes\libffi_msvc\ffitarget.h".}
+    {.compile: r"msvc\ffi.c".}
+    {.compile: r"msvc\prep_cif.c".}
+    {.compile: r"msvc\win32.c".}
+    {.compile: r"msvc\types.c".}
+    when defined(cpu64):
+      {.compile: r"msvc\win64.asm".}
   else:
-  const libffidll* = "libffi.so"
+    {.compile: r"gcc\ffi.c".}
+    {.compile: r"gcc\prep_cif.c".}
+    {.compile: r"gcc\win32.c".}
+    {.compile: r"gcc\types.c".}
+    {.compile: r"gcc\closures.c".}
+    when defined(cpu64):
+      {.compile: r"gcc\ffi64.c".}
+      {.compile: r"gcc\win64.S".}
+    else:
+      {.compile: r"gcc\win32.S".}
+
+elif defined(macosx):
+  {.pragma: mylib, dynlib: "libffi.dylib".}
+else:
+  {.pragma: mylib, dynlib: "libffi.so".}
 
 type
   TArg* = int
@@ -88,19 +116,19 @@ type
     elements*: ptr ptr TType
 
 var
-  type_void* {.importc: "ffi_type_void", dynlib: libffidll.}: TType
+  type_void* {.importc: "ffi_type_void", mylib.}: TType
-  type_uint8* {.importc: "ffi_type_uint8", dynlib: libffidll.}: TType
+  type_uint8* {.importc: "ffi_type_uint8", mylib.}: TType
-  type_sint8* {.importc: "ffi_type_sint8", dynlib: libffidll.}: TType
+  type_sint8* {.importc: "ffi_type_sint8", mylib.}: TType
-  type_uint16* {.importc: "ffi_type_uint16", dynlib: libffidll.}: TType
+  type_uint16* {.importc: "ffi_type_uint16", mylib.}: TType
-  type_sint16* {.importc: "ffi_type_sint16", dynlib: libffidll.}: TType
+  type_sint16* {.importc: "ffi_type_sint16", mylib.}: TType
-  type_uint32* {.importc: "ffi_type_uint32", dynlib: libffidll.}: TType
+  type_uint32* {.importc: "ffi_type_uint32", mylib.}: TType
-  type_sint32* {.importc: "ffi_type_sint32", dynlib: libffidll.}: TType
+  type_sint32* {.importc: "ffi_type_sint32", mylib.}: TType
-  type_uint64* {.importc: "ffi_type_uint64", dynlib: libffidll.}: TType
+  type_uint64* {.importc: "ffi_type_uint64", mylib.}: TType
-  type_sint64* {.importc: "ffi_type_sint64", dynlib: libffidll.}: TType
+  type_sint64* {.importc: "ffi_type_sint64", mylib.}: TType
-  type_float* {.importc: "ffi_type_float", dynlib: libffidll.}: TType
+  type_float* {.importc: "ffi_type_float", mylib.}: TType
-  type_double* {.importc: "ffi_type_double", dynlib: libffidll.}: TType
+  type_double* {.importc: "ffi_type_double", mylib.}: TType
-  type_pointer* {.importc: "ffi_type_pointer", dynlib: libffidll.}: TType
+  type_pointer* {.importc: "ffi_type_pointer", mylib.}: TType
-  type_longdouble* {.importc: "ffi_type_longdouble", dynlib: libffidll.}: TType
+  type_longdouble* {.importc: "ffi_type_longdouble", mylib.}: TType
 
 type 
   Tstatus* {.size: sizeof(cint).} = enum 
@@ -119,20 +147,18 @@ type
     sint*: TSArg
 
 proc raw_call*(cif: var Tcif; fn: proc () {.cdecl.}; rvalue: pointer; 
-               avalue: ptr TRaw) {.cdecl, importc: "ffi_raw_call", 
+               avalue: ptr TRaw) {.cdecl, importc: "ffi_raw_call", mylib.}
-                                   dynlib: libffidll.}
 proc ptrarray_to_raw*(cif: var Tcif; args: ptr pointer; raw: ptr TRaw) {.cdecl, 
-    importc: "ffi_ptrarray_to_raw", dynlib: libffidll.}
+    importc: "ffi_ptrarray_to_raw", mylib.}
 proc raw_to_ptrarray*(cif: var Tcif; raw: ptr TRaw; args: ptr pointer) {.cdecl, 
-    importc: "ffi_raw_to_ptrarray", dynlib: libffidll.}
+    importc: "ffi_raw_to_ptrarray", mylib.}
-proc raw_size*(cif: var Tcif): int {.cdecl, importc: "ffi_raw_size", 
+proc raw_size*(cif: var Tcif): int {.cdecl, importc: "ffi_raw_size", mylib.}
-                                     dynlib: libffidll.}
 
 proc prep_cif*(cif: var Tcif; abi: TABI; nargs: cuint; rtype: ptr TType; 
                atypes: ptr ptr TType): TStatus {.cdecl, importc: "ffi_prep_cif", 
-    dynlib: libffidll.}
+    mylib.}
 proc call*(cif: var Tcif; fn: proc () {.cdecl.}; rvalue: pointer; 
-           avalue: ptr pointer) {.cdecl, importc: "ffi_call", dynlib: libffidll.}
+           avalue: ptr pointer) {.cdecl, importc: "ffi_call", mylib.}
 
 # the same with an easier interface:
 type
@@ -141,9 +167,9 @@ type
 
 proc prep_cif*(cif: var Tcif; abi: TABI; nargs: cuint; rtype: ptr TType; 
                atypes: TParamList): TStatus {.cdecl, importc: "ffi_prep_cif",
-    dynlib: libffidll.}
+    mylib.}
 proc call*(cif: var Tcif; fn, rvalue: pointer;
-           avalue: TArgList) {.cdecl, importc: "ffi_call", dynlib: libffidll.}
+           avalue: TArgList) {.cdecl, importc: "ffi_call", mylib.}
 
 # Useful for eliminating compiler warnings 
 ##define FFI_FN(f) ((void (*)(void))f)