Refactor to allow sub-namespace declaration

newbinding/Makefile

@@ -5,18 +5,18 @@ all: _api.so _capsule.so
 _api.so _capsule.so: api.cpp capsule.cpp
 	$(PYTHON) setup.py build_ext --inplace
 
-api.cpp api.py: src/*.py include/llvm_binding/*.h gen/gen.py gen/binding.py
+api.cpp api/__init__.py: src/*.py include/llvm_binding/*.h gen/gen.py gen/binding.py
 	$(PYTHON) gen/gen.py api src
 	
 clean: cleantemp
 	rm -f _api.so _capsule.so
+	rm -rf api	
+	py_clean
 	
 cleantemp:
-	rm -f api.cpp api.py
+	rm -f api.cpp
-	rm -f src/*.pyc src/Transforms/*.pyc src/ADT/*.pyc src/Assembly/*.pyc src/CodeGen/*.pyc src/ExecutionEngine/*.pyc src/Target/*.pyc src/Transforms/*.pyc
 
-
+check: _api.so api/__init__.py
-check: _api.so api.py
 	$(PYTHON) test_binding.py
 	
 @PHONY: all clean check

newbinding/capsule.py

@@ -182,7 +182,7 @@ def downcast(obj, cls):
     fname = 'downcast_%s_to_%s' % (fromty, toty)
     fname = fname.replace('::', '_')
     try:
-        caster = getattr(_api, fname)
+        caster = getattr(_api.downcast, fname)
     except AttributeError:
         fmt = "Downcast from %s to %s is not supported"
         raise TypeError(fmt % (fromty, toty))

newbinding/gen/binding.py

@@ -1,19 +1,157 @@
 import inspect, textwrap
 import functools
 import codegen as cg
+import os
 
 _rank = 0
 namespaces = {}
 
 RESERVED = frozenset(['None'])
 
-class Namespace(object):
+def makedir(directory):
-    def __init__(self, name):
+    if not os.path.exists(directory):
-        self.name = name
+        os.makedirs(directory)
+
+class SubModule(object):
+    def __init__(self):
+        self.methods = []
         self.enums = []
         self.classes = []
-        self.functions = []
+        self.namespaces = []
+        self.attrs = []
         self.includes = set()
+
+    def aggregate_includes(self):
+        includes = set(self.includes)
+        for unit in self.iter_all():
+            if isinstance(unit, SubModule):
+                includes |= unit.aggregate_includes()
+            else:
+                includes |= unit.includes
+        return includes
+
+    def aggregate_downcast(self):
+        dclist = []
+        for cls in self.classes:
+            for bcls in cls.downcastables:
+                from_to = bcls.fullname, cls.fullname
+                name = 'downcast_%s_to_%s' % tuple(map(cg.mangle, from_to))
+                fn = Function(namespaces[''], name, ptr(cls), ptr(bcls))
+                dclist.append((from_to, fn))
+        for ns in self.namespaces:
+            dclist.extend(ns.aggregate_downcast())
+        return dclist
+
+    def iter_all(self):
+        for fn in self.methods:
+            yield fn
+        for cls in self.classes:
+            yield cls
+        for enum in self.enums:
+            yield enum
+        for attr in self.attrs:
+            yield attr
+        for ns in self.namespaces:
+            yield ns
+
+
+    def generate_method_table(self, println):
+        writer = cg.CppCodeWriter(println)
+        writer.println('static')
+        writer.println('PyMethodDef meth_%s[] = {' % cg.mangle(self.fullname))
+        with writer.indent():
+            fmt = '{ "%(name)s", (PyCFunction)%(func)s, METH_VARARGS, NULL },'
+            for meth in self.methods:
+                name = meth.name
+                func = meth.c_name
+                writer.println(fmt % locals())
+            for enumkind in self.enums:
+                for enum in enumkind.value_names:
+                    name = enum
+                    func = enumkind.c_name(enum)
+                    writer.println(fmt % locals())
+            for attr in self.attrs:
+                # getter
+                name = attr.getter_name
+                func = attr.getter_c_name
+                writer.println(fmt % locals())
+                # setter
+                name = attr.setter_name
+                func = attr.setter_c_name
+                writer.println(fmt % locals())
+            writer.println('{ NULL },')
+        writer.println('};')
+        writer.println()
+
+    def generate_downcasts(self, println):
+        for ((fromty, toty), fn) in self.downcastlist:
+            name = fn.name
+            fmt = '''
+static
+%(toty)s* %(name)s(%(fromty)s* arg)
+{
+    return typecast<%(toty)s>::from(arg);
+}
+                '''
+            println(fmt % locals())
+
+            fn.generate_cpp(println)
+
+    def generate_cpp(self, println, extras=()):
+        for unit in self.iter_all():
+            unit.generate_cpp(println)
+        self.generate_method_table(println)
+        self.generate_submodule_table(println, extras=extras)
+
+    def generate_submodule_table(self, println, extras=()):
+        writer = cg.CppCodeWriter(println)
+        writer.println('static')
+        name = cg.mangle(self.fullname)
+        writer.println('SubModuleEntry submodule_%(name)s[] = {' % locals())
+        with writer.indent():
+            for cls in self.classes:
+                name = cls.name
+                table = cg.mangle(cls.fullname)
+                writer.println('{ "%(name)s", meth_%(table)s, NULL },' %
+                               locals())
+            for ns in self.namespaces:
+                name = ns.localname
+                table = cg.mangle(ns.fullname)
+                fmt = '{ "%(name)s", meth_%(table)s, submodule_%(table)s },'
+                writer.println(fmt % locals())
+            for name, table in extras:
+                writer.println('{ "%(name)s", %(table)s, NULL },' % locals())
+            writer.println('{ NULL }')
+        writer.println('};')
+        writer.println('')
+
+    def generate_py(self, rootdir='.', name=''):
+        name = name or self.localname
+        if self.namespaces: # should make new directory
+            path = os.path.join(rootdir, name)
+            makedir(path)
+            filepath = os.path.join(path, '__init__.py')
+        else:
+            filepath = os.path.join(rootdir, '%s.py' % name)
+        with open(filepath, 'w') as pyfile:
+            println = cg.wrap_println_from_file(pyfile)
+            println('import _api, capsule')
+            for ns in self.namespaces:
+                println('from . import %s' % ns.localname)
+            println()
+            for unit in self.iter_all():
+                if not isinstance(unit, Namespace):
+                    writer = cg.PyCodeWriter(println)
+                    unit.compile_py(writer)
+
+        for ns in self.namespaces:
+            ns.generate_py(rootdir=path)
+
+
+class Namespace(SubModule):
+    def __init__(self, name):
+        SubModule.__init__(self)
+        self.name = name
         namespaces[name] = self
 
     def Class(self, *bases):
@@ -23,12 +161,12 @@ class Namespace(object):
 
     def Function(self, *args):
         fn = Function(self, *args)
-        self.functions.append(fn)
+        self.methods.append(fn)
         return fn
 
     def CustomFunction(self, *args):
         fn = CustomFunction(self, *args)
-        self.functions.append(fn)
+        self.methods.append(fn)
         return fn
 
     def Enum(self, name, *value_names):
@@ -36,12 +174,23 @@ class Namespace(object):
         enum.parent = self
         enum.name = name
         self.enums.append(enum)
+        assert name not in vars(self), 'Duplicated'
+        setattr(self, name, enum)
         return enum
 
+    def Namespace(self, name):
+        ns = Namespace('::'.join([self.name, name]))
+        self.namespaces.append(ns)
+        return ns
+
     @property
     def fullname(self):
         return self.name
 
+    @property
+    def localname(self):
+        return self.name.rsplit('::', 1)[-1]
+
     def __str__(self):
         return self.name
 
@@ -80,18 +229,15 @@ ConstCharPtr = BuiltinTypes('const char*')
 PyObjectPtr = BuiltinTypes('PyObject*')
 PyObjectPtr.format='O'
 
-class Class(_Type):
+class Class(SubModule, _Type):
     format = 'O'
 
     def __init__(self, ns, *bases):
+        SubModule.__init__(self)
         self.ns = ns
         self.bases = bases
         self._is_defined = False
-        self.methods = []
         self.pymethods = []
-        self.enums = []
-        self.attrs = []
-        self.includes = set()
         self.downcastables = set()
 
     def __call__(self, defn):
@@ -110,6 +256,7 @@ class Class(_Type):
                 self.enums.append(v)
                 v.name = k
                 v.parent = self
+                assert k not in vars(self), "Duplicated: %s" % k
                 setattr(self, k, v)
             elif isinstance(v, Attr):
                 self.attrs.append(v)
@@ -133,43 +280,6 @@ class Class(_Type):
                         self.downcastables.add(i)
         return self
 
-    def compile_cpp(self, writer):
-        # generate methods
-        for meth in self.methods:
-            meth.compile_cpp(writer)
-        for enum in self.enums:
-            enum.compile_cpp(writer)
-        for attr in self.attrs:
-            attr.compile_cpp(writer)
-
-        # generate method table
-        writer.println('static')
-        writer.println('PyMethodDef %s[] = {' % cg.mangle(self.fullname))
-        with writer.indent():
-            fmt = '{ "%(name)s", (PyCFunction)%(func)s, METH_VARARGS, NULL },'
-            for meth in self.methods:
-                name = meth.name
-                func = meth.c_name
-                writer.println(fmt % locals())
-            for enumkind in self.enums:
-                for enum in enumkind.value_names:
-                    name = enum
-                    func = enumkind.c_name(enum)
-                    writer.println(fmt % locals())
-            for attr in self.attrs:
-                # getter
-                name = attr.getter_name
-                func = attr.getter_c_name
-                writer.println(fmt % locals())
-                # setter
-                name = attr.setter_name
-                func = attr.setter_c_name
-                writer.println(fmt % locals())
-
-            writer.println('{ NULL },')
-        writer.println('};')
-        writer.println()
-
     def compile_py(self, writer):
         clsname = self.name
         bases = 'capsule.Wrapper'
@@ -256,6 +366,9 @@ class Enum(object):
     def c_name(self, enum):
         return cg.mangle("%s_%s_%s" % (self.parent, self.name, enum))
 
+    def generate_cpp(self, println):
+        self.compile_cpp(cg.CppCodeWriter(println))
+
     def compile_cpp(self, writer):
         for enum in self.value_names:
             with writer.py_function(self.c_name(enum)):
@@ -322,6 +435,9 @@ class Method(object):
     def __str__(self):
         return self.fullname
 
+    def generate_cpp(self, println):
+        self.compile_cpp(cg.CppCodeWriter(println))
+
     def compile_cpp(self, writer):
         with writer.py_function(self.c_name):
             if len(self.signatures) == 1:
@@ -454,9 +570,12 @@ class Function(Method):
         with writer.function(self.name, varargs='args') as varargs:
             unwrapped = writer.unwrap_many(varargs)
             self.process_ownedptr_args(writer, unwrapped)
-            func = self.name
+            if '::' in self.parent.name:
-            ret = writer.call('_api.%s' % func,
+                ns = self.parent.name.split('::', 1)[-1].replace('::', '.')
-                              varargs=unwrapped)
+                func = '.'.join([ns, self.name])
+            else:
+                func = self.name
+            ret = writer.call('_api.%s' % func, varargs=unwrapped)
             wrapped = writer.wrap(ret, self.is_return_ownedptr())
             writer.return_value(wrapped)
         writer.println()
@@ -630,6 +749,7 @@ class Attr(object):
     def __init__(self, getter, setter):
         self.getter = getter
         self.setter = setter
+        self.includes = set()
 
     @property
     def fullname(self):
@@ -658,6 +778,9 @@ class Attr(object):
     def setter_c_name(self):
         return cg.mangle('%s_set' % self.fullname)
 
+    def generate_cpp(self, println):
+        self.compile_cpp(cg.CppCodeWriter(println))
+
     def compile_cpp(self, writer):
         # getter
         with writer.py_function(self.getter_c_name):

newbinding/gen/codegen.py

@@ -4,6 +4,12 @@ NULL = 'NULL'
 
 _symbols = set()
 
+def wrap_println_from_file(file):
+    def println(s=''):
+        file.write(s)
+        file.write('\n')
+    return println
+
 def indent(println):
     def _println(s=''):
         println("%s%s" % (' '* 4, s))

newbinding/gen/gen.py

@@ -14,7 +14,7 @@ PyInit_%(module)s(void)
 {
 PyObject *module = create_python_module("%(module)s", %(methtable)s);
 if (module) {
-if (populate_submodules(module, submodules))
+if (populate_submodules(module, submodule_llvm))
 return module;
 }
 return NULL;
@@ -27,7 +27,7 @@ init%(module)s(void)
 {
 PyObject *module = create_python_module("%(module)s", %(methtable)s);
 if (module) {
-populate_submodules(module, submodules);
+populate_submodules(module, submodule_llvm);
 }
 }
 #endif
@@ -49,65 +49,25 @@ def populate_headers(println):
         println('#include "%s"' % inc)
     println()
 
-
-def wrap_println_from_file(file):
-    def println(s=''):
-        file.write(s)
-        file.write('\n')
-    return println
-
 def main():
     outputfilename = sys.argv[1]
     entry_modname = sys.argv[2]
     sys.path += [os.path.dirname(os.curdir)]
     entry_module = __import__(entry_modname)
 
-    units = []
+    rootns = namespaces['llvm']
-    for ns in namespaces.values():
-        print 'namespace', ns
-        for fn in ns.functions:
-            print fn
-            units.append(fn)
-        for cls in ns.classes:
-            print cls
-            units.append(cls)
-        for enum in ns.enums:
-            print enum
-            units.append(enum)
-
-    # add extra stuffs
-    downcastlist = []
-    ## add downcast
-    for cls in units:
-        if isinstance(cls, Class):
-            for bcls in cls.downcastables:
-                from_to = bcls.fullname, cls.fullname
-                name = 'downcast_%s_to_%s' % tuple(map(codegen.mangle, from_to))
-                fn = Function(namespaces[''], name, ptr(cls), ptr(bcls))
-                downcastlist.append((from_to, fn))
-                units.append(fn)
-
 
     # Generate C++ source
     with open('%s.cpp' % outputfilename, 'w') as cppfile:
-        println = wrap_println_from_file(cppfile)
+        println = codegen.wrap_println_from_file(cppfile)
-
+        populate_headers(println)                  # extra headers
-        # extra headers
+        # print all includes
-        populate_headers(println)
+        for inc in rootns.aggregate_includes():
-
-        # required headers
-        includes = set()
-        for ns in namespaces.values():
-            includes |= ns.includes
-        for u in units:
-            includes |= u.includes
-
-        for inc in includes:
             println('#include "%s"' % inc)
         println()
-
+        # print all downcast
-        # generate downcast
+        downcast_fns = rootns.aggregate_downcast()
-        for ((fromty, toty), fn) in downcastlist:
+        for ((fromty, toty), fn) in downcast_fns:
             name = fn.name
             fmt = '''
 static
@@ -118,63 +78,26 @@ static
 '''
             println(fmt % locals())
     
-        # write methods and method tables
+            fn.generate_cpp(println)
-        for u in units:
-            writer = codegen.CppCodeWriter(println)
-            u.compile_cpp(writer)
-        else:
-            del writer
 
-        # write function table
+        println('static')
-        writer = codegen.CppCodeWriter(println)
+        println('PyMethodDef downcast_methodtable[] = {')
-        writer.println('static')
+        fmt = '{ "%(name)s", (PyCFunction)%(func)s, METH_VARARGS, NULL },'
-        writer.println('PyMethodDef methtable[] = {')
+        for _, fn in downcast_fns:
-        with writer.indent():
+            name = fn.name
-            fmt = '{ "%(name)s", (PyCFunction)%(func)s, METH_VARARGS, NULL },'
+            func = fn.c_name
-            for u in units:
+            println(fmt % locals())
-                if isinstance(u, Function):
+        println('{ NULL }')
-                    name = u.name
+        println('};')
-                    func = u.c_name
+        println()
-                    writer.println(fmt % locals())
+        # generate submodule
-            for u in units:
+        rootns.generate_cpp(println, extras=[('extra', 'extra_methodtable'),
-                if isinstance(u, Enum):
+                                             ('downcast', 'downcast_methodtable')])
-                    for enum in u.value_names:
+        println(extension_entry % {'module'   : '_api',
-                        name = enum
+                                   'methtable': 'meth_llvm'})
-                        func = u.c_name(enum)
-                        writer.println(fmt % locals())
-
-            writer.println('{ NULL },')
-        writer.println('};')
-        writer.println()
-        del writer
-        
-
-        # write submodule table
-        writer = codegen.CppCodeWriter(println)
-        writer.println('static')
-        writer.println('SubModuleEntry submodules[] = {')
-        with writer.indent():
-            for cls in units:
-                if isinstance(cls, Class):
-                    name = cls.name
-                    table = codegen.mangle(cls.fullname)
-                    writer.println('{ "%(name)s", %(table)s },' % locals())
-            writer.println('{ "extra", extra_methodtable },')
-            writer.println('{ NULL }')
-        writer.println('};')
-        writer.println('')
-        del writer
-
-        println(extension_entry % {'module': '_api', 'methtable': 'methtable'})
 
     # Generate Python source
-    with open('%s.py' % outputfilename, 'w') as pyfile:
+    rootns.generate_py(rootdir='.', name='api')
-        println = wrap_println_from_file(pyfile)
-        println('import _api, capsule')
-        println()
-        for u in units:
-            writer = codegen.PyCodeWriter(println)
-            u.compile_py(writer)
 
 
 if __name__ == '__main__':

newbinding/include/llvm_binding/binding.h

@@ -63,13 +63,23 @@ create_python_submodule(PyObject* parent, const char* name,
 struct SubModuleEntry{
     const char* name;
     PyMethodDef* methtable;
+    SubModuleEntry* submodule;
 };
 
 static
 int populate_submodules(PyObject* parent, SubModuleEntry* entries){
     for(SubModuleEntry* iter = entries; iter->name; ++iter){
-        if (!create_python_submodule(parent, iter->name, iter->methtable))
+        PyObject* submodule = create_python_submodule(parent,
+                                                      iter->name,
+                                                      iter->methtable);
+        if (!submodule){
             return 0;
+        } else if (iter->submodule){
+            // Recursively populate submodules
+            if (!populate_submodules(submodule, iter->submodule)) {
+                return 0;
+            }
+        }
     }
     return 1;
 }

newbinding/src/CallingConv.py

@@ -1,9 +1,8 @@
 from binding import *
 from namespace import llvm
 
-@llvm.Class()  # actually a namespace
+CallingConv = llvm.Namespace('CallingConv')
-class CallingConv:
+ID = CallingConv.Enum('ID', '''
-    ID = Enum('''
         C, Fast, Cold, GHC, FirstTargetCC, X86_StdCall, X86_FastCall,
         ARM_APCS, ARM_AAPCS, ARM_AAPCS_VFP, MSP430_INTR, X86_ThisCall,
         PTX_Kernel, PTX_Device, MBLAZE_INTR, MBLAZE_SVOL, SPIR_FUNC,

newbinding/src/EngineBuilder.py

@@ -10,9 +10,8 @@ from ADT.Triple import Triple
 
 EngineBuilder = llvm.Class()
 
-@llvm.Class() # a fake class (actually a namespace)
+EngineKind = llvm.Namespace('EngineKind')
-class EngineKind:
+Kind = EngineKind.Enum('Kind', 'JIT', 'Interpreter')
-    Kind = Enum('JIT', 'Interpreter')
 
 @EngineBuilder
 class EngineBuilder:
@@ -22,7 +21,7 @@ class EngineBuilder:
     def _setter(*args):
         return Method(ref(EngineBuilder), *args)
 
-    setEngineKind = _setter(EngineKind.Kind)
+    setEngineKind = _setter(Kind)
     setJITMemoryManager = _setter(ptr(JITMemoryManager))
 
     setErrorStr = CustomMethod('EngineBuilder_setErrorStr',

newbinding/src/Intrinsics.py

@@ -5,13 +5,12 @@ from Module import Module
 from Function import Function
 
 
-Intrinsic = llvm.Class() # fake class (actually a namespace)
+Intrinsic = llvm.Namespace('Intrinsic')
 
-@Intrinsic
+getDeclaration = Intrinsic.CustomFunction('getDeclaration',
-class Intrinsic:
+                                          'Intrinsic_getDeclaration',
-    getDeclaration = CustomStaticMethod('Intrinsic_getDeclaration',
+                                          PyObjectPtr,          # Function*
-                                        PyObjectPtr,          # Function*
+                                          ptr(Module),
-                                        ptr(Module),
+                                          cast(int, Unsigned),  # intrinsic id
-                                        cast(int, Unsigned),  # intrinsic id
+                                          PyObjectPtr,          # list of Type
-                                        PyObjectPtr,          # list of Type
+                                          ).require_only(2)
-                                        ).require_only(2)

newbinding/src/Support/CodeGen.py

@@ -1,20 +1,20 @@
 from binding import *
 from ..namespace import llvm
 
-@llvm.Class()
-class Reloc:
-    Model = Enum('Default', 'Static', 'PIC_', 'DynamicNoPIC')
 
-@llvm.Class()
+Reloc = llvm.Namespace('Reloc')
-class CodeModel:
+Reloc.Enum('Model',
-    Model = Enum('Default', 'JITDefault', 'Small', 'Kernel', 'Medium', 'Large')
+           'Default', 'Static', 'PIC_', 'DynamicNoPIC')
 
-@llvm.Class()
+CodeModel = llvm.Namespace('CodeModel')
-class TLSModel:
+CodeModel.Enum('Model',
-    Model = Enum('GeneralDynamic', 'LocalDynamic', 'InitialExec', 'LocalExec')
+               'Default', 'JITDefault', 'Small', 'Kernel', 'Medium', 'Large')
 
-@llvm.Class()
+TLSModel = llvm.Namespace('TLSModel')
-class CodeGenOpt:
+TLSModel.Enum('Model',
-    'Actually a namespace'
+             'GeneralDynamic', 'LocalDynamic', 'InitialExec', 'LocalExec')
-    Level = Enum('None', 'Less', 'Default', 'Aggressive')
+
+CodeGenOpt = llvm.Namespace('CodeGenOpt')
+CodeGenOpt.Enum('Level',
+                'None', 'Less', 'Default', 'Aggressive')
 

newbinding/src/Support/DynamicLibrary.py

@@ -0,0 +1,6 @@
+from binding import *
+from ..namespace import sys
+
+DynamicLibrary = sys.Namespace('DynamicLibrary')
+#LoadLibraryPermanently
+

newbinding/src/Target/TargetLibraryInfo.py

@@ -6,9 +6,8 @@ from ..ADT.StringRef import StringRef
 
 TargetLibraryInfo = llvm.Class(ImmutablePass)
 
-@llvm.Class()
+LibFunc = llvm.Namespace('LibFunc')
-class LibFunc:
+LibFunc.Enum('Func',  '''
-    Func = Enum('''
                 ZdaPv, ZdlPv, Znaj, ZnajRKSt9nothrow_t,
                 Znam, ZnamRKSt9nothrow_t, Znwj, ZnwjRKSt9nothrow_t,
                 Znwm, ZnwmRKSt9nothrow_t, cxa_atexit, cxa_guard_abort,

newbinding/src/namespace.py

@@ -1,4 +1,7 @@
 from binding import *
 
-llvm = Namespace('llvm')
 default = Namespace('')
+
+llvm = Namespace('llvm')
+sys = llvm.Namespace('sys')
+