From 7c39bd1f5beb2182a6863ad48bd6d874041a53c1 Mon Sep 17 00:00:00 2001 From: Siu Kwan Lam Date: Fri, 8 Feb 2013 16:03:29 -0600 Subject: [PATCH] Refactor to allow sub-namespace declaration --- newbinding/Makefile | 10 +- newbinding/capsule.py | 2 +- newbinding/gen/binding.py | 225 ++++++++++++++++----- newbinding/gen/codegen.py | 6 + newbinding/gen/gen.py | 135 +++---------- newbinding/include/llvm_binding/binding.h | 12 +- newbinding/src/CallingConv.py | 5 +- newbinding/src/EngineBuilder.py | 7 +- newbinding/src/Intrinsics.py | 17 +- newbinding/src/Support/CodeGen.py | 26 +-- newbinding/src/Support/DynamicLibrary.py | 6 + newbinding/src/Target/TargetLibraryInfo.py | 5 +- newbinding/src/namespace.py | 5 +- 13 files changed, 264 insertions(+), 197 deletions(-) create mode 100644 newbinding/src/Support/DynamicLibrary.py diff --git a/newbinding/Makefile b/newbinding/Makefile index 0e7924f..9de1fb0 100644 --- a/newbinding/Makefile +++ b/newbinding/Makefile @@ -5,18 +5,18 @@ all: _api.so _capsule.so _api.so _capsule.so: api.cpp capsule.cpp $(PYTHON) setup.py build_ext --inplace -api.cpp api.py: src/*.py include/llvm_binding/*.h gen/gen.py gen/binding.py +api.cpp api/__init__.py: src/*.py include/llvm_binding/*.h gen/gen.py gen/binding.py $(PYTHON) gen/gen.py api src clean: cleantemp rm -f _api.so _capsule.so + rm -rf api + py_clean cleantemp: - rm -f api.cpp api.py - rm -f src/*.pyc src/Transforms/*.pyc src/ADT/*.pyc src/Assembly/*.pyc src/CodeGen/*.pyc src/ExecutionEngine/*.pyc src/Target/*.pyc src/Transforms/*.pyc + rm -f api.cpp - -check: _api.so api.py +check: _api.so api/__init__.py $(PYTHON) test_binding.py @PHONY: all clean check diff --git a/newbinding/capsule.py b/newbinding/capsule.py index e29948d..d98a23d 100644 --- a/newbinding/capsule.py +++ b/newbinding/capsule.py @@ -182,7 +182,7 @@ def downcast(obj, cls): fname = 'downcast_%s_to_%s' % (fromty, toty) fname = fname.replace('::', '_') try: - caster = getattr(_api, fname) + caster = getattr(_api.downcast, fname) except AttributeError: fmt = "Downcast from %s to %s is not supported" raise TypeError(fmt % (fromty, toty)) diff --git a/newbinding/gen/binding.py b/newbinding/gen/binding.py index c1e1d1e..601ac6e 100644 --- a/newbinding/gen/binding.py +++ b/newbinding/gen/binding.py @@ -1,19 +1,157 @@ import inspect, textwrap import functools import codegen as cg +import os _rank = 0 namespaces = {} RESERVED = frozenset(['None']) -class Namespace(object): - def __init__(self, name): - self.name = name +def makedir(directory): + if not os.path.exists(directory): + os.makedirs(directory) + +class SubModule(object): + def __init__(self): + self.methods = [] self.enums = [] self.classes = [] - self.functions = [] + self.namespaces = [] + self.attrs = [] self.includes = set() + + def aggregate_includes(self): + includes = set(self.includes) + for unit in self.iter_all(): + if isinstance(unit, SubModule): + includes |= unit.aggregate_includes() + else: + includes |= unit.includes + return includes + + def aggregate_downcast(self): + dclist = [] + for cls in self.classes: + for bcls in cls.downcastables: + from_to = bcls.fullname, cls.fullname + name = 'downcast_%s_to_%s' % tuple(map(cg.mangle, from_to)) + fn = Function(namespaces[''], name, ptr(cls), ptr(bcls)) + dclist.append((from_to, fn)) + for ns in self.namespaces: + dclist.extend(ns.aggregate_downcast()) + return dclist + + def iter_all(self): + for fn in self.methods: + yield fn + for cls in self.classes: + yield cls + for enum in self.enums: + yield enum + for attr in self.attrs: + yield attr + for ns in self.namespaces: + yield ns + + + def generate_method_table(self, println): + writer = cg.CppCodeWriter(println) + writer.println('static') + writer.println('PyMethodDef meth_%s[] = {' % cg.mangle(self.fullname)) + with writer.indent(): + fmt = '{ "%(name)s", (PyCFunction)%(func)s, METH_VARARGS, NULL },' + for meth in self.methods: + name = meth.name + func = meth.c_name + writer.println(fmt % locals()) + for enumkind in self.enums: + for enum in enumkind.value_names: + name = enum + func = enumkind.c_name(enum) + writer.println(fmt % locals()) + for attr in self.attrs: + # getter + name = attr.getter_name + func = attr.getter_c_name + writer.println(fmt % locals()) + # setter + name = attr.setter_name + func = attr.setter_c_name + writer.println(fmt % locals()) + writer.println('{ NULL },') + writer.println('};') + writer.println() + + def generate_downcasts(self, println): + for ((fromty, toty), fn) in self.downcastlist: + name = fn.name + fmt = ''' +static +%(toty)s* %(name)s(%(fromty)s* arg) +{ + return typecast<%(toty)s>::from(arg); +} + ''' + println(fmt % locals()) + + fn.generate_cpp(println) + + def generate_cpp(self, println, extras=()): + for unit in self.iter_all(): + unit.generate_cpp(println) + self.generate_method_table(println) + self.generate_submodule_table(println, extras=extras) + + def generate_submodule_table(self, println, extras=()): + writer = cg.CppCodeWriter(println) + writer.println('static') + name = cg.mangle(self.fullname) + writer.println('SubModuleEntry submodule_%(name)s[] = {' % locals()) + with writer.indent(): + for cls in self.classes: + name = cls.name + table = cg.mangle(cls.fullname) + writer.println('{ "%(name)s", meth_%(table)s, NULL },' % + locals()) + for ns in self.namespaces: + name = ns.localname + table = cg.mangle(ns.fullname) + fmt = '{ "%(name)s", meth_%(table)s, submodule_%(table)s },' + writer.println(fmt % locals()) + for name, table in extras: + writer.println('{ "%(name)s", %(table)s, NULL },' % locals()) + writer.println('{ NULL }') + writer.println('};') + writer.println('') + + def generate_py(self, rootdir='.', name=''): + name = name or self.localname + if self.namespaces: # should make new directory + path = os.path.join(rootdir, name) + makedir(path) + filepath = os.path.join(path, '__init__.py') + else: + filepath = os.path.join(rootdir, '%s.py' % name) + with open(filepath, 'w') as pyfile: + println = cg.wrap_println_from_file(pyfile) + println('import _api, capsule') + for ns in self.namespaces: + println('from . import %s' % ns.localname) + println() + for unit in self.iter_all(): + if not isinstance(unit, Namespace): + writer = cg.PyCodeWriter(println) + unit.compile_py(writer) + + for ns in self.namespaces: + ns.generate_py(rootdir=path) + + +class Namespace(SubModule): + def __init__(self, name): + SubModule.__init__(self) + self.name = name namespaces[name] = self def Class(self, *bases): @@ -23,12 +161,12 @@ class Namespace(object): def Function(self, *args): fn = Function(self, *args) - self.functions.append(fn) + self.methods.append(fn) return fn def CustomFunction(self, *args): fn = CustomFunction(self, *args) - self.functions.append(fn) + self.methods.append(fn) return fn def Enum(self, name, *value_names): @@ -36,12 +174,23 @@ class Namespace(object): enum.parent = self enum.name = name self.enums.append(enum) + assert name not in vars(self), 'Duplicated' + setattr(self, name, enum) return enum + def Namespace(self, name): + ns = Namespace('::'.join([self.name, name])) + self.namespaces.append(ns) + return ns + @property def fullname(self): return self.name + @property + def localname(self): + return self.name.rsplit('::', 1)[-1] + def __str__(self): return self.name @@ -80,18 +229,15 @@ ConstCharPtr = BuiltinTypes('const char*') PyObjectPtr = BuiltinTypes('PyObject*') PyObjectPtr.format='O' -class Class(_Type): +class Class(SubModule, _Type): format = 'O' def __init__(self, ns, *bases): + SubModule.__init__(self) self.ns = ns self.bases = bases self._is_defined = False - self.methods = [] self.pymethods = [] - self.enums = [] - self.attrs = [] - self.includes = set() self.downcastables = set() def __call__(self, defn): @@ -110,6 +256,7 @@ class Class(_Type): self.enums.append(v) v.name = k v.parent = self + assert k not in vars(self), "Duplicated: %s" % k setattr(self, k, v) elif isinstance(v, Attr): self.attrs.append(v) @@ -133,43 +280,6 @@ class Class(_Type): self.downcastables.add(i) return self - def compile_cpp(self, writer): - # generate methods - for meth in self.methods: - meth.compile_cpp(writer) - for enum in self.enums: - enum.compile_cpp(writer) - for attr in self.attrs: - attr.compile_cpp(writer) - - # generate method table - writer.println('static') - writer.println('PyMethodDef %s[] = {' % cg.mangle(self.fullname)) - with writer.indent(): - fmt = '{ "%(name)s", (PyCFunction)%(func)s, METH_VARARGS, NULL },' - for meth in self.methods: - name = meth.name - func = meth.c_name - writer.println(fmt % locals()) - for enumkind in self.enums: - for enum in enumkind.value_names: - name = enum - func = enumkind.c_name(enum) - writer.println(fmt % locals()) - for attr in self.attrs: - # getter - name = attr.getter_name - func = attr.getter_c_name - writer.println(fmt % locals()) - # setter - name = attr.setter_name - func = attr.setter_c_name - writer.println(fmt % locals()) - - writer.println('{ NULL },') - writer.println('};') - writer.println() - def compile_py(self, writer): clsname = self.name bases = 'capsule.Wrapper' @@ -256,6 +366,9 @@ class Enum(object): def c_name(self, enum): return cg.mangle("%s_%s_%s" % (self.parent, self.name, enum)) + def generate_cpp(self, println): + self.compile_cpp(cg.CppCodeWriter(println)) + def compile_cpp(self, writer): for enum in self.value_names: with writer.py_function(self.c_name(enum)): @@ -322,6 +435,9 @@ class Method(object): def __str__(self): return self.fullname + def generate_cpp(self, println): + self.compile_cpp(cg.CppCodeWriter(println)) + def compile_cpp(self, writer): with writer.py_function(self.c_name): if len(self.signatures) == 1: @@ -454,9 +570,12 @@ class Function(Method): with writer.function(self.name, varargs='args') as varargs: unwrapped = writer.unwrap_many(varargs) self.process_ownedptr_args(writer, unwrapped) - func = self.name - ret = writer.call('_api.%s' % func, - varargs=unwrapped) + if '::' in self.parent.name: + ns = self.parent.name.split('::', 1)[-1].replace('::', '.') + func = '.'.join([ns, self.name]) + else: + func = self.name + ret = writer.call('_api.%s' % func, varargs=unwrapped) wrapped = writer.wrap(ret, self.is_return_ownedptr()) writer.return_value(wrapped) writer.println() @@ -630,6 +749,7 @@ class Attr(object): def __init__(self, getter, setter): self.getter = getter self.setter = setter + self.includes = set() @property def fullname(self): @@ -658,6 +778,9 @@ class Attr(object): def setter_c_name(self): return cg.mangle('%s_set' % self.fullname) + def generate_cpp(self, println): + self.compile_cpp(cg.CppCodeWriter(println)) + def compile_cpp(self, writer): # getter with writer.py_function(self.getter_c_name): diff --git a/newbinding/gen/codegen.py b/newbinding/gen/codegen.py index 5a31e38..dd499d6 100644 --- a/newbinding/gen/codegen.py +++ b/newbinding/gen/codegen.py @@ -4,6 +4,12 @@ NULL = 'NULL' _symbols = set() +def wrap_println_from_file(file): + def println(s=''): + file.write(s) + file.write('\n') + return println + def indent(println): def _println(s=''): println("%s%s" % (' '* 4, s)) diff --git a/newbinding/gen/gen.py b/newbinding/gen/gen.py index 89399f3..bb7eab3 100644 --- a/newbinding/gen/gen.py +++ b/newbinding/gen/gen.py @@ -14,7 +14,7 @@ PyInit_%(module)s(void) { PyObject *module = create_python_module("%(module)s", %(methtable)s); if (module) { -if (populate_submodules(module, submodules)) +if (populate_submodules(module, submodule_llvm)) return module; } return NULL; @@ -27,7 +27,7 @@ init%(module)s(void) { PyObject *module = create_python_module("%(module)s", %(methtable)s); if (module) { -populate_submodules(module, submodules); +populate_submodules(module, submodule_llvm); } } #endif @@ -49,65 +49,25 @@ def populate_headers(println): println('#include "%s"' % inc) println() - -def wrap_println_from_file(file): - def println(s=''): - file.write(s) - file.write('\n') - return println - def main(): outputfilename = sys.argv[1] entry_modname = sys.argv[2] sys.path += [os.path.dirname(os.curdir)] entry_module = __import__(entry_modname) - units = [] - for ns in namespaces.values(): - print 'namespace', ns - for fn in ns.functions: - print fn - units.append(fn) - for cls in ns.classes: - print cls - units.append(cls) - for enum in ns.enums: - print enum - units.append(enum) - - # add extra stuffs - downcastlist = [] - ## add downcast - for cls in units: - if isinstance(cls, Class): - for bcls in cls.downcastables: - from_to = bcls.fullname, cls.fullname - name = 'downcast_%s_to_%s' % tuple(map(codegen.mangle, from_to)) - fn = Function(namespaces[''], name, ptr(cls), ptr(bcls)) - downcastlist.append((from_to, fn)) - units.append(fn) - + rootns = namespaces['llvm'] # Generate C++ source with open('%s.cpp' % outputfilename, 'w') as cppfile: - println = wrap_println_from_file(cppfile) - - # extra headers - populate_headers(println) - - # required headers - includes = set() - for ns in namespaces.values(): - includes |= ns.includes - for u in units: - includes |= u.includes - - for inc in includes: + println = codegen.wrap_println_from_file(cppfile) + populate_headers(println) # extra headers + # print all includes + for inc in rootns.aggregate_includes(): println('#include "%s"' % inc) println() - - # generate downcast - for ((fromty, toty), fn) in downcastlist: + # print all downcast + downcast_fns = rootns.aggregate_downcast() + for ((fromty, toty), fn) in downcast_fns: name = fn.name fmt = ''' static @@ -117,65 +77,28 @@ static } ''' println(fmt % locals()) + + fn.generate_cpp(println) - # write methods and method tables - for u in units: - writer = codegen.CppCodeWriter(println) - u.compile_cpp(writer) - else: - del writer - - # write function table - writer = codegen.CppCodeWriter(println) - writer.println('static') - writer.println('PyMethodDef methtable[] = {') - with writer.indent(): - fmt = '{ "%(name)s", (PyCFunction)%(func)s, METH_VARARGS, NULL },' - for u in units: - if isinstance(u, Function): - name = u.name - func = u.c_name - writer.println(fmt % locals()) - for u in units: - if isinstance(u, Enum): - for enum in u.value_names: - name = enum - func = u.c_name(enum) - writer.println(fmt % locals()) - - writer.println('{ NULL },') - writer.println('};') - writer.println() - del writer - - - # write submodule table - writer = codegen.CppCodeWriter(println) - writer.println('static') - writer.println('SubModuleEntry submodules[] = {') - with writer.indent(): - for cls in units: - if isinstance(cls, Class): - name = cls.name - table = codegen.mangle(cls.fullname) - writer.println('{ "%(name)s", %(table)s },' % locals()) - writer.println('{ "extra", extra_methodtable },') - writer.println('{ NULL }') - writer.println('};') - writer.println('') - del writer - - println(extension_entry % {'module': '_api', 'methtable': 'methtable'}) + println('static') + println('PyMethodDef downcast_methodtable[] = {') + fmt = '{ "%(name)s", (PyCFunction)%(func)s, METH_VARARGS, NULL },' + for _, fn in downcast_fns: + name = fn.name + func = fn.c_name + println(fmt % locals()) + println('{ NULL }') + println('};') + println() + # generate submodule + rootns.generate_cpp(println, extras=[('extra', 'extra_methodtable'), + ('downcast', 'downcast_methodtable')]) + println(extension_entry % {'module' : '_api', + 'methtable': 'meth_llvm'}) # Generate Python source - with open('%s.py' % outputfilename, 'w') as pyfile: - println = wrap_println_from_file(pyfile) - println('import _api, capsule') - println() - for u in units: - writer = codegen.PyCodeWriter(println) - u.compile_py(writer) - + rootns.generate_py(rootdir='.', name='api') + if __name__ == '__main__': main() diff --git a/newbinding/include/llvm_binding/binding.h b/newbinding/include/llvm_binding/binding.h index 58a8ad0..68af615 100644 --- a/newbinding/include/llvm_binding/binding.h +++ b/newbinding/include/llvm_binding/binding.h @@ -63,13 +63,23 @@ create_python_submodule(PyObject* parent, const char* name, struct SubModuleEntry{ const char* name; PyMethodDef* methtable; + SubModuleEntry* submodule; }; static int populate_submodules(PyObject* parent, SubModuleEntry* entries){ for(SubModuleEntry* iter = entries; iter->name; ++iter){ - if (!create_python_submodule(parent, iter->name, iter->methtable)) + PyObject* submodule = create_python_submodule(parent, + iter->name, + iter->methtable); + if (!submodule){ return 0; + } else if (iter->submodule){ + // Recursively populate submodules + if (!populate_submodules(submodule, iter->submodule)) { + return 0; + } + } } return 1; } diff --git a/newbinding/src/CallingConv.py b/newbinding/src/CallingConv.py index 6d0baea..71b4ad3 100644 --- a/newbinding/src/CallingConv.py +++ b/newbinding/src/CallingConv.py @@ -1,9 +1,8 @@ from binding import * from namespace import llvm -@llvm.Class() # actually a namespace -class CallingConv: - ID = Enum(''' +CallingConv = llvm.Namespace('CallingConv') +ID = CallingConv.Enum('ID', ''' C, Fast, Cold, GHC, FirstTargetCC, X86_StdCall, X86_FastCall, ARM_APCS, ARM_AAPCS, ARM_AAPCS_VFP, MSP430_INTR, X86_ThisCall, PTX_Kernel, PTX_Device, MBLAZE_INTR, MBLAZE_SVOL, SPIR_FUNC, diff --git a/newbinding/src/EngineBuilder.py b/newbinding/src/EngineBuilder.py index 4b68a25..c3feafe 100644 --- a/newbinding/src/EngineBuilder.py +++ b/newbinding/src/EngineBuilder.py @@ -10,9 +10,8 @@ from ADT.Triple import Triple EngineBuilder = llvm.Class() -@llvm.Class() # a fake class (actually a namespace) -class EngineKind: - Kind = Enum('JIT', 'Interpreter') +EngineKind = llvm.Namespace('EngineKind') +Kind = EngineKind.Enum('Kind', 'JIT', 'Interpreter') @EngineBuilder class EngineBuilder: @@ -22,7 +21,7 @@ class EngineBuilder: def _setter(*args): return Method(ref(EngineBuilder), *args) - setEngineKind = _setter(EngineKind.Kind) + setEngineKind = _setter(Kind) setJITMemoryManager = _setter(ptr(JITMemoryManager)) setErrorStr = CustomMethod('EngineBuilder_setErrorStr', diff --git a/newbinding/src/Intrinsics.py b/newbinding/src/Intrinsics.py index 0f1bcb4..5314112 100644 --- a/newbinding/src/Intrinsics.py +++ b/newbinding/src/Intrinsics.py @@ -5,13 +5,12 @@ from Module import Module from Function import Function -Intrinsic = llvm.Class() # fake class (actually a namespace) +Intrinsic = llvm.Namespace('Intrinsic') -@Intrinsic -class Intrinsic: - getDeclaration = CustomStaticMethod('Intrinsic_getDeclaration', - PyObjectPtr, # Function* - ptr(Module), - cast(int, Unsigned), # intrinsic id - PyObjectPtr, # list of Type - ).require_only(2) +getDeclaration = Intrinsic.CustomFunction('getDeclaration', + 'Intrinsic_getDeclaration', + PyObjectPtr, # Function* + ptr(Module), + cast(int, Unsigned), # intrinsic id + PyObjectPtr, # list of Type + ).require_only(2) diff --git a/newbinding/src/Support/CodeGen.py b/newbinding/src/Support/CodeGen.py index 013caa1..3bc6f9d 100644 --- a/newbinding/src/Support/CodeGen.py +++ b/newbinding/src/Support/CodeGen.py @@ -1,20 +1,20 @@ from binding import * from ..namespace import llvm -@llvm.Class() -class Reloc: - Model = Enum('Default', 'Static', 'PIC_', 'DynamicNoPIC') -@llvm.Class() -class CodeModel: - Model = Enum('Default', 'JITDefault', 'Small', 'Kernel', 'Medium', 'Large') +Reloc = llvm.Namespace('Reloc') +Reloc.Enum('Model', + 'Default', 'Static', 'PIC_', 'DynamicNoPIC') -@llvm.Class() -class TLSModel: - Model = Enum('GeneralDynamic', 'LocalDynamic', 'InitialExec', 'LocalExec') +CodeModel = llvm.Namespace('CodeModel') +CodeModel.Enum('Model', + 'Default', 'JITDefault', 'Small', 'Kernel', 'Medium', 'Large') -@llvm.Class() -class CodeGenOpt: - 'Actually a namespace' - Level = Enum('None', 'Less', 'Default', 'Aggressive') +TLSModel = llvm.Namespace('TLSModel') +TLSModel.Enum('Model', + 'GeneralDynamic', 'LocalDynamic', 'InitialExec', 'LocalExec') + +CodeGenOpt = llvm.Namespace('CodeGenOpt') +CodeGenOpt.Enum('Level', + 'None', 'Less', 'Default', 'Aggressive') diff --git a/newbinding/src/Support/DynamicLibrary.py b/newbinding/src/Support/DynamicLibrary.py new file mode 100644 index 0000000..7f661d3 --- /dev/null +++ b/newbinding/src/Support/DynamicLibrary.py @@ -0,0 +1,6 @@ +from binding import * +from ..namespace import sys + +DynamicLibrary = sys.Namespace('DynamicLibrary') +#LoadLibraryPermanently + diff --git a/newbinding/src/Target/TargetLibraryInfo.py b/newbinding/src/Target/TargetLibraryInfo.py index b90c619..d98b00b 100644 --- a/newbinding/src/Target/TargetLibraryInfo.py +++ b/newbinding/src/Target/TargetLibraryInfo.py @@ -6,9 +6,8 @@ from ..ADT.StringRef import StringRef TargetLibraryInfo = llvm.Class(ImmutablePass) -@llvm.Class() -class LibFunc: - Func = Enum(''' +LibFunc = llvm.Namespace('LibFunc') +LibFunc.Enum('Func', ''' ZdaPv, ZdlPv, Znaj, ZnajRKSt9nothrow_t, Znam, ZnamRKSt9nothrow_t, Znwj, ZnwjRKSt9nothrow_t, Znwm, ZnwmRKSt9nothrow_t, cxa_atexit, cxa_guard_abort, diff --git a/newbinding/src/namespace.py b/newbinding/src/namespace.py index 3a081af..f833af8 100644 --- a/newbinding/src/namespace.py +++ b/newbinding/src/namespace.py @@ -1,4 +1,7 @@ from binding import * -llvm = Namespace('llvm') default = Namespace('') + +llvm = Namespace('llvm') +sys = llvm.Namespace('sys') +