diff --git a/.depend b/.depend new file mode 100644 index 0000000..d1ad466 --- /dev/null +++ b/.depend @@ -0,0 +1,21 @@ +find_source.cmo : h_program-lang/skip_code.cmi \ + lang_cpp/parsing/lib_parsing_cpp.cmi lang_c/parsing/lib_parsing_c.cmi \ + commons/common.cmi find_source.cmi +find_source.cmx : h_program-lang/skip_code.cmx \ + lang_cpp/parsing/lib_parsing_cpp.cmx lang_c/parsing/lib_parsing_c.cmx \ + commons/common.cmx find_source.cmi +find_source.cmi : commons/common.cmi +main.cmo : lang_cpp/parsing/test_parsing_cpp.cmi \ + lang_c/parsing/test_parsing_c.cmi external/jsonwheel/json_io.cmi \ + external/jsonwheel/json_in.cmo generators/nim/generate_nim.cmi \ + lang_cpp/parsing/flag_parsing_cpp.cmo globals/config_pfff.cmo \ + commons/common2.cmi commons/common.cmi +main.cmx : lang_cpp/parsing/test_parsing_cpp.cmx \ + lang_c/parsing/test_parsing_c.cmx external/jsonwheel/json_io.cmx \ + external/jsonwheel/json_in.cmx generators/nim/generate_nim.cmx \ + lang_cpp/parsing/flag_parsing_cpp.cmx globals/config_pfff.cmx \ + commons/common2.cmx commons/common.cmx +main_test.cmo : generators/nim/unit_generate_nim.cmi commons/oUnit.cmi \ + globals/config_pfff.cmo commons/common2.cmi commons/common.cmi +main_test.cmx : generators/nim/unit_generate_nim.cmx commons/oUnit.cmx \ + globals/config_pfff.cmx commons/common2.cmx commons/common.cmx diff --git a/.gitignore b/.gitignore index 7479501..b5ea600 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,8 @@ *.cmo *.cma *.cmi +*.cmti +*.cmt *.a *.o *.cmx @@ -19,9 +21,6 @@ _build/ setup.data setup.log -# Merlin configuring file for Vim and Emacs -.merlin - # Swap [._]*.s[a-v][a-z] [._]*.sw[a-p] diff --git a/.merlin b/.merlin index 2720f95..70c7871 100644 --- a/.merlin +++ b/.merlin @@ -14,3 +14,5 @@ S lang_c/** B lang_c/** S lang_cpp/** B lang_cpp/** +S generators/** +B generators/** diff --git a/Makefile b/Makefile index 5af1b5b..ed9f811 100644 --- a/Makefile +++ b/Makefile @@ -19,7 +19,7 @@ TARGET=pfff PROGS=pfff -#PROGS+=pfff_test +PROGS+=pfff_test OPTPROGS= $(PROGS:=.opt) @@ -59,7 +59,8 @@ BASICLIBS=commons/commons.cma \ globals/lib.cma \ h_program-lang/lib.cma \ lang_cpp/parsing/lib.cma \ - lang_c/parsing/lib.cma + lang_c/parsing/lib.cma \ + generators/nim/lib.cma # commons/commons_features.cma \ @@ -75,6 +76,7 @@ LIBS= commons/commons.cma \ h_program-lang/lib.cma \ lang_cpp/parsing/lib.cma \ lang_c/parsing/lib.cma \ + generators/nim/lib.cma MAKESUBDIRS=commons commons_core \ $(JSONDIR) \ @@ -83,6 +85,7 @@ MAKESUBDIRS=commons commons_core \ h_program-lang \ lang_cpp/parsing \ lang_c/parsing \ + generators/nim INCLUDEDIRS=$(MAKESUBDIRS) diff --git a/data/cpp_stdlib/macros.h b/data/cpp_stdlib/macros.h new file mode 100644 index 0000000..fdc41b6 --- /dev/null +++ b/data/cpp_stdlib/macros.h @@ -0,0 +1,137 @@ +// **************************************************************************** +// Prelude +// **************************************************************************** +// This file must be passed to the -macros option of the C/C++ parsers in pfff. +// It should be stored as a default in Flag_parsing_cpp.macros_h + +// **************************************************************************** +// MacroString +// **************************************************************************** + +/* String macros are normally handled quite well by the LALR(k) technique, + * but sometimes it's not enough. For instance with 'XX YY', it could + * be considered as a declaration with XX being a typedef, so we would + * have an ambiguity. So by adding a few special cases (e.g. KERN_WARNING + * for the linux kernel), we can catch more string-macros. The idea is + * that we need to have at least one witness, a string to recognize + * a sequence of strings. + */ +//ex: #define KERN_WARNING "WARNING" + +/* EX_TABLE & co. + * + * Replaced by a string. We can't put everything as comment + * because it can be part of an expression where we wait for + * something, where we wait for a string. So at least we + * must keep the EX_TABLE token and transform it as a string. + * + * normally not needed if have good stringification of macro + * but those macros are sometimes used multiple times + * as in EX_TABLE(0b) EX_TABLE(1b) and we don't detect + * it well yet. + */ + +//ex: #define EX_TABLE(x) "TOTO" + +// **************************************************************************** +// MacroIterator +// **************************************************************************** + +// foreach +#define FOR_EACH(a,b) for(;;) + +// **************************************************************************** +// MacroDeclarator +// **************************************************************************** + +// static DECLARATOR(x); +// LIST_HEAD stuff +// (used in qemu, freebsd) + +// **************************************************************************** +// MacroStmt +// **************************************************************************** + +// with or without parameters, but no ';' + +// **************************************************************************** +// MacroField +// **************************************************************************** + +// **************************************************************************** +// MacroInitializer +// **************************************************************************** + +// no PTVirg or shortcut for array designator + +// **************************************************************************** +// MacroAttributes +// **************************************************************************** + +//ex: #define __init YACFE_ATTRIBUTE +// TODO: could perhaps generalize via "__.*" + +// linkage + +// params (IN, OUT) + +// windows: WINAPI, STDCALL, ... +#define WINAPI +#define __cdecl + +// **************************************************************************** +// MacroKeywordAlias +// **************************************************************************** + +// const, often defined via macro for backward compatibility with old compiler +// I guess. + +// private/public + +#define __asm__ asm +#define __attribute __attribute__ +#define __volatile__ volatile +#define __restrict __restrict__ + +#define CONST const + +// for xv6 +#define __asm asm +#define __volatile volatile + +// **************************************************************************** +// Prototype +// **************************************************************************** + +// __P + +// PARAMS + +// **************************************************************************** +// Declarator +// **************************************************************************** + +/* cf gcc-linux.h + * A trick to suppress uninitialized variable warning without generating any + * code + */ + +// #define uninitialized_var(x) x = x +// as in u16 uninitialized_var(ioboard_type); /* GCC be quiet */ + +// **************************************************************************** +// Misc +// **************************************************************************** + +// LIST_HEAD +// GENTEST, GENHEADER +// structure +// MACHINE_START +// higher order, ASSERTCMP +// parts of stuff, start of stuff +// begin end wierd, as in C++ firefox and NS_DECLARE_BEGIN/END with code in +// the middle +// testcase, reflexivity on name +// IDENT in sparse, wierd case +// if-like macros +//#define G_BEGIN_DECLS diff --git a/data/cpp_stdlib/other_macros.h b/data/cpp_stdlib/other_macros.h new file mode 100644 index 0000000..62cbde1 --- /dev/null +++ b/data/cpp_stdlib/other_macros.h @@ -0,0 +1,63 @@ +// TODO: now you can have a pfff_macros.h local per-project file (like I have +// a local per-projet skip list), so distribute this to the relevant +// places in software-src/ + +// **************************************************************************** +// git stuff +// **************************************************************************** + +#define internal_function +#define NOINLINE +#define FORCEINLINE +#define THROWSPEC +#define NORETURN +#define NORETURN_PTR +#define MAYBE_UNUSED + +#define __MINGW_NOTHROW +#define WSAAPI + +#define for_each_string_list_item(a,b) for(;;) + +// **************************************************************************** +// sparse stuff +// **************************************************************************** + +//YACFE_MACROITERATOR +#define FOR_EACH_PTR(a,b) for(;;) +#define FOR_EACH_PTR_NOTAG(a,b) for(;;) +#define FOR_EACH_PTR_REVERSE(a,b) for(;;) +#define RECURSE_PTR_REVERSE(a,b) for(;;) +//#define DO_END_FOR_EACH_REVERSE(a,b,c,d) + +#define __percpu +#define __user + +#define SENTINEL_ATTR +#define FORMAT_ATTR(a) +#define NORETURN_ATTR + +// **************************************************************************** +// Boost +// **************************************************************************** +#define BOOST_AUTO_TEST_CASE(a) void a() +#define BOOST_STATIC_ASSERT(a) static int x +#define BOOST_FOR_EACH(a,b) for(;;) +#define BOOST_FOREACH(a,b) for(;;) + +// **************************************************************************** +// KDE +// **************************************************************************** +#define Q_OBJECT +#define Q_CLASSINFO(a,b) + +// **************************************************************************** +// Jansson +// **************************************************************************** + +#define JSON_INLINE +#define json_object_foreach(a,b,c) for(;;) + +// **************************************************************************** +// Plan9 +// **************************************************************************** diff --git a/data/cpp_stdlib/readme.txt b/data/cpp_stdlib/readme.txt new file mode 100644 index 0000000..6260dd7 --- /dev/null +++ b/data/cpp_stdlib/readme.txt @@ -0,0 +1,2 @@ +You can also cp -a /usr/include/c++/.../ and /usr/include/boost wherever +your project is. diff --git a/generators/.merlin b/generators/.merlin new file mode 100644 index 0000000..2ba6169 --- /dev/null +++ b/generators/.merlin @@ -0,0 +1 @@ +REC diff --git a/generators/nim/.depend b/generators/nim/.depend new file mode 100644 index 0000000..c4b319a --- /dev/null +++ b/generators/nim/.depend @@ -0,0 +1,14 @@ +generate_nim.cmo : ../../h_program-lang/parse_info.cmi \ + ../../lang_cpp/parsing/parse_cpp.cmi \ + ../../lang_cpp/parsing/flag_parsing_cpp.cmo ../../commons/common.cmi \ + ../../lang_cpp/parsing/ast_cpp.cmo generate_nim.cmi +generate_nim.cmx : ../../h_program-lang/parse_info.cmx \ + ../../lang_cpp/parsing/parse_cpp.cmx \ + ../../lang_cpp/parsing/flag_parsing_cpp.cmx ../../commons/common.cmx \ + ../../lang_cpp/parsing/ast_cpp.cmx generate_nim.cmi +generate_nim.cmi : ../../commons/common.cmi +unit_generate_nim.cmo : ../../commons/oUnit.cmi generate_nim.cmi \ + ../../commons/common.cmi unit_generate_nim.cmi +unit_generate_nim.cmx : ../../commons/oUnit.cmx generate_nim.cmx \ + ../../commons/common.cmx unit_generate_nim.cmi +unit_generate_nim.cmi : ../../commons/oUnit.cmi diff --git a/generators/nim/.merlin b/generators/nim/.merlin new file mode 100644 index 0000000..2ba6169 --- /dev/null +++ b/generators/nim/.merlin @@ -0,0 +1 @@ +REC diff --git a/generators/nim/Makefile b/generators/nim/Makefile new file mode 100644 index 0000000..09fb833 --- /dev/null +++ b/generators/nim/Makefile @@ -0,0 +1,45 @@ +TOP=../.. +############################################################################## +# Variables +############################################################################## +TARGET=lib + +-include $(TOP)/Makefile.config + +SRC= generate_nim.ml unit_generate_nim.ml + +SYSLIBS= str.cma unix.cma + +LIBS=$(TOP)/commons/lib.cma \ + $(TOP)/h_program-lang/lib.cma \ + +INCLUDEDIRS= \ + $(TOP)/commons \ + $(TOP)/commons_core \ + $(TOP)/globals \ + $(TOP)/h_program-lang \ + $(TOP)/lang_cpp/parsing + +############################################################################## +# Generic variables +############################################################################## + +-include $(TOP)/Makefile.common + +############################################################################## +# Top rules +############################################################################## +all:: $(TARGET).cma +all.opt:: $(TARGET).cmxa + +$(TARGET).cma: $(OBJS) + $(OCAMLC) -a -o $(TARGET).cma $(OBJS) + +$(TARGET).cmxa: $(OPTOBJS) $(LIBS:.cma=.cmxa) + $(OCAMLOPT) -a -o $(TARGET).cmxa $(OPTOBJS) + +$(TARGET).top: $(OBJS) $(LIBS) + $(OCAMLMKTOP) -o $(TARGET).top $(SYSLIBS) $(LIBS) $(OBJS) + +clean:: + rm -f $(TARGET).top diff --git a/lang_cpp/parsing/test_dump_nim.ml b/generators/nim/generate_nim.ml similarity index 98% rename from lang_cpp/parsing/test_dump_nim.ml rename to generators/nim/generate_nim.ml index a744b34..15167d6 100644 --- a/lang_cpp/parsing/test_dump_nim.ml +++ b/generators/nim/generate_nim.ml @@ -845,7 +845,7 @@ and process_declaration = | NameSpaceAnon ((v1, v2)) -> (*let v1 = vof_tok v1 and v2 = vof_brace (Ocaml.vof_list vof_declaration_sequencable) v2 - in Ocaml.VSum (("NameSpaceAnon", [ v1; v2 ]))*) + in Ocaml.VSu:m (("NameSpaceAnon", [ v1; v2 ]))*) "" | EmptyDef def -> process_token def | DeclTodo -> "# TODO" @@ -864,8 +864,19 @@ and process_toplevel = function let iter_ast ast = List.map process_toplevel ast -let test_dump_nim file = - Parse_cpp.init_defs !Flag.macros_h; - let ast = Parse_cpp.parse_program file in +let generate_nim cfile macro_files = + Parse_cpp.init_defs cfile; + List.iter Parse_cpp.add_defs macro_files; + let ast = Parse_cpp.parse_program cfile in let res = iter_ast ast in - List.iter pr res + String.concat "\n" res + +let test_gen_nim file = + let macro_list = [!Flag.macros_h] in + let nim_str = generate_nim file macro_list in + pr nim_str + +let actions () = [ + "-generate-nim", " ", + Common.mk_action_1_arg test_gen_nim; +] diff --git a/generators/nim/generate_nim.mli b/generators/nim/generate_nim.mli new file mode 100644 index 0000000..03b8f3c --- /dev/null +++ b/generators/nim/generate_nim.mli @@ -0,0 +1,4 @@ +val test_gen_nim : + Common.filename -> unit + +val actions : unit -> Common.cmdline_actions diff --git a/generators/nim/unit_generate_nim.ml b/generators/nim/unit_generate_nim.ml new file mode 100644 index 0000000..7bb4072 --- /dev/null +++ b/generators/nim/unit_generate_nim.ml @@ -0,0 +1,11 @@ +open Common +open OUnit +open Generate_nim + + +let unittest = + "generating_nim" >::: [ + "Test function declarations" >:: (fun () -> + pr "None" + ) + ] diff --git a/generators/nim/unit_generate_nim.mli b/generators/nim/unit_generate_nim.mli new file mode 100644 index 0000000..8b798e0 --- /dev/null +++ b/generators/nim/unit_generate_nim.mli @@ -0,0 +1 @@ +val unittest: OUnit.test diff --git a/globals/config_pfff.ml b/globals/config_pfff.ml index 579b7e2..5f039d7 100644 --- a/globals/config_pfff.ml +++ b/globals/config_pfff.ml @@ -2,7 +2,7 @@ let version = "0.29" let path = try (Sys.getenv "PFFF_HOME") - with Not_found->"/usr/local/share/pfff" + with Not_found->"./" let std_xxx = ref (Filename.concat path "xxx.yyy") diff --git a/lang_cpp/parsing/.depend b/lang_cpp/parsing/.depend index 0e1c43d..20019dd 100644 --- a/lang_cpp/parsing/.depend +++ b/lang_cpp/parsing/.depend @@ -135,25 +135,18 @@ pp_token.cmx : token_views_cpp.cmx token_helpers_cpp.cmx token_cpp.cmx \ ../../commons/common2.cmx ../../commons/common.cmx ast_cpp.cmx \ pp_token.cmi pp_token.cmi : token_views_cpp.cmi parser_cpp.cmi ../../commons/common.cmi -test_dump_nim.cmo : ../../h_program-lang/parse_info.cmi parse_cpp.cmi \ - flag_parsing_cpp.cmo ../../commons/common.cmi ast_cpp.cmo \ - test_dump_nim.cmi -test_dump_nim.cmx : ../../h_program-lang/parse_info.cmx parse_cpp.cmx \ - flag_parsing_cpp.cmx ../../commons/common.cmx ast_cpp.cmx \ - test_dump_nim.cmi -test_dump_nim.cmi : ../../commons/common.cmi test_parsing_cpp.cmo : token_views_cpp.cmi token_views_context.cmi \ - token_helpers_cpp.cmi test_dump_nim.cmi \ - ../../h_program-lang/skip_code.cmi parsing_hacks_cpp.cmi parser_cpp.cmi \ - ../../h_program-lang/parse_info.cmi parse_cpp.cmi ../../commons/ocaml.cmi \ + token_helpers_cpp.cmi ../../h_program-lang/skip_code.cmi \ + parsing_hacks_cpp.cmi parser_cpp.cmi ../../h_program-lang/parse_info.cmi \ + parse_cpp.cmi ../../commons/ocaml.cmi \ ../../h_program-lang/meta_ast_generic.cmi meta_ast_cpp.cmi \ lib_parsing_cpp.cmi flag_parsing_cpp.cmo ../../commons_core/console.cmi \ ../../commons/common.cmi ../../h_program-lang/ast_fuzzy.cmi ast_cpp.cmo \ test_parsing_cpp.cmi test_parsing_cpp.cmx : token_views_cpp.cmx token_views_context.cmx \ - token_helpers_cpp.cmx test_dump_nim.cmx \ - ../../h_program-lang/skip_code.cmx parsing_hacks_cpp.cmx parser_cpp.cmx \ - ../../h_program-lang/parse_info.cmx parse_cpp.cmx ../../commons/ocaml.cmx \ + token_helpers_cpp.cmx ../../h_program-lang/skip_code.cmx \ + parsing_hacks_cpp.cmx parser_cpp.cmx ../../h_program-lang/parse_info.cmx \ + parse_cpp.cmx ../../commons/ocaml.cmx \ ../../h_program-lang/meta_ast_generic.cmx meta_ast_cpp.cmx \ lib_parsing_cpp.cmx flag_parsing_cpp.cmx ../../commons_core/console.cmx \ ../../commons/common.cmx ../../h_program-lang/ast_fuzzy.cmx ast_cpp.cmx \ diff --git a/lang_cpp/parsing/Makefile b/lang_cpp/parsing/Makefile index 5f0e80c..9f27a75 100644 --- a/lang_cpp/parsing/Makefile +++ b/lang_cpp/parsing/Makefile @@ -19,7 +19,6 @@ SRC= flag_parsing_cpp.ml \ parsing_hacks.ml \ parsing_recovery_cpp.ml \ parse_cpp.ml \ - test_dump_nim.ml \ test_parsing_cpp.ml unit_parsing_cpp.ml SYSLIBS= str.cma unix.cma diff --git a/lang_cpp/parsing/flag_parsing_cpp.ml b/lang_cpp/parsing/flag_parsing_cpp.ml index e5a1905..3ca4abb 100644 --- a/lang_cpp/parsing/flag_parsing_cpp.ml +++ b/lang_cpp/parsing/flag_parsing_cpp.ml @@ -3,7 +3,7 @@ (* types *) (*****************************************************************************) -type language = +type language = | C | Cplusplus @@ -11,7 +11,7 @@ type language = (* macros *) (*****************************************************************************) -let macros_h = +let macros_h = ref (Filename.concat Config_pfff.path "/data/cpp_stdlib/macros.h") let cmdline_flags_macrofile () = [ diff --git a/lang_cpp/parsing/test_dump_nim.mli b/lang_cpp/parsing/test_dump_nim.mli deleted file mode 100644 index 4bb2d5a..0000000 --- a/lang_cpp/parsing/test_dump_nim.mli +++ /dev/null @@ -1,2 +0,0 @@ -val test_dump_nim : - Common.filename -> unit diff --git a/lang_cpp/parsing/test_parsing_cpp.ml b/lang_cpp/parsing/test_parsing_cpp.ml index 2ce914e..c0f0cf6 100644 --- a/lang_cpp/parsing/test_parsing_cpp.ml +++ b/lang_cpp/parsing/test_parsing_cpp.ml @@ -99,9 +99,6 @@ let actions () = [ "-dump_cpp", " ", Common.mk_action_1_arg test_dump_cpp; - "-dump_nim", " ", - Common.mk_action_1_arg Test_dump_nim.test_dump_nim; - "-dump_cpp_full", " ", Common.mk_action_1_arg test_dump_cpp_full; "-dump_cpp_view", " ", diff --git a/main.ml b/main.ml index f74c950..a0396ba 100644 --- a/main.ml +++ b/main.ml @@ -67,6 +67,7 @@ let all_actions () = pfff_extra_actions() @ Test_parsing_c.actions()@ Test_parsing_cpp.actions()@ + Generate_nim.actions()@ (* Test_analyze_cpp.actions () ++ diff --git a/main_test.ml b/main_test.ml new file mode 100644 index 0000000..3ecd2d9 --- /dev/null +++ b/main_test.ml @@ -0,0 +1,127 @@ +open OUnit +open Common2 + +(*****************************************************************************) +(* Flags *) +(*****************************************************************************) + +(* In addition to flags that can be tweaked via -xxx options (cf the + * full list of options in the "the options" section below), this + * program also depends on external files ? + *) +let verbose = ref false + +(* action mode *) +let action = ref "" + +let test regexp = + + (* There is no reflection in OCaml so the unit test framework OUnit requires + * us to explicitely build the test suites (which is not that bad). + *) + let tests = + "all" >::: [ + Unit_generate_nim.unittest; + ] + in + let suite = + if regexp = "all" + then tests + else + let paths = + OUnit.test_case_paths tests +> List.map OUnit.string_of_path in + let keep = paths + +> List.filter (fun path -> + pr2 path; + path =~ (".*" ^ regexp)) + in + Common2.some (OUnit.test_filter keep tests) + in + + let results = OUnit.run_test_tt ~verbose:!verbose suite in + let has_an_error = + results +> List.exists (function + | OUnit.RSuccess _ | OUnit.RSkip _ | OUnit.RTodo _ -> false + | OUnit.RFailure _ | OUnit.RError _ -> true + ) + in + raise (Common.UnixExit (if has_an_error then 1 else 0)) + +let main_action x = + test x + + + +let all_actions () = + [] + +let options () = [ + "-verbose", Arg.Set verbose, + " "; + ] @ + Common.options_of_actions action (all_actions()) @ + Common2.cmdline_flags_devel () @ + Common2.cmdline_flags_other () @ + [ + "-version", Arg.Unit (fun () -> + pr2 (spf "pfff (test) version: %s" Config_pfff.version); + exit 0; + ), + " guess what"; + (* this can not be factorized in Common *) + "-date", Arg.Unit (fun () -> + pr2 "version: $Date: 2008/10/26 00:44:57 $"; + raise (Common.UnixExit 0) + ), + " guess what"; + ] + +(*****************************************************************************) +(* Main entry point *) +(*****************************************************************************) + +let main () = + + Gc.set {(Gc.get ()) with Gc.stack_limit = 1000 * 1024 * 1024}; + + let usage_msg = + "Usage: " ^ Common2.basename Sys.argv.(0) ^ + " [options] " ^ "\n" ^ "Options are:" + in + (* does side effect on many global flags *) + let args = Common.parse_options (options()) usage_msg Sys.argv in + + (* must be done after Arg.parse, because Common.profile is set by it *) + Common.profile_code "Main total" (fun () -> + + (match args with + + (* --------------------------------------------------------- *) + (* actions, useful to debug subpart *) + (* --------------------------------------------------------- *) + | xs when List.mem !action (Common.action_list (all_actions())) -> + Common.do_action !action xs (all_actions()) + + | _ when not (Common.null_string !action) -> + failwith ("unrecognized action or wrong params: " ^ !action) + + (* --------------------------------------------------------- *) + (* main entry *) + (* --------------------------------------------------------- *) + | [x] -> + main_action x + + (* --------------------------------------------------------- *) + (* empty entry *) + (* --------------------------------------------------------- *) + | _ -> + Common.usage usage_msg (options()); + failwith "too few or too many arguments" + ) + ) + +(*****************************************************************************) +let _ = + Common.main_boilerplate (fun () -> + main (); + )