Add testing infrastructure and move out gen nim

.depend

@@ -0,0 +1,21 @@
+find_source.cmo : h_program-lang/skip_code.cmi \
+    lang_cpp/parsing/lib_parsing_cpp.cmi lang_c/parsing/lib_parsing_c.cmi \
+    commons/common.cmi find_source.cmi
+find_source.cmx : h_program-lang/skip_code.cmx \
+    lang_cpp/parsing/lib_parsing_cpp.cmx lang_c/parsing/lib_parsing_c.cmx \
+    commons/common.cmx find_source.cmi
+find_source.cmi : commons/common.cmi
+main.cmo : lang_cpp/parsing/test_parsing_cpp.cmi \
+    lang_c/parsing/test_parsing_c.cmi external/jsonwheel/json_io.cmi \
+    external/jsonwheel/json_in.cmo generators/nim/generate_nim.cmi \
+    lang_cpp/parsing/flag_parsing_cpp.cmo globals/config_pfff.cmo \
+    commons/common2.cmi commons/common.cmi
+main.cmx : lang_cpp/parsing/test_parsing_cpp.cmx \
+    lang_c/parsing/test_parsing_c.cmx external/jsonwheel/json_io.cmx \
+    external/jsonwheel/json_in.cmx generators/nim/generate_nim.cmx \
+    lang_cpp/parsing/flag_parsing_cpp.cmx globals/config_pfff.cmx \
+    commons/common2.cmx commons/common.cmx
+main_test.cmo : generators/nim/unit_generate_nim.cmi commons/oUnit.cmi \
+    globals/config_pfff.cmo commons/common2.cmi commons/common.cmi
+main_test.cmx : generators/nim/unit_generate_nim.cmx commons/oUnit.cmx \
+    globals/config_pfff.cmx commons/common2.cmx commons/common.cmx

.gitignore

@@ -2,6 +2,8 @@
 *.cmo
 *.cma
 *.cmi
+*.cmti
+*.cmt
 *.a
 *.o
 *.cmx
@@ -19,9 +21,6 @@ _build/
 setup.data
 setup.log
 
-# Merlin configuring file for Vim and Emacs
-.merlin
-
 # Swap
 [._]*.s[a-v][a-z]
 [._]*.sw[a-p]

.merlin

@@ -14,3 +14,5 @@ S lang_c/**
 B lang_c/**
 S lang_cpp/**
 B lang_cpp/**
+S generators/**
+B generators/**

Makefile

@@ -19,7 +19,7 @@ TARGET=pfff
 
 PROGS=pfff
 
-#PROGS+=pfff_test
+PROGS+=pfff_test
 
 OPTPROGS= $(PROGS:=.opt)
 
@@ -59,7 +59,8 @@ BASICLIBS=commons/commons.cma \
  globals/lib.cma \
  h_program-lang/lib.cma \
  lang_cpp/parsing/lib.cma \
- lang_c/parsing/lib.cma
+ lang_c/parsing/lib.cma \
+ generators/nim/lib.cma
 
 #       commons/commons_features.cma \
 
@@ -75,6 +76,7 @@ LIBS= commons/commons.cma \
 	h_program-lang/lib.cma \
     lang_cpp/parsing/lib.cma \
     lang_c/parsing/lib.cma \
+    generators/nim/lib.cma
 
 MAKESUBDIRS=commons commons_core \
   $(JSONDIR) \
@@ -83,6 +85,7 @@ MAKESUBDIRS=commons commons_core \
   h_program-lang \
   lang_cpp/parsing \
   lang_c/parsing \
+  generators/nim
 
 INCLUDEDIRS=$(MAKESUBDIRS)
 

data/cpp_stdlib/macros.h

@@ -0,0 +1,137 @@
+// ****************************************************************************
+// Prelude
+// ****************************************************************************
+// This file must be passed to the -macros option of the C/C++ parsers in pfff.
+// It should be stored as a default in Flag_parsing_cpp.macros_h
+
+// ****************************************************************************
+// MacroString
+// ****************************************************************************
+
+/* String macros are normally handled quite well by the LALR(k) technique,
+ * but sometimes it's not enough. For instance with 'XX YY', it could
+ * be considered as a declaration with XX being a typedef, so we would
+ * have an ambiguity. So by adding a few special cases (e.g. KERN_WARNING
+ * for the linux kernel), we can catch more string-macros. The idea is
+ * that we need to have at least one witness, a string to recognize
+ * a sequence of strings.
+ */
+//ex: #define KERN_WARNING "WARNING"
+
+/* EX_TABLE & co. 
+ *
+ * Replaced by a string. We can't put everything as comment
+ * because it can be part of an expression where we wait for
+ * something, where we wait for a string. So at least we 
+ * must keep the EX_TABLE token and transform it as a string.
+ *
+ * normally not needed if have good stringification of macro 
+ * but those macros are sometimes used multiple times 
+ * as in EX_TABLE(0b) EX_TABLE(1b)  and we don't detect
+ * it well yet.
+ */
+
+//ex: #define EX_TABLE(x)  "TOTO"
+
+// ****************************************************************************
+// MacroIterator
+// ****************************************************************************
+
+// foreach
+#define FOR_EACH(a,b) for(;;)
+
+// ****************************************************************************
+// MacroDeclarator
+// ****************************************************************************
+
+// static DECLARATOR(x);
+// LIST_HEAD stuff 
+// (used in qemu, freebsd)
+
+// ****************************************************************************
+// MacroStmt
+// ****************************************************************************
+
+// with or without parameters, but no ';'
+
+// ****************************************************************************
+// MacroField
+// ****************************************************************************
+
+// ****************************************************************************
+// MacroInitializer
+// ****************************************************************************
+
+// no PTVirg or shortcut for array designator
+
+// ****************************************************************************
+// MacroAttributes
+// ****************************************************************************
+
+//ex: #define __init YACFE_ATTRIBUTE
+// TODO: could perhaps generalize via "__.*"
+
+// linkage
+
+// params (IN, OUT)
+
+// windows: WINAPI, STDCALL, ...
+#define WINAPI
+#define  __cdecl
+
+// ****************************************************************************
+// MacroKeywordAlias
+// ****************************************************************************
+
+// const, often defined via macro for backward compatibility with old compiler
+// I guess.
+
+// private/public
+
+#define __asm__ asm
+#define __attribute __attribute__
+#define __volatile__ volatile
+#define __restrict __restrict__
+
+#define CONST const
+
+// for xv6
+#define __asm asm
+#define __volatile volatile
+
+// ****************************************************************************
+// Prototype
+// ****************************************************************************
+
+// __P
+
+// PARAMS
+
+// ****************************************************************************
+// Declarator
+// ****************************************************************************
+
+/* cf gcc-linux.h
+ * A trick to suppress uninitialized variable warning without generating any
+ * code
+ */
+
+// #define uninitialized_var(x) x = x
+// as in u16 uninitialized_var(ioboard_type);	/* GCC be quiet */ 
+
+// ****************************************************************************
+// Misc
+// ****************************************************************************
+
+// LIST_HEAD
+// GENTEST, GENHEADER
+// structure
+// MACHINE_START 
+// higher order, ASSERTCMP
+// parts of stuff, start of stuff
+// begin end wierd, as in C++ firefox and NS_DECLARE_BEGIN/END with code in
+// the middle
+// testcase, reflexivity on name
+// IDENT in sparse, wierd case
+// if-like macros
+//#define G_BEGIN_DECLS

data/cpp_stdlib/other_macros.h

@@ -0,0 +1,63 @@
+// TODO: now you can have a pfff_macros.h local per-project file (like I have
+// a local per-projet skip list), so distribute this to the relevant
+// places in software-src/
+
+// ****************************************************************************
+// git stuff
+// ****************************************************************************
+
+#define internal_function
+#define NOINLINE
+#define FORCEINLINE
+#define THROWSPEC
+#define NORETURN
+#define NORETURN_PTR
+#define MAYBE_UNUSED
+
+#define __MINGW_NOTHROW
+#define WSAAPI
+
+#define for_each_string_list_item(a,b) for(;;)
+
+// ****************************************************************************
+// sparse stuff
+// ****************************************************************************
+
+//YACFE_MACROITERATOR
+#define FOR_EACH_PTR(a,b) for(;;)
+#define FOR_EACH_PTR_NOTAG(a,b) for(;;)
+#define FOR_EACH_PTR_REVERSE(a,b) for(;;)
+#define RECURSE_PTR_REVERSE(a,b) for(;;)
+//#define DO_END_FOR_EACH_REVERSE(a,b,c,d)
+
+#define __percpu
+#define __user
+
+#define SENTINEL_ATTR
+#define FORMAT_ATTR(a)
+#define NORETURN_ATTR
+
+// ****************************************************************************
+// Boost
+// ****************************************************************************
+#define BOOST_AUTO_TEST_CASE(a) void a()
+#define BOOST_STATIC_ASSERT(a) static int x
+#define BOOST_FOR_EACH(a,b) for(;;)
+#define BOOST_FOREACH(a,b) for(;;)
+
+// ****************************************************************************
+// KDE
+// ****************************************************************************
+#define Q_OBJECT
+#define Q_CLASSINFO(a,b)
+
+// ****************************************************************************
+// Jansson
+// ****************************************************************************
+
+#define JSON_INLINE
+#define json_object_foreach(a,b,c) for(;;)
+
+// ****************************************************************************
+// Plan9
+// ****************************************************************************

data/cpp_stdlib/readme.txt

@@ -0,0 +1,2 @@
+You can also cp -a /usr/include/c++/.../ and /usr/include/boost wherever
+your project is.

generators/.merlin

@@ -0,0 +1 @@
+REC

generators/nim/.depend

@@ -0,0 +1,14 @@
+generate_nim.cmo : ../../h_program-lang/parse_info.cmi \
+    ../../lang_cpp/parsing/parse_cpp.cmi \
+    ../../lang_cpp/parsing/flag_parsing_cpp.cmo ../../commons/common.cmi \
+    ../../lang_cpp/parsing/ast_cpp.cmo generate_nim.cmi
+generate_nim.cmx : ../../h_program-lang/parse_info.cmx \
+    ../../lang_cpp/parsing/parse_cpp.cmx \
+    ../../lang_cpp/parsing/flag_parsing_cpp.cmx ../../commons/common.cmx \
+    ../../lang_cpp/parsing/ast_cpp.cmx generate_nim.cmi
+generate_nim.cmi : ../../commons/common.cmi
+unit_generate_nim.cmo : ../../commons/oUnit.cmi generate_nim.cmi \
+    ../../commons/common.cmi unit_generate_nim.cmi
+unit_generate_nim.cmx : ../../commons/oUnit.cmx generate_nim.cmx \
+    ../../commons/common.cmx unit_generate_nim.cmi
+unit_generate_nim.cmi : ../../commons/oUnit.cmi

generators/nim/.merlin

@@ -0,0 +1 @@
+REC

generators/nim/Makefile

@@ -0,0 +1,45 @@
+TOP=../..
+##############################################################################
+# Variables
+##############################################################################
+TARGET=lib
+
+-include $(TOP)/Makefile.config
+
+SRC= generate_nim.ml unit_generate_nim.ml
+
+SYSLIBS= str.cma unix.cma 
+
+LIBS=$(TOP)/commons/lib.cma \
+ $(TOP)/h_program-lang/lib.cma \
+
+INCLUDEDIRS= \
+ $(TOP)/commons \
+ $(TOP)/commons_core \
+ $(TOP)/globals \
+ $(TOP)/h_program-lang \
+ $(TOP)/lang_cpp/parsing
+
+##############################################################################
+# Generic variables
+##############################################################################
+
+-include $(TOP)/Makefile.common
+
+##############################################################################
+# Top rules
+##############################################################################
+all:: $(TARGET).cma
+all.opt:: $(TARGET).cmxa
+
+$(TARGET).cma: $(OBJS)
+	$(OCAMLC) -a -o $(TARGET).cma $(OBJS)
+
+$(TARGET).cmxa: $(OPTOBJS) $(LIBS:.cma=.cmxa)
+	$(OCAMLOPT) -a -o $(TARGET).cmxa $(OPTOBJS)
+
+$(TARGET).top: $(OBJS) $(LIBS)
+	$(OCAMLMKTOP) -o $(TARGET).top $(SYSLIBS) $(LIBS) $(OBJS)
+
+clean::
+	rm -f $(TARGET).top

generators/nim/generate_nim.ml

@@ -845,7 +845,7 @@ and process_declaration =
   | NameSpaceAnon ((v1, v2)) ->
       (*let v1 = vof_tok v1
       and v2 = vof_brace (Ocaml.vof_list vof_declaration_sequencable) v2
-      in Ocaml.VSum (("NameSpaceAnon", [ v1; v2 ]))*)
+      in Ocaml.VSu:m (("NameSpaceAnon", [ v1; v2 ]))*)
     ""
   | EmptyDef def -> process_token def
   | DeclTodo -> "# TODO"
@@ -864,8 +864,19 @@ and process_toplevel = function
 let iter_ast ast =
   List.map process_toplevel ast
 
-let test_dump_nim file =
+let generate_nim cfile macro_files =
-  Parse_cpp.init_defs !Flag.macros_h;
+  Parse_cpp.init_defs cfile;
-  let ast = Parse_cpp.parse_program file in
+  List.iter Parse_cpp.add_defs macro_files;
+  let ast = Parse_cpp.parse_program cfile in
   let res = iter_ast ast in
-  List.iter pr res
+  String.concat "\n" res
+
+let test_gen_nim file =
+  let macro_list = [!Flag.macros_h] in
+  let nim_str = generate_nim file macro_list in
+  pr nim_str
+
+let actions () = [
+    "-generate-nim", "   <file>",
+    Common.mk_action_1_arg test_gen_nim;
+]

generators/nim/generate_nim.mli

@@ -0,0 +1,4 @@
+val test_gen_nim :
+  Common.filename -> unit
+
+val actions : unit -> Common.cmdline_actions

generators/nim/unit_generate_nim.ml

@@ -0,0 +1,11 @@
+open Common
+open OUnit
+open Generate_nim
+
+
+let unittest =
+  "generating_nim" >::: [
+    "Test function declarations" >:: (fun () ->
+        pr "None"
+      )
+  ]

generators/nim/unit_generate_nim.mli

@@ -0,0 +1 @@
+val unittest: OUnit.test

globals/config_pfff.ml

@@ -2,7 +2,7 @@ let version = "0.29"
 
 let path =
   try (Sys.getenv "PFFF_HOME")
-  with Not_found->"/usr/local/share/pfff"
+  with Not_found->"./"
 
 let std_xxx = ref (Filename.concat path "xxx.yyy")
 

lang_cpp/parsing/.depend

@@ -135,25 +135,18 @@ pp_token.cmx : token_views_cpp.cmx token_helpers_cpp.cmx token_cpp.cmx \
     ../../commons/common2.cmx ../../commons/common.cmx ast_cpp.cmx \
     pp_token.cmi
 pp_token.cmi : token_views_cpp.cmi parser_cpp.cmi ../../commons/common.cmi
-test_dump_nim.cmo : ../../h_program-lang/parse_info.cmi parse_cpp.cmi \
-    flag_parsing_cpp.cmo ../../commons/common.cmi ast_cpp.cmo \
-    test_dump_nim.cmi
-test_dump_nim.cmx : ../../h_program-lang/parse_info.cmx parse_cpp.cmx \
-    flag_parsing_cpp.cmx ../../commons/common.cmx ast_cpp.cmx \
-    test_dump_nim.cmi
-test_dump_nim.cmi : ../../commons/common.cmi
 test_parsing_cpp.cmo : token_views_cpp.cmi token_views_context.cmi \
-    token_helpers_cpp.cmi test_dump_nim.cmi \
+    token_helpers_cpp.cmi ../../h_program-lang/skip_code.cmi \
-    ../../h_program-lang/skip_code.cmi parsing_hacks_cpp.cmi parser_cpp.cmi \
+    parsing_hacks_cpp.cmi parser_cpp.cmi ../../h_program-lang/parse_info.cmi \
-    ../../h_program-lang/parse_info.cmi parse_cpp.cmi ../../commons/ocaml.cmi \
+    parse_cpp.cmi ../../commons/ocaml.cmi \
     ../../h_program-lang/meta_ast_generic.cmi meta_ast_cpp.cmi \
     lib_parsing_cpp.cmi flag_parsing_cpp.cmo ../../commons_core/console.cmi \
     ../../commons/common.cmi ../../h_program-lang/ast_fuzzy.cmi ast_cpp.cmo \
     test_parsing_cpp.cmi
 test_parsing_cpp.cmx : token_views_cpp.cmx token_views_context.cmx \
-    token_helpers_cpp.cmx test_dump_nim.cmx \
+    token_helpers_cpp.cmx ../../h_program-lang/skip_code.cmx \
-    ../../h_program-lang/skip_code.cmx parsing_hacks_cpp.cmx parser_cpp.cmx \
+    parsing_hacks_cpp.cmx parser_cpp.cmx ../../h_program-lang/parse_info.cmx \
-    ../../h_program-lang/parse_info.cmx parse_cpp.cmx ../../commons/ocaml.cmx \
+    parse_cpp.cmx ../../commons/ocaml.cmx \
     ../../h_program-lang/meta_ast_generic.cmx meta_ast_cpp.cmx \
     lib_parsing_cpp.cmx flag_parsing_cpp.cmx ../../commons_core/console.cmx \
     ../../commons/common.cmx ../../h_program-lang/ast_fuzzy.cmx ast_cpp.cmx \

lang_cpp/parsing/Makefile

@@ -19,7 +19,6 @@ SRC= flag_parsing_cpp.ml \
      parsing_hacks.ml \
      parsing_recovery_cpp.ml \
      parse_cpp.ml \
-	 test_dump_nim.ml \
      test_parsing_cpp.ml unit_parsing_cpp.ml
 
 SYSLIBS= str.cma unix.cma 

lang_cpp/parsing/test_dump_nim.mli

@@ -1,2 +0,0 @@
-val test_dump_nim :
-  Common.filename -> unit

lang_cpp/parsing/test_parsing_cpp.ml

@@ -99,9 +99,6 @@ let actions () = [
     "-dump_cpp", "   <file>",
     Common.mk_action_1_arg test_dump_cpp;
 
-    "-dump_nim", "   <file>",
-    Common.mk_action_1_arg Test_dump_nim.test_dump_nim;
-
     "-dump_cpp_full", "   <file>",
     Common.mk_action_1_arg test_dump_cpp_full;
     "-dump_cpp_view", "   <file>",

main.ml

@@ -67,6 +67,7 @@ let all_actions () =
   pfff_extra_actions() @
   Test_parsing_c.actions()@
   Test_parsing_cpp.actions()@
+  Generate_nim.actions()@
 
 (*
   Test_analyze_cpp.actions () ++

main_test.ml

@@ -0,0 +1,127 @@
+open OUnit
+open Common2
+
+(*****************************************************************************)
+(* Flags *)
+(*****************************************************************************)
+
+(* In addition to flags that can be tweaked via -xxx options (cf the
+ * full list of options in the "the options" section below), this
+ * program also depends on external files ?
+ *)
+let verbose = ref false
+
+(* action mode *)
+let action = ref ""
+
+let test regexp =
+
+  (* There is no reflection in OCaml so the unit test framework OUnit requires
+   * us to explicitely build the test suites (which is not that bad).
+   *)
+  let tests =
+    "all" >::: [
+      Unit_generate_nim.unittest;
+    ]
+  in
+  let suite =
+    if regexp = "all"
+    then tests
+    else
+      let paths =
+        OUnit.test_case_paths tests +> List.map OUnit.string_of_path in
+      let keep = paths
+        +> List.filter (fun path ->
+          pr2 path;
+          path =~ (".*" ^ regexp))
+      in
+      Common2.some (OUnit.test_filter keep tests)
+  in
+
+  let results = OUnit.run_test_tt ~verbose:!verbose suite in
+  let has_an_error =
+    results +> List.exists (function
+    | OUnit.RSuccess _ | OUnit.RSkip _ | OUnit.RTodo _ -> false
+    | OUnit.RFailure _ | OUnit.RError _ -> true
+    )
+  in
+  raise (Common.UnixExit (if has_an_error then 1 else 0))
+
+let main_action x =
+  test x
+
+
+
+let all_actions () =
+  []
+
+let options () = [
+  "-verbose", Arg.Set verbose,
+  " ";
+  ] @
+  Common.options_of_actions action (all_actions()) @
+  Common2.cmdline_flags_devel () @
+  Common2.cmdline_flags_other () @
+  [
+    "-version",   Arg.Unit (fun () ->
+      pr2 (spf "pfff (test) version: %s" Config_pfff.version);
+      exit 0;
+    ),
+    "  guess what";
+    (* this can not be factorized in Common *)
+    "-date",   Arg.Unit (fun () ->
+      pr2 "version: $Date: 2008/10/26 00:44:57 $";
+      raise (Common.UnixExit 0)
+    ),
+    "   guess what";
+  ]
+
+(*****************************************************************************)
+(* Main entry point *)
+(*****************************************************************************)
+
+let main () =
+
+  Gc.set {(Gc.get ()) with Gc.stack_limit = 1000 * 1024 * 1024};
+
+  let usage_msg =
+    "Usage: " ^ Common2.basename Sys.argv.(0) ^
+      " [options] <file or dir> " ^ "\n" ^ "Options are:"
+  in
+  (* does side effect on many global flags *)
+  let args = Common.parse_options (options()) usage_msg Sys.argv in
+
+  (* must be done after Arg.parse, because Common.profile is set by it *)
+  Common.profile_code "Main total" (fun () ->
+
+    (match args with
+
+    (* --------------------------------------------------------- *)
+    (* actions, useful to debug subpart *)
+    (* --------------------------------------------------------- *)
+    | xs when List.mem !action (Common.action_list (all_actions())) ->
+        Common.do_action !action xs (all_actions())
+
+    | _ when not (Common.null_string !action) ->
+        failwith ("unrecognized action or wrong params: " ^ !action)
+
+    (* --------------------------------------------------------- *)
+    (* main entry *)
+    (* --------------------------------------------------------- *)
+    | [x] ->
+        main_action x
+
+    (* --------------------------------------------------------- *)
+    (* empty entry *)
+    (* --------------------------------------------------------- *)
+    | _ ->
+        Common.usage usage_msg (options());
+        failwith "too few or too many arguments"
+    )
+  )
+
+(*****************************************************************************)
+let _ =
+  Common.main_boilerplate (fun () ->
+    main ();
+  )