Add testing infrastructure and move out gen nim

This commit is contained in:
Joey Yakimowich-Payne 2018-05-26 12:40:09 +09:00
commit 801658b9ea
23 changed files with 462 additions and 32 deletions

21
.depend Normal file
View file

@ -0,0 +1,21 @@
find_source.cmo : h_program-lang/skip_code.cmi \
lang_cpp/parsing/lib_parsing_cpp.cmi lang_c/parsing/lib_parsing_c.cmi \
commons/common.cmi find_source.cmi
find_source.cmx : h_program-lang/skip_code.cmx \
lang_cpp/parsing/lib_parsing_cpp.cmx lang_c/parsing/lib_parsing_c.cmx \
commons/common.cmx find_source.cmi
find_source.cmi : commons/common.cmi
main.cmo : lang_cpp/parsing/test_parsing_cpp.cmi \
lang_c/parsing/test_parsing_c.cmi external/jsonwheel/json_io.cmi \
external/jsonwheel/json_in.cmo generators/nim/generate_nim.cmi \
lang_cpp/parsing/flag_parsing_cpp.cmo globals/config_pfff.cmo \
commons/common2.cmi commons/common.cmi
main.cmx : lang_cpp/parsing/test_parsing_cpp.cmx \
lang_c/parsing/test_parsing_c.cmx external/jsonwheel/json_io.cmx \
external/jsonwheel/json_in.cmx generators/nim/generate_nim.cmx \
lang_cpp/parsing/flag_parsing_cpp.cmx globals/config_pfff.cmx \
commons/common2.cmx commons/common.cmx
main_test.cmo : generators/nim/unit_generate_nim.cmi commons/oUnit.cmi \
globals/config_pfff.cmo commons/common2.cmi commons/common.cmi
main_test.cmx : generators/nim/unit_generate_nim.cmx commons/oUnit.cmx \
globals/config_pfff.cmx commons/common2.cmx commons/common.cmx

5
.gitignore vendored
View file

@ -2,6 +2,8 @@
*.cmo
*.cma
*.cmi
*.cmti
*.cmt
*.a
*.o
*.cmx
@ -19,9 +21,6 @@ _build/
setup.data
setup.log
# Merlin configuring file for Vim and Emacs
.merlin
# Swap
[._]*.s[a-v][a-z]
[._]*.sw[a-p]

View file

@ -14,3 +14,5 @@ S lang_c/**
B lang_c/**
S lang_cpp/**
B lang_cpp/**
S generators/**
B generators/**

View file

@ -19,7 +19,7 @@ TARGET=pfff
PROGS=pfff
#PROGS+=pfff_test
PROGS+=pfff_test
OPTPROGS= $(PROGS:=.opt)
@ -59,7 +59,8 @@ BASICLIBS=commons/commons.cma \
globals/lib.cma \
h_program-lang/lib.cma \
lang_cpp/parsing/lib.cma \
lang_c/parsing/lib.cma
lang_c/parsing/lib.cma \
generators/nim/lib.cma
# commons/commons_features.cma \
@ -75,6 +76,7 @@ LIBS= commons/commons.cma \
h_program-lang/lib.cma \
lang_cpp/parsing/lib.cma \
lang_c/parsing/lib.cma \
generators/nim/lib.cma
MAKESUBDIRS=commons commons_core \
$(JSONDIR) \
@ -83,6 +85,7 @@ MAKESUBDIRS=commons commons_core \
h_program-lang \
lang_cpp/parsing \
lang_c/parsing \
generators/nim
INCLUDEDIRS=$(MAKESUBDIRS)

137
data/cpp_stdlib/macros.h Normal file
View file

@ -0,0 +1,137 @@
// ****************************************************************************
// Prelude
// ****************************************************************************
// This file must be passed to the -macros option of the C/C++ parsers in pfff.
// It should be stored as a default in Flag_parsing_cpp.macros_h
// ****************************************************************************
// MacroString
// ****************************************************************************
/* String macros are normally handled quite well by the LALR(k) technique,
* but sometimes it's not enough. For instance with 'XX YY', it could
* be considered as a declaration with XX being a typedef, so we would
* have an ambiguity. So by adding a few special cases (e.g. KERN_WARNING
* for the linux kernel), we can catch more string-macros. The idea is
* that we need to have at least one witness, a string to recognize
* a sequence of strings.
*/
//ex: #define KERN_WARNING "WARNING"
/* EX_TABLE & co.
*
* Replaced by a string. We can't put everything as comment
* because it can be part of an expression where we wait for
* something, where we wait for a string. So at least we
* must keep the EX_TABLE token and transform it as a string.
*
* normally not needed if have good stringification of macro
* but those macros are sometimes used multiple times
* as in EX_TABLE(0b) EX_TABLE(1b) and we don't detect
* it well yet.
*/
//ex: #define EX_TABLE(x) "TOTO"
// ****************************************************************************
// MacroIterator
// ****************************************************************************
// foreach
#define FOR_EACH(a,b) for(;;)
// ****************************************************************************
// MacroDeclarator
// ****************************************************************************
// static DECLARATOR(x);
// LIST_HEAD stuff
// (used in qemu, freebsd)
// ****************************************************************************
// MacroStmt
// ****************************************************************************
// with or without parameters, but no ';'
// ****************************************************************************
// MacroField
// ****************************************************************************
// ****************************************************************************
// MacroInitializer
// ****************************************************************************
// no PTVirg or shortcut for array designator
// ****************************************************************************
// MacroAttributes
// ****************************************************************************
//ex: #define __init YACFE_ATTRIBUTE
// TODO: could perhaps generalize via "__.*"
// linkage
// params (IN, OUT)
// windows: WINAPI, STDCALL, ...
#define WINAPI
#define __cdecl
// ****************************************************************************
// MacroKeywordAlias
// ****************************************************************************
// const, often defined via macro for backward compatibility with old compiler
// I guess.
// private/public
#define __asm__ asm
#define __attribute __attribute__
#define __volatile__ volatile
#define __restrict __restrict__
#define CONST const
// for xv6
#define __asm asm
#define __volatile volatile
// ****************************************************************************
// Prototype
// ****************************************************************************
// __P
// PARAMS
// ****************************************************************************
// Declarator
// ****************************************************************************
/* cf gcc-linux.h
* A trick to suppress uninitialized variable warning without generating any
* code
*/
// #define uninitialized_var(x) x = x
// as in u16 uninitialized_var(ioboard_type); /* GCC be quiet */
// ****************************************************************************
// Misc
// ****************************************************************************
// LIST_HEAD
// GENTEST, GENHEADER
// structure
// MACHINE_START
// higher order, ASSERTCMP
// parts of stuff, start of stuff
// begin end wierd, as in C++ firefox and NS_DECLARE_BEGIN/END with code in
// the middle
// testcase, reflexivity on name
// IDENT in sparse, wierd case
// if-like macros
//#define G_BEGIN_DECLS

View file

@ -0,0 +1,63 @@
// TODO: now you can have a pfff_macros.h local per-project file (like I have
// a local per-projet skip list), so distribute this to the relevant
// places in software-src/
// ****************************************************************************
// git stuff
// ****************************************************************************
#define internal_function
#define NOINLINE
#define FORCEINLINE
#define THROWSPEC
#define NORETURN
#define NORETURN_PTR
#define MAYBE_UNUSED
#define __MINGW_NOTHROW
#define WSAAPI
#define for_each_string_list_item(a,b) for(;;)
// ****************************************************************************
// sparse stuff
// ****************************************************************************
//YACFE_MACROITERATOR
#define FOR_EACH_PTR(a,b) for(;;)
#define FOR_EACH_PTR_NOTAG(a,b) for(;;)
#define FOR_EACH_PTR_REVERSE(a,b) for(;;)
#define RECURSE_PTR_REVERSE(a,b) for(;;)
//#define DO_END_FOR_EACH_REVERSE(a,b,c,d)
#define __percpu
#define __user
#define SENTINEL_ATTR
#define FORMAT_ATTR(a)
#define NORETURN_ATTR
// ****************************************************************************
// Boost
// ****************************************************************************
#define BOOST_AUTO_TEST_CASE(a) void a()
#define BOOST_STATIC_ASSERT(a) static int x
#define BOOST_FOR_EACH(a,b) for(;;)
#define BOOST_FOREACH(a,b) for(;;)
// ****************************************************************************
// KDE
// ****************************************************************************
#define Q_OBJECT
#define Q_CLASSINFO(a,b)
// ****************************************************************************
// Jansson
// ****************************************************************************
#define JSON_INLINE
#define json_object_foreach(a,b,c) for(;;)
// ****************************************************************************
// Plan9
// ****************************************************************************

View file

@ -0,0 +1,2 @@
You can also cp -a /usr/include/c++/.../ and /usr/include/boost wherever
your project is.

1
generators/.merlin Normal file
View file

@ -0,0 +1 @@
REC

14
generators/nim/.depend Normal file
View file

@ -0,0 +1,14 @@
generate_nim.cmo : ../../h_program-lang/parse_info.cmi \
../../lang_cpp/parsing/parse_cpp.cmi \
../../lang_cpp/parsing/flag_parsing_cpp.cmo ../../commons/common.cmi \
../../lang_cpp/parsing/ast_cpp.cmo generate_nim.cmi
generate_nim.cmx : ../../h_program-lang/parse_info.cmx \
../../lang_cpp/parsing/parse_cpp.cmx \
../../lang_cpp/parsing/flag_parsing_cpp.cmx ../../commons/common.cmx \
../../lang_cpp/parsing/ast_cpp.cmx generate_nim.cmi
generate_nim.cmi : ../../commons/common.cmi
unit_generate_nim.cmo : ../../commons/oUnit.cmi generate_nim.cmi \
../../commons/common.cmi unit_generate_nim.cmi
unit_generate_nim.cmx : ../../commons/oUnit.cmx generate_nim.cmx \
../../commons/common.cmx unit_generate_nim.cmi
unit_generate_nim.cmi : ../../commons/oUnit.cmi

1
generators/nim/.merlin Normal file
View file

@ -0,0 +1 @@
REC

45
generators/nim/Makefile Normal file
View file

@ -0,0 +1,45 @@
TOP=../..
##############################################################################
# Variables
##############################################################################
TARGET=lib
-include $(TOP)/Makefile.config
SRC= generate_nim.ml unit_generate_nim.ml
SYSLIBS= str.cma unix.cma
LIBS=$(TOP)/commons/lib.cma \
$(TOP)/h_program-lang/lib.cma \
INCLUDEDIRS= \
$(TOP)/commons \
$(TOP)/commons_core \
$(TOP)/globals \
$(TOP)/h_program-lang \
$(TOP)/lang_cpp/parsing
##############################################################################
# Generic variables
##############################################################################
-include $(TOP)/Makefile.common
##############################################################################
# Top rules
##############################################################################
all:: $(TARGET).cma
all.opt:: $(TARGET).cmxa
$(TARGET).cma: $(OBJS)
$(OCAMLC) -a -o $(TARGET).cma $(OBJS)
$(TARGET).cmxa: $(OPTOBJS) $(LIBS:.cma=.cmxa)
$(OCAMLOPT) -a -o $(TARGET).cmxa $(OPTOBJS)
$(TARGET).top: $(OBJS) $(LIBS)
$(OCAMLMKTOP) -o $(TARGET).top $(SYSLIBS) $(LIBS) $(OBJS)
clean::
rm -f $(TARGET).top

View file

@ -845,7 +845,7 @@ and process_declaration =
| NameSpaceAnon ((v1, v2)) ->
(*let v1 = vof_tok v1
and v2 = vof_brace (Ocaml.vof_list vof_declaration_sequencable) v2
in Ocaml.VSum (("NameSpaceAnon", [ v1; v2 ]))*)
in Ocaml.VSu:m (("NameSpaceAnon", [ v1; v2 ]))*)
""
| EmptyDef def -> process_token def
| DeclTodo -> "# TODO"
@ -864,8 +864,19 @@ and process_toplevel = function
let iter_ast ast =
List.map process_toplevel ast
let test_dump_nim file =
Parse_cpp.init_defs !Flag.macros_h;
let ast = Parse_cpp.parse_program file in
let generate_nim cfile macro_files =
Parse_cpp.init_defs cfile;
List.iter Parse_cpp.add_defs macro_files;
let ast = Parse_cpp.parse_program cfile in
let res = iter_ast ast in
List.iter pr res
String.concat "\n" res
let test_gen_nim file =
let macro_list = [!Flag.macros_h] in
let nim_str = generate_nim file macro_list in
pr nim_str
let actions () = [
"-generate-nim", " <file>",
Common.mk_action_1_arg test_gen_nim;
]

View file

@ -0,0 +1,4 @@
val test_gen_nim :
Common.filename -> unit
val actions : unit -> Common.cmdline_actions

View file

@ -0,0 +1,11 @@
open Common
open OUnit
open Generate_nim
let unittest =
"generating_nim" >::: [
"Test function declarations" >:: (fun () ->
pr "None"
)
]

View file

@ -0,0 +1 @@
val unittest: OUnit.test

View file

@ -2,7 +2,7 @@ let version = "0.29"
let path =
try (Sys.getenv "PFFF_HOME")
with Not_found->"/usr/local/share/pfff"
with Not_found->"./"
let std_xxx = ref (Filename.concat path "xxx.yyy")

View file

@ -135,25 +135,18 @@ pp_token.cmx : token_views_cpp.cmx token_helpers_cpp.cmx token_cpp.cmx \
../../commons/common2.cmx ../../commons/common.cmx ast_cpp.cmx \
pp_token.cmi
pp_token.cmi : token_views_cpp.cmi parser_cpp.cmi ../../commons/common.cmi
test_dump_nim.cmo : ../../h_program-lang/parse_info.cmi parse_cpp.cmi \
flag_parsing_cpp.cmo ../../commons/common.cmi ast_cpp.cmo \
test_dump_nim.cmi
test_dump_nim.cmx : ../../h_program-lang/parse_info.cmx parse_cpp.cmx \
flag_parsing_cpp.cmx ../../commons/common.cmx ast_cpp.cmx \
test_dump_nim.cmi
test_dump_nim.cmi : ../../commons/common.cmi
test_parsing_cpp.cmo : token_views_cpp.cmi token_views_context.cmi \
token_helpers_cpp.cmi test_dump_nim.cmi \
../../h_program-lang/skip_code.cmi parsing_hacks_cpp.cmi parser_cpp.cmi \
../../h_program-lang/parse_info.cmi parse_cpp.cmi ../../commons/ocaml.cmi \
token_helpers_cpp.cmi ../../h_program-lang/skip_code.cmi \
parsing_hacks_cpp.cmi parser_cpp.cmi ../../h_program-lang/parse_info.cmi \
parse_cpp.cmi ../../commons/ocaml.cmi \
../../h_program-lang/meta_ast_generic.cmi meta_ast_cpp.cmi \
lib_parsing_cpp.cmi flag_parsing_cpp.cmo ../../commons_core/console.cmi \
../../commons/common.cmi ../../h_program-lang/ast_fuzzy.cmi ast_cpp.cmo \
test_parsing_cpp.cmi
test_parsing_cpp.cmx : token_views_cpp.cmx token_views_context.cmx \
token_helpers_cpp.cmx test_dump_nim.cmx \
../../h_program-lang/skip_code.cmx parsing_hacks_cpp.cmx parser_cpp.cmx \
../../h_program-lang/parse_info.cmx parse_cpp.cmx ../../commons/ocaml.cmx \
token_helpers_cpp.cmx ../../h_program-lang/skip_code.cmx \
parsing_hacks_cpp.cmx parser_cpp.cmx ../../h_program-lang/parse_info.cmx \
parse_cpp.cmx ../../commons/ocaml.cmx \
../../h_program-lang/meta_ast_generic.cmx meta_ast_cpp.cmx \
lib_parsing_cpp.cmx flag_parsing_cpp.cmx ../../commons_core/console.cmx \
../../commons/common.cmx ../../h_program-lang/ast_fuzzy.cmx ast_cpp.cmx \

View file

@ -19,7 +19,6 @@ SRC= flag_parsing_cpp.ml \
parsing_hacks.ml \
parsing_recovery_cpp.ml \
parse_cpp.ml \
test_dump_nim.ml \
test_parsing_cpp.ml unit_parsing_cpp.ml
SYSLIBS= str.cma unix.cma

View file

@ -3,7 +3,7 @@
(* types *)
(*****************************************************************************)
type language =
type language =
| C
| Cplusplus
@ -11,7 +11,7 @@ type language =
(* macros *)
(*****************************************************************************)
let macros_h =
let macros_h =
ref (Filename.concat Config_pfff.path "/data/cpp_stdlib/macros.h")
let cmdline_flags_macrofile () = [

View file

@ -1,2 +0,0 @@
val test_dump_nim :
Common.filename -> unit

View file

@ -99,9 +99,6 @@ let actions () = [
"-dump_cpp", " <file>",
Common.mk_action_1_arg test_dump_cpp;
"-dump_nim", " <file>",
Common.mk_action_1_arg Test_dump_nim.test_dump_nim;
"-dump_cpp_full", " <file>",
Common.mk_action_1_arg test_dump_cpp_full;
"-dump_cpp_view", " <file>",

View file

@ -67,6 +67,7 @@ let all_actions () =
pfff_extra_actions() @
Test_parsing_c.actions()@
Test_parsing_cpp.actions()@
Generate_nim.actions()@
(*
Test_analyze_cpp.actions () ++

127
main_test.ml Normal file
View file

@ -0,0 +1,127 @@
open OUnit
open Common2
(*****************************************************************************)
(* Flags *)
(*****************************************************************************)
(* In addition to flags that can be tweaked via -xxx options (cf the
* full list of options in the "the options" section below), this
* program also depends on external files ?
*)
let verbose = ref false
(* action mode *)
let action = ref ""
let test regexp =
(* There is no reflection in OCaml so the unit test framework OUnit requires
* us to explicitely build the test suites (which is not that bad).
*)
let tests =
"all" >::: [
Unit_generate_nim.unittest;
]
in
let suite =
if regexp = "all"
then tests
else
let paths =
OUnit.test_case_paths tests +> List.map OUnit.string_of_path in
let keep = paths
+> List.filter (fun path ->
pr2 path;
path =~ (".*" ^ regexp))
in
Common2.some (OUnit.test_filter keep tests)
in
let results = OUnit.run_test_tt ~verbose:!verbose suite in
let has_an_error =
results +> List.exists (function
| OUnit.RSuccess _ | OUnit.RSkip _ | OUnit.RTodo _ -> false
| OUnit.RFailure _ | OUnit.RError _ -> true
)
in
raise (Common.UnixExit (if has_an_error then 1 else 0))
let main_action x =
test x
let all_actions () =
[]
let options () = [
"-verbose", Arg.Set verbose,
" ";
] @
Common.options_of_actions action (all_actions()) @
Common2.cmdline_flags_devel () @
Common2.cmdline_flags_other () @
[
"-version", Arg.Unit (fun () ->
pr2 (spf "pfff (test) version: %s" Config_pfff.version);
exit 0;
),
" guess what";
(* this can not be factorized in Common *)
"-date", Arg.Unit (fun () ->
pr2 "version: $Date: 2008/10/26 00:44:57 $";
raise (Common.UnixExit 0)
),
" guess what";
]
(*****************************************************************************)
(* Main entry point *)
(*****************************************************************************)
let main () =
Gc.set {(Gc.get ()) with Gc.stack_limit = 1000 * 1024 * 1024};
let usage_msg =
"Usage: " ^ Common2.basename Sys.argv.(0) ^
" [options] <file or dir> " ^ "\n" ^ "Options are:"
in
(* does side effect on many global flags *)
let args = Common.parse_options (options()) usage_msg Sys.argv in
(* must be done after Arg.parse, because Common.profile is set by it *)
Common.profile_code "Main total" (fun () ->
(match args with
(* --------------------------------------------------------- *)
(* actions, useful to debug subpart *)
(* --------------------------------------------------------- *)
| xs when List.mem !action (Common.action_list (all_actions())) ->
Common.do_action !action xs (all_actions())
| _ when not (Common.null_string !action) ->
failwith ("unrecognized action or wrong params: " ^ !action)
(* --------------------------------------------------------- *)
(* main entry *)
(* --------------------------------------------------------- *)
| [x] ->
main_action x
(* --------------------------------------------------------- *)
(* empty entry *)
(* --------------------------------------------------------- *)
| _ ->
Common.usage usage_msg (options());
failwith "too few or too many arguments"
)
)
(*****************************************************************************)
let _ =
Common.main_boilerplate (fun () ->
main ();
)