Replace pfff with flitter
This commit is contained in:
Joey Yakimowich-Payne
parent
cccf36eb4f
commit
c8efc2a088
27 changed files with 295 additions and 293 deletions
|
|
@ -6,7 +6,7 @@
|
|||
* modify it under the terms of the GNU Lesser General Public License
|
||||
* version 2.1 as published by the Free Software Foundation, with the
|
||||
* special exception on linking described in file license.txt.
|
||||
*
|
||||
*
|
||||
* This library is distributed in the hope that it will be useful, but
|
||||
* WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the file
|
||||
|
|
@ -22,81 +22,81 @@ open Common
|
|||
* code "layers" (a.k.a. code "aspects"). The idea is to imitate google
|
||||
* earth layers (e.g. the wikipedia layer, panoramio layer, etc), but
|
||||
* for code. One can have a deadcode layer, a test coverage layer,
|
||||
* and then can display those layers or not on an existing codebase in
|
||||
* codemap. The layer is basically some mapping from files to a
|
||||
* set of lines with a specific color code.
|
||||
*
|
||||
*
|
||||
* and then can display those layers or not on an existing codebase in
|
||||
* codemap. The layer is basically some mapping from files to a
|
||||
* set of lines with a specific color code.
|
||||
*
|
||||
*
|
||||
* A few design choices:
|
||||
*
|
||||
* - one could store such information directly into database_xxx.ml
|
||||
* and have pfff_db compute such information (for instance each function
|
||||
* could have a set of properties like unit_test, or dead) but this
|
||||
* would force people to build their own db to visualize the results.
|
||||
* One could compute this information in database_light_xxx.ml, but this
|
||||
*
|
||||
* - one could store such information directly into database_xxx.ml
|
||||
* and have flitter_db compute such information (for instance each function
|
||||
* could have a set of properties like unit_test, or dead) but this
|
||||
* would force people to build their own db to visualize the results.
|
||||
* One could compute this information in database_light_xxx.ml, but this
|
||||
* will augment the size of the light db slowing down the codemap launch
|
||||
* even when the people don't use the layers. So it's more flexible to just
|
||||
* separate layer_code.ml from database_code.ml and have multiple persistent
|
||||
* files for each information. Also it's quite convenient to have
|
||||
* utilities like sgrep to be easily extendable to transform a query result
|
||||
* into a layer.
|
||||
*
|
||||
*
|
||||
* - How to represent a layer at the macro and micro level in codemap ?
|
||||
*
|
||||
*
|
||||
* At the micro-level one has just to display the line with the
|
||||
* requested color. At the macro-level have to either do a majority
|
||||
* scheme or mixing scheme where for instance draw half of the
|
||||
* treemap rectangle in red and the other in green.
|
||||
*
|
||||
* scheme or mixing scheme where for instance draw half of the
|
||||
* treemap rectangle in red and the other in green.
|
||||
*
|
||||
* Because different layers could have different composition needs
|
||||
* it is simpler to just have the layer say how it should be displayed
|
||||
* at the macro_level. See the 'macro_level' field below.
|
||||
*
|
||||
*
|
||||
* - how to have a layer data-structure that can cope with many
|
||||
* needs ?
|
||||
*
|
||||
* needs ?
|
||||
*
|
||||
* Here are some examples of layers and how they are "encoded" by the
|
||||
* 'layer' type below:
|
||||
*
|
||||
*
|
||||
* * deadcode (dead function, dead class, dead statement, dead assignnements)
|
||||
*
|
||||
*
|
||||
* How? dead lines in red color. At the macro_level one can give
|
||||
* a grey_xxx color with a percentage (e.g. grey53).
|
||||
*
|
||||
*
|
||||
* * test coverage (static or dynamic)
|
||||
*
|
||||
*
|
||||
* How? covered lines in green, not covered in red ? Also
|
||||
* convey a GreyLevel visualization by setting the 'macro_level' field.
|
||||
*
|
||||
*
|
||||
* * age of file
|
||||
*
|
||||
*
|
||||
* How? 2010 in green, 2009 in yelow, 2008 in red and so on.
|
||||
* At the macro_level can do a mix of colors.
|
||||
*
|
||||
*
|
||||
* * bad smells
|
||||
*
|
||||
*
|
||||
* How? each bad smell could have a different color and macro_level
|
||||
* showing a percentage of the rectangle with the right color
|
||||
* for each smells in the file.
|
||||
*
|
||||
*
|
||||
* * security patterns (bad smells)
|
||||
*
|
||||
* * activity ?
|
||||
*
|
||||
*
|
||||
* * activity ?
|
||||
*
|
||||
* How whow add and delete information ?
|
||||
* At the micro_level can't show the delete, but at macro_level
|
||||
* could divide the treemap_rectangle in 2 where percentage of
|
||||
* add and delete, and also maybe white to show the amount of add
|
||||
* and delete. Could also use my big circle scheme.
|
||||
* How link to commit message ? TODO
|
||||
*
|
||||
*
|
||||
* later:
|
||||
*
|
||||
*
|
||||
* later:
|
||||
* - could associate more than just a color, e.g. a commit message when want
|
||||
* to display a version-control layer, or some filling-patterns in
|
||||
* addition to the color.
|
||||
* - Could have better precision than the line.
|
||||
*
|
||||
*
|
||||
* history:
|
||||
* - I was writing some treemap generator specific for the deadcode
|
||||
* analysis, the static coverage, the dynamic coverage, and the activity
|
||||
|
|
@ -104,7 +104,7 @@ open Common
|
|||
* way to visualize the result (DegradeArchiColor | GreyLevel | YesNo).
|
||||
* It was working fine but there was no easy way to combine 2
|
||||
* visualisations, like the age "layer" and the "deadcode" layer
|
||||
* to see correlations. Also adding simple layers like
|
||||
* to see correlations. Also adding simple layers like
|
||||
* visualizing all calls to HTML() or XHP was requiring to
|
||||
* write another treemap generator. To be more generic and flexible require
|
||||
* a real 'layer' type.
|
||||
|
|
@ -118,19 +118,19 @@ type color = string (* Simple_color.emacs_color *)
|
|||
|
||||
(* note: the filenames must be in readable format so layer files can be reused
|
||||
* by multiple users.
|
||||
*
|
||||
*
|
||||
* alternatives:
|
||||
* - could have line range ? useful for layer matching lots of
|
||||
* consecutive lines in a file ?
|
||||
* - todo? have more precision than just the line ? precise pos range ?
|
||||
*
|
||||
*
|
||||
* - could for the lines instead of a 'kind' to have a 'count',
|
||||
* and then some mappings from range of values to a color.
|
||||
* For instance on a coverage layer one could say that from X to Y
|
||||
* then choose this color, from Y to Z another color.
|
||||
* But can emulate that by having a "coverage1", "coverage2"
|
||||
* kind with the current scheme.
|
||||
*
|
||||
*
|
||||
* - have a macro_level_composing_scheme: Majority | Mixed
|
||||
* that is then interpreted in codemap instead of forcing
|
||||
* the layer creator to specific how to show the micro_level
|
||||
|
|
@ -149,7 +149,7 @@ type layer = {
|
|||
|
||||
(* The list can be empty in which case codemap can use
|
||||
* the micro_level information and show a mix of colors.
|
||||
*
|
||||
*
|
||||
* The list can have just one element too and have a kind
|
||||
* different than the one used in the micro_level. For instance
|
||||
* for the coverage one can have red/green at micro_level
|
||||
|
|
@ -220,16 +220,16 @@ let heat_map_properties = [
|
|||
(*****************************************************************************)
|
||||
|
||||
(* Am I reinventing database indexing ? Should use a real database
|
||||
* to store layer information so one can then just use SQL to
|
||||
* to store layer information so one can then just use SQL to
|
||||
* fastly get all the information relevant to a file and a line ?
|
||||
* I doubt MySQL can be as fast and light as my JSON + hashtbl indexing.
|
||||
*)
|
||||
let build_index_of_layers ~root layers =
|
||||
let build_index_of_layers ~root layers =
|
||||
let hmicro = Common2.hash_with_default (fun () -> Hashtbl.create 101) in
|
||||
let hmacro = Common2.hash_with_default (fun () -> []) in
|
||||
|
||||
layers
|
||||
+> List.filter (fun (_layer, active) -> active)
|
||||
|
||||
layers
|
||||
+> List.filter (fun (_layer, active) -> active)
|
||||
+> List.iter (fun (layer, _active) ->
|
||||
let hkind = Common.hash_of_list layer.kinds in
|
||||
|
||||
|
|
@ -237,33 +237,33 @@ let build_index_of_layers ~root layers =
|
|||
|
||||
let file = Filename.concat root file in
|
||||
|
||||
(* todo? v is supposed to be a float representing a percentage of
|
||||
(* todo? v is supposed to be a float representing a percentage of
|
||||
* the rectangle but below we will add the macro info of multiple
|
||||
* layers together which mean the float may not represent percentage
|
||||
* anynore. They still represent a part of the file though.
|
||||
* The caller would have to first recompute the sum of all those
|
||||
* floats to recompute the actual multi-layer percentage.
|
||||
*)
|
||||
let color_macro_level =
|
||||
let color_macro_level =
|
||||
finfo.macro_level +> Common.map_filter (fun (kind, v) ->
|
||||
(* some sanity checking *)
|
||||
try Some (v, Hashtbl.find hkind kind)
|
||||
with Not_found ->
|
||||
with Not_found ->
|
||||
(* I was originally doing a failwith, but it can be convenient
|
||||
* to be able to filter kinds in codemap by just editing the
|
||||
* JSON file and removing certain kind definitions
|
||||
*)
|
||||
pr2_once (spf "PB: kind %s was not defined" kind);
|
||||
None
|
||||
)
|
||||
)
|
||||
in
|
||||
hmacro#update file (fun old -> color_macro_level @ old);
|
||||
|
||||
finfo.micro_level +> List.iter (fun (line, kind) ->
|
||||
try
|
||||
try
|
||||
let color = Hashtbl.find hkind kind in
|
||||
|
||||
hmicro#update file (fun oldh ->
|
||||
hmicro#update file (fun oldh ->
|
||||
(* We add so the same line could be assigned multiple colors.
|
||||
* The order of the layer could determine which color should
|
||||
* have priority.
|
||||
|
|
@ -369,36 +369,36 @@ open Ocaml
|
|||
module J = Json_type
|
||||
|
||||
(*
|
||||
let stag_incorrect_n_args _loc tag _v =
|
||||
let stag_incorrect_n_args _loc tag _v =
|
||||
failwith ("stag_incorrect_n_args on: " ^ tag)
|
||||
*)
|
||||
|
||||
(*
|
||||
let unexpected_stag loc v =
|
||||
let unexpected_stag loc v =
|
||||
failwith ("unexpected_stag:")
|
||||
*)
|
||||
|
||||
(*
|
||||
let record_only_pairs_expected loc v =
|
||||
let record_only_pairs_expected loc v =
|
||||
failwith ("record_only_pairs_expected:")
|
||||
*)
|
||||
|
||||
let record_duplicate_fields _loc _dup_flds _v =
|
||||
let record_duplicate_fields _loc _dup_flds _v =
|
||||
failwith ("record_duplicate_fields:")
|
||||
|
||||
let record_extra_fields _loc _flds _v =
|
||||
failwith ("record_extra_fields:")
|
||||
|
||||
let record_undefined_elements _loc _v _xs =
|
||||
let record_undefined_elements _loc _v _xs =
|
||||
failwith ("record_undefined_elements:")
|
||||
|
||||
let record_list_instead_atom _loc _v =
|
||||
let record_list_instead_atom _loc _v =
|
||||
failwith ("record_list_instead_atom:")
|
||||
|
||||
let tuple_of_size_n_expected _loc n v =
|
||||
let tuple_of_size_n_expected _loc n v =
|
||||
failwith (spf "tuple_of_size_n_expected: %d, got %s" n (Common2.dump v))
|
||||
|
||||
let rec json_of_v v =
|
||||
let rec json_of_v v =
|
||||
match v with
|
||||
| VString s -> J.String s
|
||||
| VSum ((s, vs)) ->J.Array ((J.String s)::(List.map json_of_v vs ))
|
||||
|
|
@ -414,7 +414,7 @@ let rec json_of_v v =
|
|||
| VBool b -> J.Bool b
|
||||
|
||||
(* Note that 'Inf' can be used as a constructor but is also recognized
|
||||
* by float_of_string as a float (infinity), so when I was implementing
|
||||
* by float_of_string as a float (infinity), so when I was implementing
|
||||
* this code by reverse engineering the generated sexp, it was important
|
||||
* to guard certain code.
|
||||
*)
|
||||
|
|
@ -427,7 +427,7 @@ let rec json_of_v v =
|
|||
| VArrow _v1 ->
|
||||
failwith "json_of_v: VArrow not handled"
|
||||
|
||||
(*
|
||||
(*
|
||||
* Assumes the json was generated via 'ocamltarzan -choice json_of', which
|
||||
* have certain conventions on how to encode variants for instance.
|
||||
*)
|
||||
|
|
@ -439,7 +439,7 @@ let rec (v_of_json: Json_type.json_type -> v) = fun j ->
|
|||
| J.Bool b -> VBool b
|
||||
| J.Null -> raise Todo
|
||||
|
||||
(* Arrays are used for represent constructors or regular list. Have to
|
||||
(* Arrays are used for represent constructors or regular list. Have to
|
||||
* go sligtly deeper to disambiguate.
|
||||
*)
|
||||
| J.Array xs ->
|
||||
|
|
@ -448,7 +448,7 @@ let rec (v_of_json: Json_type.json_type -> v) = fun j ->
|
|||
* of strings where the first element is a string that happen to
|
||||
* look like a constructor. With this ugly code we currently
|
||||
* not handle that :(
|
||||
*
|
||||
*
|
||||
* update: in the layer json file, one can have a filename
|
||||
* like Makefile and we don't want it to be a constructor ...
|
||||
* so for now I just generate constructors strings like
|
||||
|
|
@ -465,7 +465,7 @@ let rec (v_of_json: Json_type.json_type -> v) = fun j ->
|
|||
s, v_of_json fld
|
||||
))
|
||||
|
||||
let save_json file json =
|
||||
let save_json file json =
|
||||
let s = Json_out.string_of_json json in
|
||||
Common.write_file ~file s
|
||||
|
||||
|
|
@ -671,8 +671,8 @@ let save_layer layer file =
|
|||
* subdirs and so on.
|
||||
*)
|
||||
let simple_layer_of_parse_infos ~root ~title ?(description="") xs kinds =
|
||||
let ranks_kinds =
|
||||
kinds +> List.map (fun (k, _color) -> k)
|
||||
let ranks_kinds =
|
||||
kinds +> List.map (fun (k, _color) -> k)
|
||||
+> Common.index_list_1 +> Common.hash_of_list
|
||||
in
|
||||
|
||||
|
|
@ -680,48 +680,48 @@ let simple_layer_of_parse_infos ~root ~title ?(description="") xs kinds =
|
|||
let files_and_lines = xs +> List.map (fun (tok, kind) ->
|
||||
let file = Parse_info.file_of_info tok in
|
||||
let line = Parse_info.line_of_info tok in
|
||||
let file' = Common2.relative_to_absolute file in
|
||||
let file' = Common2.relative_to_absolute file in
|
||||
Common.readable ~root file', (line, kind)
|
||||
)
|
||||
in
|
||||
|
||||
let (group_by_file: (Common.filename * (int * kind) list) list) =
|
||||
Common.group_assoc_bykey_eff files_and_lines
|
||||
let (group_by_file: (Common.filename * (int * kind) list) list) =
|
||||
Common.group_assoc_bykey_eff files_and_lines
|
||||
in
|
||||
|
||||
{
|
||||
{
|
||||
title = title;
|
||||
description = description;
|
||||
kinds = kinds;
|
||||
files = group_by_file +> List.map (fun (file, lines_and_kinds) ->
|
||||
|
||||
let (group_by_line: (int * kind list) list) =
|
||||
Common.group_assoc_bykey_eff lines_and_kinds
|
||||
let (group_by_line: (int * kind list) list) =
|
||||
Common.group_assoc_bykey_eff lines_and_kinds
|
||||
in
|
||||
let all_kinds_in_file =
|
||||
let all_kinds_in_file =
|
||||
group_by_line +> List.map snd +> List.flatten +> Common2.uniq in
|
||||
|
||||
(file, {
|
||||
micro_level =
|
||||
group_by_line +> List.map (fun (line, kinds) ->
|
||||
(file, {
|
||||
micro_level =
|
||||
group_by_line +> List.map (fun (line, kinds) ->
|
||||
let kinds = Common2.uniq kinds in
|
||||
(* many kinds om same line, keep highest prio *)
|
||||
match kinds with
|
||||
| [] -> raise Impossible
|
||||
| [x] -> line, x
|
||||
| _ ->
|
||||
let sorted = kinds +> List.map (fun x ->
|
||||
let sorted = kinds +> List.map (fun x ->
|
||||
x, Hashtbl.find ranks_kinds x) +> Common.sort_by_val_lowfirst
|
||||
in
|
||||
line, List.hd sorted +> fst
|
||||
);
|
||||
|
||||
macro_level =
|
||||
macro_level =
|
||||
(* we could give a percentage per kind but right now
|
||||
* we instead give a priority based on the rank of the kinds
|
||||
* in the kind list
|
||||
*)
|
||||
all_kinds_in_file +> List.map (fun kind ->
|
||||
all_kinds_in_file +> List.map (fun kind ->
|
||||
(kind, 1. /. (float_of_int (Hashtbl.find ranks_kinds kind)))
|
||||
)
|
||||
})
|
||||
|
|
@ -729,26 +729,26 @@ let simple_layer_of_parse_infos ~root ~title ?(description="") xs kinds =
|
|||
}
|
||||
|
||||
|
||||
(* old: superseded by Layer_code.layer.files and file_info
|
||||
* type stat_per_file =
|
||||
(* old: superseded by Layer_code.layer.files and file_info
|
||||
* type stat_per_file =
|
||||
* (string (* a property *), int list (* lines *)) Common.assoc
|
||||
*
|
||||
* type stats =
|
||||
*
|
||||
* type stats =
|
||||
* (Common.filename, stat_per_file) Hashtbl.t
|
||||
*
|
||||
*
|
||||
*
|
||||
* old:
|
||||
* let (print_statistics: stats -> unit) = fun h ->
|
||||
* let xxs = Common.hash_to_list h in
|
||||
* pr2_gen (xxs);
|
||||
* ()
|
||||
*
|
||||
* let gen_security_layer xs =
|
||||
* let gen_security_layer xs =
|
||||
* let _root = Common.common_prefix_of_files_or_dirs xs in
|
||||
* let files = Lib_parsing_php.find_php_files_of_dir_or_files xs in
|
||||
*
|
||||
*
|
||||
* let h = Hashtbl.create 101 in
|
||||
*
|
||||
*
|
||||
* files +> Common.index_list_and_total +> List.iter (fun (file, i, total) ->
|
||||
* pr2 (spf "processing: %s (%d/%d)" file i total);
|
||||
* let ast = Parse_php.parse_program file in
|
||||
|
|
@ -778,8 +778,8 @@ let layer_red_green_and_heatmap ~root ~output xs =
|
|||
*)
|
||||
let stat_of_layer layer =
|
||||
let h = Common2.hash_with_default (fun () -> 0) in
|
||||
|
||||
layer.kinds +> List.iter (fun (kind, _color) ->
|
||||
|
||||
layer.kinds +> List.iter (fun (kind, _color) ->
|
||||
h#add kind 0
|
||||
);
|
||||
layer.files +> List.iter (fun (_file, finfo) ->
|
||||
|
|
@ -791,6 +791,6 @@ let stat_of_layer layer =
|
|||
|
||||
|
||||
let filter_layer f layer =
|
||||
{ layer with
|
||||
{ layer with
|
||||
files = layer.files +> List.filter (fun (file, _) -> f file);
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue