Add poc files
This commit is contained in:
Joey Yakimowich-Payne
parent
30da2412e3
commit
fa600b98f7
220 changed files with 45679 additions and 0 deletions
710
lang_cpp/parsing/lexer_cpp.mll
Normal file
710
lang_cpp/parsing/lexer_cpp.mll
Normal file
|
|
@ -0,0 +1,710 @@
|
|||
{
|
||||
(* Yoann Padioleau
|
||||
*
|
||||
* Copyright (C) 2002 Yoann Padioleau
|
||||
* Copyright (C) 2006-2007 Ecole des Mines de Nantes
|
||||
* Copyright (C) 2008-2009 University of Urbana Champaign
|
||||
* Copyright (C) 2010-2013 Facebook
|
||||
*
|
||||
* This program is free software; you can redistribute it and/or
|
||||
* modify it under the terms of the GNU General Public License (GPL)
|
||||
* version 2 as published by the Free Software Foundation.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* file license.txt for more details.
|
||||
*)
|
||||
open Common
|
||||
|
||||
open Parser_cpp
|
||||
open Ast_cpp (* to factorise tokens with OpAssign, ... *)
|
||||
|
||||
module Flag = Flag_parsing_cpp
|
||||
module Ast = Ast_cpp
|
||||
module PI = Parse_info
|
||||
|
||||
(*****************************************************************************)
|
||||
(* Prelude *)
|
||||
(*****************************************************************************)
|
||||
|
||||
(* The C/cpp/C++ lexer.
|
||||
*
|
||||
* This lexer generates tokens for C (int, while, ...), C++ (new, delete, ...),
|
||||
* CPP (#define, #ifdef, ...).
|
||||
* It also generate tokens for comments and spaces. This means that
|
||||
* it can not be used as-is. Some post-filtering
|
||||
* has to be done to feed it to a parser. Note that C and C++ are not
|
||||
* context free languages and so some idents must be disambiguated
|
||||
* in some ways. TIdent below must thus be post-processed too (as well
|
||||
* as other tokens like '<' for C++). See parsing_hack.ml for examples.
|
||||
*
|
||||
* note: We can't use Lexer_parser._lexer_hint here to do different
|
||||
* things because we now call the lexer to get all the tokens
|
||||
* and then only we parse. So we can use the hint only
|
||||
* in parse_cpp.ml. For the same reason, we don't handle typedefs
|
||||
* here anymore. We really just tokenize ...
|
||||
*)
|
||||
|
||||
(*****************************************************************************)
|
||||
(* Helpers *)
|
||||
(*****************************************************************************)
|
||||
exception Lexical of string
|
||||
|
||||
let error s =
|
||||
if !Flag.strict_lexer
|
||||
then raise (Lexical s)
|
||||
else
|
||||
if !Flag.verbose_lexing
|
||||
then pr2 ("LEXER: " ^ s)
|
||||
else ()
|
||||
|
||||
let tok lexbuf =
|
||||
Lexing.lexeme lexbuf
|
||||
|
||||
let tokinfo lexbuf =
|
||||
Parse_info.tokinfo_str_pos (tok lexbuf) (Lexing.lexeme_start lexbuf)
|
||||
|
||||
let tok_add_s = Parse_info.tok_add_s
|
||||
|
||||
(* ---------------------------------------------------------------------- *)
|
||||
(* Keywords *)
|
||||
(* ---------------------------------------------------------------------- *)
|
||||
|
||||
(* opti: less convenient, but using a hash is faster than using a match *)
|
||||
let keyword_table = Common.hash_of_list [
|
||||
|
||||
(* c: *)
|
||||
"void", (fun ii -> Tvoid ii);
|
||||
"char", (fun ii -> Tchar ii);
|
||||
"short", (fun ii -> Tshort ii); "int", (fun ii -> Tint ii);
|
||||
"long", (fun ii -> Tlong ii);
|
||||
"float", (fun ii -> Tfloat ii); "double", (fun ii -> Tdouble ii);
|
||||
|
||||
"unsigned", (fun ii -> Tunsigned ii); "signed", (fun ii -> Tsigned ii);
|
||||
|
||||
"auto", (fun ii -> Tauto ii);
|
||||
"register", (fun ii -> Tregister ii);
|
||||
"extern", (fun ii -> Textern ii);
|
||||
"static", (fun ii -> Tstatic ii);
|
||||
|
||||
"const", (fun ii -> Tconst ii); "volatile", (fun ii -> Tvolatile ii);
|
||||
|
||||
"struct", (fun ii -> Tstruct ii);
|
||||
"union", (fun ii -> Tunion ii);
|
||||
"enum", (fun ii -> Tenum ii);
|
||||
|
||||
"typedef", (fun ii -> Ttypedef ii);
|
||||
|
||||
"if", (fun ii -> Tif ii); "else", (fun ii -> Telse ii);
|
||||
"break", (fun ii -> Tbreak ii); "continue", (fun ii -> Tcontinue ii);
|
||||
"switch", (fun ii -> Tswitch ii);
|
||||
"case", (fun ii -> Tcase ii); "default", (fun ii -> Tdefault ii);
|
||||
"for", (fun ii -> Tfor ii);
|
||||
"do", (fun ii -> Tdo ii);
|
||||
"while", (fun ii -> Twhile ii);
|
||||
"return", (fun ii -> Treturn ii);
|
||||
"goto", (fun ii -> Tgoto ii);
|
||||
|
||||
"sizeof", (fun ii -> Tsizeof ii);
|
||||
|
||||
(* gccext: more (cpp) aliases are in macros.h *)
|
||||
"asm", (fun ii -> Tasm ii);
|
||||
"__attribute__", (fun ii -> Tattribute ii);
|
||||
"typeof", (fun ii -> Ttypeof ii);
|
||||
(* also a c++ext: *)
|
||||
"inline", (fun ii -> Tinline ii);
|
||||
|
||||
(* c99: *)
|
||||
"__restrict__", (fun ii -> Trestrict ii);
|
||||
|
||||
(* c++ext: see also TH.is_cpp_keyword *)
|
||||
"class", (fun ii -> Tclass ii);
|
||||
"this", (fun ii -> Tthis ii);
|
||||
|
||||
"new" , (fun ii -> Tnew ii);
|
||||
"delete" , (fun ii -> Tdelete ii);
|
||||
|
||||
"template" , (fun ii -> Ttemplate ii);
|
||||
"typeid" , (fun ii -> Ttypeid ii);
|
||||
"typename" , (fun ii -> Ttypename ii);
|
||||
|
||||
"catch" , (fun ii -> Tcatch ii);
|
||||
"try" , (fun ii -> Ttry ii);
|
||||
"throw" , (fun ii -> Tthrow ii);
|
||||
|
||||
"operator", (fun ii -> Toperator ii);
|
||||
|
||||
"public" , (fun ii -> Tpublic ii);
|
||||
"private" , (fun ii -> Tprivate ii);
|
||||
"protected" , (fun ii -> Tprotected ii);
|
||||
|
||||
"friend" , (fun ii -> Tfriend ii);
|
||||
|
||||
"virtual", (fun ii -> Tvirtual ii);
|
||||
|
||||
"namespace", (fun ii -> Tnamespace ii);
|
||||
"using", (fun ii -> Tusing ii);
|
||||
|
||||
"bool", (fun ii -> Tbool ii);
|
||||
|
||||
"true", (fun ii -> Ttrue ii); "false", (fun ii -> Tfalse ii);
|
||||
|
||||
"wchar_t", (fun ii -> Twchar_t ii);
|
||||
|
||||
"const_cast" , (fun ii -> Tconst_cast ii);
|
||||
"dynamic_cast" , (fun ii -> Tdynamic_cast ii);
|
||||
"static_cast" , (fun ii -> Tstatic_cast ii);
|
||||
"reinterpret_cast" , (fun ii -> Treinterpret_cast ii);
|
||||
|
||||
"explicit", (fun ii -> Texplicit ii);
|
||||
"mutable", (fun ii -> Tmutable ii);
|
||||
|
||||
"export", (fun ii -> Texport ii);
|
||||
]
|
||||
|
||||
let error_radix s =
|
||||
("numeric " ^ s ^ " constant contains digits beyond the radix:")
|
||||
|
||||
}
|
||||
(*****************************************************************************)
|
||||
(* Regexps aliases *)
|
||||
(*****************************************************************************)
|
||||
let letter = ['A'-'Z' 'a'-'z' '_']
|
||||
let digit = ['0'-'9']
|
||||
|
||||
(* not used for the moment *)
|
||||
let punctuation = ['!' '"' '#' '%' '&' '\'' '(' ')' '*' '+' ',' '-' '.' '/' ':'
|
||||
';' '<' '=' '>' '?' '[' '\\' ']' '^' '{' '|' '}' '~']
|
||||
let space = [' ' '\t' '\n' '\r' '\011' '\012' ]
|
||||
let additionnal = [ ' ' '\b' '\t' '\011' '\n' '\r' '\007' ]
|
||||
(* 7 = \a = bell in C. this is not the only char allowed !!
|
||||
* ex @ and $ ` are valid too
|
||||
*)
|
||||
|
||||
let cchar = (letter | digit | punctuation | additionnal)
|
||||
|
||||
let sp = [' ' '\t']+
|
||||
let spopt = [' ' '\t']*
|
||||
|
||||
let dec = ['0'-'9']
|
||||
let oct = ['0'-'7']
|
||||
let hex = ['0'-'9' 'a'-'f' 'A'-'F']
|
||||
|
||||
let decimal = ('0' | (['1'-'9'] dec*))
|
||||
let octal = ['0'] oct+
|
||||
let hexa = ("0x" |"0X") hex+
|
||||
|
||||
let pent = dec+
|
||||
let pfract = dec+
|
||||
let sign = ['-' '+']
|
||||
let exp = ['e''E'] sign? dec+
|
||||
let real = pent exp | ((pent? '.' pfract | pent '.' pfract? ) exp?)
|
||||
|
||||
let id = letter (letter | digit) *
|
||||
|
||||
(*****************************************************************************)
|
||||
(* Rule token *)
|
||||
(*****************************************************************************)
|
||||
rule token = parse
|
||||
|
||||
(* ----------------------------------------------------------------------- *)
|
||||
(* Spaces, comments *)
|
||||
(* ----------------------------------------------------------------------- *)
|
||||
|
||||
(* note: this lexer generate tokens for comments! So we can not give
|
||||
* this lexer as-is to the parsing function. We must postprocess it, and
|
||||
* use techniques like cur_tok ref in parse_cpp.ml
|
||||
*)
|
||||
|
||||
| [' ' '\t' ]+
|
||||
{ TCommentSpace (tokinfo lexbuf) }
|
||||
|
||||
(* see also TCppEscapedNewline below *)
|
||||
| [ '\n' '\r' '\011' '\012']
|
||||
{ TCommentNewline (tokinfo lexbuf) }
|
||||
|
||||
| "/*"
|
||||
{ let info = tokinfo lexbuf in
|
||||
let com = comment lexbuf in
|
||||
TComment(info +> tok_add_s com)
|
||||
}
|
||||
|
||||
(* C++ comments are allowed via gccext, but normally they are deleted by cpp.
|
||||
* So we need this here only because we dont call cpp before.
|
||||
* Note that we don't keep the trailing \n; it will be in another token.
|
||||
*)
|
||||
| "//" [^'\r' '\n' '\011']* { TComment (tokinfo lexbuf) }
|
||||
|
||||
(* ---------------------- *)
|
||||
(* #include *)
|
||||
(* ---------------------- *)
|
||||
|
||||
(* The difference between a local "" and standard <> include is computed
|
||||
* later in parser_cpp.mly. So we redo a little bit of lexing there. It's
|
||||
* ugly but simpler to generate a single token here. *)
|
||||
| (("#" [' ''\t']* ("include" | "include_next" | "import")
|
||||
[' ' '\t']*) as includes)
|
||||
(('"' ([^ '"']+) '"' |
|
||||
'<' [^ '>']+ '>' |
|
||||
['A'-'Z''_']+
|
||||
) as filename)
|
||||
{ (* less: generate 2 info so highlight_cpp.ml can colorize the
|
||||
* directive and the filename differently
|
||||
*)
|
||||
TInclude (includes, filename, tokinfo lexbuf)
|
||||
}
|
||||
|
||||
(* ---------------------- *)
|
||||
(* #ifdef *)
|
||||
(* ---------------------- *)
|
||||
|
||||
| "#" [' ' '\t']* "if" [' ' '\t']* '0' (* [^'\n']* '\n' *)
|
||||
{ let info = tokinfo lexbuf in
|
||||
TIfdefBool (false, info(* +> tok_add_s (cpp_eat_until_nl lexbuf)*))
|
||||
}
|
||||
| "#" [' ' '\t']* "if" [' ' '\t']* '1' (* [^'\n']* '\n' *)
|
||||
{ let info = tokinfo lexbuf in
|
||||
TIfdefBool (true, info)
|
||||
}
|
||||
| "#" [' ' '\t']* "ifdef" [' ' '\t']* "__cplusplus" [^'\n']* '\n'
|
||||
{ let info = tokinfo lexbuf in
|
||||
TIfdefMisc (false, info)
|
||||
}
|
||||
|
||||
(* can have some ifdef 0 hence the letter|digit even at beginning of word *)
|
||||
| "#" [' ''\t']* "ifdef" [' ''\t']+ (letter|digit)((letter|digit)*) [' ''\t']*
|
||||
{ TIfdef (tokinfo lexbuf) }
|
||||
| "#" [' ''\t']* "ifndef" [' ''\t']+ (letter|digit)((letter|digit)*)[' ''\t']*
|
||||
{ TIfdef (tokinfo lexbuf) }
|
||||
| "#" [' ''\t']* "if" [' ' '\t']+
|
||||
{ let info = tokinfo lexbuf in
|
||||
TIfdef (info +> tok_add_s (cpp_eat_until_nl lexbuf))
|
||||
}
|
||||
| "#" [' ' '\t']* "if" '('
|
||||
{ let info = tokinfo lexbuf in
|
||||
TIfdef (info +> tok_add_s (cpp_eat_until_nl lexbuf))
|
||||
}
|
||||
|
||||
| "#" [' ' '\t']* "elif" [' ' '\t']+
|
||||
{ let info = tokinfo lexbuf in
|
||||
TIfdefelif (info +> tok_add_s (cpp_eat_until_nl lexbuf))
|
||||
}
|
||||
|
||||
(* bugfix: can have #endif LINUX but at the same time if I eat everything
|
||||
* until next line, I may miss some TComment which for some tools
|
||||
* are important such as aComment
|
||||
*)
|
||||
| "#" [' ' '\t']* "endif" (*[^'\n']* '\n'*)
|
||||
{ TEndif (tokinfo lexbuf) }
|
||||
| "#" [' ' '\t']* "else" [' ' '\t' '\n']
|
||||
{ TIfdefelse (tokinfo lexbuf) }
|
||||
|
||||
(* ---------------------- *)
|
||||
(* #define, #undef *)
|
||||
(* ---------------------- *)
|
||||
|
||||
(* The rest of the lexing/parsing of #define is done in fix_tokens_define
|
||||
* where we parse all TCppEscapedNewline and finally generate a TDefEol
|
||||
*)
|
||||
| "#" [' ' '\t']* "define" { TDefine (tokinfo lexbuf) }
|
||||
|
||||
(* note: in some cases we can have stuff after the ident as in #undef XXX 50,
|
||||
* but I currently don't handle it cos I think it's bad code.
|
||||
*)
|
||||
| (("#" [' ' '\t']* "undef" [' ' '\t']+) as _undef) (id as id)
|
||||
(* alt: +> tok_add_s (cpp_eat_until_nl lexbuf)) *)
|
||||
{ TUndef (id, tokinfo lexbuf) }
|
||||
|
||||
(* ---------------------- *)
|
||||
(* #define body *)
|
||||
(* ---------------------- *)
|
||||
|
||||
(* We could generate separate tokens for #, ## and then extend
|
||||
* the grammar, but there can be ident in many different places, in
|
||||
* expression but also in declaration, in function name. So having 3 tokens
|
||||
* for an ident does not work well with how we add info in
|
||||
* ast_cpp.ml. So it's better to generate just one token, just one info,
|
||||
* even if have later to reanalyse those tokens and unsplit.
|
||||
*
|
||||
* less: do as in yacfe, generate multiple tokens for those constructs?
|
||||
*)
|
||||
|
||||
| ((id as s) "...")
|
||||
{ TDefParamVariadic (s, tokinfo lexbuf) }
|
||||
|
||||
(* cppext: string concatenation *)
|
||||
| id ([' ''\t']* "##" [' ''\t']* id)+
|
||||
{ let info = tokinfo lexbuf in
|
||||
TIdent (tok lexbuf, info)
|
||||
}
|
||||
|
||||
(* cppext: stringification
|
||||
* bugfix: this case must be after the other cases such as #endif
|
||||
* otherwise take precedent.
|
||||
*)
|
||||
| "#" (*spopt*) id
|
||||
{ let info = tokinfo lexbuf in
|
||||
TIdent (tok lexbuf, info)
|
||||
}
|
||||
|
||||
(* cppext: gccext: ##args for variadic macro *)
|
||||
| "##" [' ''\t']* id
|
||||
{ let info = tokinfo lexbuf in
|
||||
TIdent (tok lexbuf, info)
|
||||
}
|
||||
|
||||
(* only in define body normally *)
|
||||
| "\\" '\n' { TCppEscapedNewline (tokinfo lexbuf) }
|
||||
|
||||
(* ---------------------- *)
|
||||
(* cpp pragmas *)
|
||||
(* ---------------------- *)
|
||||
|
||||
(* bugfix: I want to keep comments so cant do a sp [^'\n']+ '\n'
|
||||
* http://gcc.gnu.org/onlinedocs/gcc/Pragmas.html
|
||||
*)
|
||||
| "#" spopt "pragma" sp [^'\n']* '\n'
|
||||
| "#" spopt "ident" sp [^'\n']* '\n'
|
||||
| "#" spopt "line" sp [^'\n']* '\n'
|
||||
| "#" spopt "error" sp [^'\n']* '\n'
|
||||
| "#" spopt "warning" sp [^'\n']* '\n'
|
||||
| "#" spopt "abort" sp [^'\n']* '\n'
|
||||
{ TCppDirectiveOther (tokinfo lexbuf) }
|
||||
|
||||
(* This appears only after calling cpp cpp, as in:
|
||||
* # 1 "include/linux/module.h" 1
|
||||
* Because we handle cpp ourselves, why handle it here?
|
||||
* Why not ... also one could want to use our parser on
|
||||
* expanded files sometimes.
|
||||
*)
|
||||
| "#" sp pent sp '"' [^ '"']* '"' (spopt pent)* spopt '\n'
|
||||
{ TCppDirectiveOther (tokinfo lexbuf) }
|
||||
|
||||
(* ?? *)
|
||||
| "#" [' ' '\t']* '\n'
|
||||
{ TCppDirectiveOther (tokinfo lexbuf) }
|
||||
|
||||
(* ----------------------------------------------------------------------- *)
|
||||
(* C symbols *)
|
||||
(* ----------------------------------------------------------------------- *)
|
||||
(* stdC:
|
||||
* ... && -= >= ~ + ; ]
|
||||
* <<= &= -> >> % , < ^
|
||||
* >>= *= /= ^= & - = {
|
||||
* != ++ << |= ( . > |
|
||||
* %= += <= || ) / ? }
|
||||
* -- == ! * : [
|
||||
* recent addition: <: :> <% %>
|
||||
* only at processing: %: %:%: # ##
|
||||
*)
|
||||
|
||||
| '[' { TOCro(tokinfo lexbuf) } | ']' { TCCro(tokinfo lexbuf) }
|
||||
| '(' { TOPar(tokinfo lexbuf) } | ')' { TCPar(tokinfo lexbuf) }
|
||||
| '{' { TOBrace(tokinfo lexbuf) } | '}' { TCBrace(tokinfo lexbuf) }
|
||||
|
||||
| '+' { TPlus(tokinfo lexbuf) } | '*' { TMul(tokinfo lexbuf) }
|
||||
| '-' { TMinus(tokinfo lexbuf) } | '/' { TDiv(tokinfo lexbuf) }
|
||||
| '%' { TMod(tokinfo lexbuf) }
|
||||
|
||||
| "++"{ TInc(tokinfo lexbuf) } | "--"{ TDec(tokinfo lexbuf) }
|
||||
|
||||
| "=" { TEq(tokinfo lexbuf) }
|
||||
|
||||
| "-=" { TAssign (OpAssign Minus, (tokinfo lexbuf))}
|
||||
| "+=" { TAssign (OpAssign Plus, (tokinfo lexbuf))}
|
||||
| "*=" { TAssign (OpAssign Mul, (tokinfo lexbuf))}
|
||||
| "/=" { TAssign (OpAssign Div, (tokinfo lexbuf))}
|
||||
| "%=" { TAssign (OpAssign Mod, (tokinfo lexbuf))}
|
||||
| "&=" { TAssign (OpAssign And, (tokinfo lexbuf))}
|
||||
| "|=" { TAssign (OpAssign Or, (tokinfo lexbuf)) }
|
||||
| "^=" { TAssign(OpAssign Xor, (tokinfo lexbuf))}
|
||||
| "<<=" {TAssign (OpAssign DecLeft, (tokinfo lexbuf)) }
|
||||
| ">>=" {TAssign (OpAssign DecRight, (tokinfo lexbuf))}
|
||||
|
||||
| "==" { TEqEq(tokinfo lexbuf) } | "!=" { TNotEq(tokinfo lexbuf) }
|
||||
| ">=" { TSupEq(tokinfo lexbuf) } | "<=" { TInfEq(tokinfo lexbuf) }
|
||||
(* c++ext: transformed in TInf_Template in parsing_hacks_cpp.ml *)
|
||||
| "<" { TInf(tokinfo lexbuf) } | ">" { TSup(tokinfo lexbuf) }
|
||||
|
||||
| "&&" { TAndLog(tokinfo lexbuf) } | "||" { TOrLog(tokinfo lexbuf) }
|
||||
| ">>" { TShr(tokinfo lexbuf) } | "<<" { TShl(tokinfo lexbuf) }
|
||||
| "&" { TAnd(tokinfo lexbuf) } | "|" { TOr(tokinfo lexbuf) }
|
||||
| "^" { TXor(tokinfo lexbuf) }
|
||||
| "..." { TEllipsis(tokinfo lexbuf) }
|
||||
| "->" { TPtrOp(tokinfo lexbuf) } | '.' { TDot(tokinfo lexbuf) }
|
||||
| ',' { TComma(tokinfo lexbuf) }
|
||||
| ";" { TPtVirg(tokinfo lexbuf) }
|
||||
| "?" { TWhy(tokinfo lexbuf) } | ":" { TCol(tokinfo lexbuf) }
|
||||
| "!" { TBang(tokinfo lexbuf) } | "~" { TTilde(tokinfo lexbuf) }
|
||||
|
||||
|
||||
| "<:" { TOCro(tokinfo lexbuf) } | ":>" { TCCro(tokinfo lexbuf) }
|
||||
| "<%" { TOBrace(tokinfo lexbuf) } | "%>" { TCBrace(tokinfo lexbuf) }
|
||||
|
||||
(* c++ext: *)
|
||||
| "::" { TColCol(tokinfo lexbuf) }
|
||||
| "->*" { TPtrOpStar(tokinfo lexbuf) } | ".*" { TDotStar(tokinfo lexbuf) }
|
||||
|
||||
(* ----------------------------------------------------------------------- *)
|
||||
(* C keywords and ident *)
|
||||
(* ----------------------------------------------------------------------- *)
|
||||
|
||||
(* StdC: "must handle at least name of length > 509, but can
|
||||
* truncate to 31 when compare and truncate to 6 and even lowerise
|
||||
* in the external linkage phase"
|
||||
*)
|
||||
| letter (letter | digit) *
|
||||
{ let info = tokinfo lexbuf in
|
||||
let s = tok lexbuf in
|
||||
Common.profile_code "C parsing.lex_ident" (fun () ->
|
||||
match Common2.optionise (fun () -> Hashtbl.find keyword_table s) with
|
||||
| Some f -> f info
|
||||
|
||||
(* typedef_hack. note: now this is no more useful, cos
|
||||
* as we use tokens_all, we first parse then all as idents and
|
||||
* later transform some idents into typedefs. So this job is
|
||||
* now done in parse_cpp.ml.
|
||||
*
|
||||
* old:
|
||||
* if Lexer_parser.is_typedef s
|
||||
* then Ident_Typedef (s, info)
|
||||
* else TIdent (s, info)
|
||||
*)
|
||||
| None -> TIdent (s, info)
|
||||
)
|
||||
}
|
||||
|
||||
(* gccext: apparently gcc allows dollar in variable names. I've found such
|
||||
* things a few times in Linux and in glibc.
|
||||
* No need to look in keyword_table here; definitly a TIdent.
|
||||
*)
|
||||
| (letter | '$') (letter | digit | '$')*
|
||||
{
|
||||
let s = tok lexbuf in
|
||||
if not !Flag.sgrep_mode
|
||||
then error ("identifier with dollar: " ^ s);
|
||||
TIdent (s, tokinfo lexbuf)
|
||||
}
|
||||
|
||||
|
||||
(* ----------------------------------------------------------------------- *)
|
||||
(* C constant *)
|
||||
(* ----------------------------------------------------------------------- *)
|
||||
|
||||
| "'"
|
||||
{ let info = tokinfo lexbuf in
|
||||
let s = char lexbuf in
|
||||
TChar ((s, IsChar), (info +> tok_add_s (s ^ "'")))
|
||||
}
|
||||
| '"'
|
||||
{ let info = tokinfo lexbuf in
|
||||
let s = string lexbuf in
|
||||
TString ((s, IsChar), (info +> tok_add_s (s ^ "\"")))
|
||||
}
|
||||
(* wide character encoding, TODO L'toto' valid ? what is allowed ? *)
|
||||
| 'L' "'"
|
||||
{ let info = tokinfo lexbuf in
|
||||
let s = char lexbuf in
|
||||
TChar ((s, IsWchar), (info +> tok_add_s (s ^ "'")))
|
||||
}
|
||||
| 'L' '"'
|
||||
{ let info = tokinfo lexbuf in
|
||||
let s = string lexbuf in
|
||||
TString ((s, IsWchar), (info +> tok_add_s (s ^ "\"")))
|
||||
}
|
||||
|
||||
(* Take care of the order ? No because lex try the longest match. The
|
||||
* strange diff between decimal and octal constant semantic is not
|
||||
* understood too by refman :) refman:11.1.4, and ritchie.
|
||||
*)
|
||||
| (( decimal | hexa | octal)
|
||||
( ['u' 'U']
|
||||
| ['l' 'L']
|
||||
| (['l' 'L'] ['u' 'U'])
|
||||
| (['u' 'U'] ['l' 'L'])
|
||||
| (['u' 'U'] ['l' 'L'] ['l' 'L'])
|
||||
| (['l' 'L'] ['l' 'L'])
|
||||
)?
|
||||
) as x { TInt (x, tokinfo lexbuf) }
|
||||
|
||||
| (real ['f' 'F']) as x { TFloat ((x, CFloat), tokinfo lexbuf) }
|
||||
| (real ['l' 'L']) as x { TFloat ((x, CLongDouble), tokinfo lexbuf) }
|
||||
| (real as x) { TFloat ((x, CDouble), tokinfo lexbuf) }
|
||||
|
||||
| ['0'] ['0'-'9']+
|
||||
{ error (error_radix "octal" ^ tok lexbuf);
|
||||
TUnknown (tokinfo lexbuf)
|
||||
}
|
||||
| ("0x" |"0X") ['0'-'9' 'a'-'z' 'A'-'Z']+
|
||||
{ error (error_radix "hexa" ^ tok lexbuf);
|
||||
TUnknown (tokinfo lexbuf)
|
||||
}
|
||||
|
||||
(* !put after other rules! otherwise 0xff will be parsed as an ident *)
|
||||
| ['0'-'9']+ letter (letter | digit) *
|
||||
{ error ("ZARB integer_string, certainly a macro:" ^ tok lexbuf);
|
||||
TUnknown (tokinfo lexbuf)
|
||||
}
|
||||
|
||||
(* gccext: http://gcc.gnu.org/onlinedocs/gcc/Binary-constants.html *)
|
||||
(*
|
||||
| "0b" ['0'-'1'] { TInt (((tok lexbuf)<!!>(??,??)) +> int_of_stringbits) }
|
||||
| ['0'-'1']+'b' { TInt (((tok lexbuf)<!!>(0,-2)) +> int_of_stringbits) }
|
||||
*)
|
||||
(*------------------------------------------------------------------------ *)
|
||||
| eof { EOF (tokinfo lexbuf +> PI.rewrap_str "") }
|
||||
|
||||
| _ {
|
||||
error("unrecognised symbol, in token rule:" ^ tok lexbuf);
|
||||
TUnknown (tokinfo lexbuf)
|
||||
}
|
||||
|
||||
(*****************************************************************************)
|
||||
(* Rule char *)
|
||||
(*****************************************************************************)
|
||||
and char = parse
|
||||
(* c++ext: or firefoxext: unicode char may take multiple char as in 'MOSS'
|
||||
| (_ as x) "'" { String.make 1 x }
|
||||
|
||||
(* todo?: as for octal, do exception beyond radix exception ? *)
|
||||
| (("\\" (oct | oct oct | oct oct oct)) as x "'") { x }
|
||||
(* this rule must be after the one with octal, lex try first longest
|
||||
* and when \7 we want an octal, not an exn.
|
||||
*)
|
||||
| (("\\x" ((hex | hex hex))) as x "'") { x }
|
||||
| (("\\" (_ as v)) as x "'")
|
||||
{
|
||||
(match v with (* Machine specific ? *)
|
||||
| 'n' -> () | 't' -> () | 'v' -> () | 'b' -> () | 'r' -> ()
|
||||
| 'f' -> () | 'a' -> ()
|
||||
| '\\' -> () | '?' -> () | '\'' -> () | '"' -> ()
|
||||
| 'e' -> () (* linuxext: ? *)
|
||||
| _ ->
|
||||
error ("unrecognised symbol in char:"^tok lexbuf);
|
||||
);
|
||||
x
|
||||
}
|
||||
| _
|
||||
{ error ("unrecognised symbol in char:"^tok lexbuf);
|
||||
tok lexbuf
|
||||
}
|
||||
*)
|
||||
(* c++ext: mostly copy paste of string but s/"/'/ " and s/string/char *)
|
||||
| '\'' { "" }
|
||||
| (_ as x)
|
||||
{ Common2.string_of_char x^char lexbuf}
|
||||
|
||||
| ("\\" (oct | oct oct | oct oct oct)) as x { x ^ char lexbuf }
|
||||
| ("\\x" (hex | hex hex)) as x { x ^ char lexbuf }
|
||||
| ("\\" (_ as v)) as x
|
||||
{
|
||||
(match v with (* Machine specific ? *)
|
||||
| 'n' -> () | 't' -> () | 'v' -> () | 'b' -> () | 'r' -> ()
|
||||
| 'f' -> () | 'a' -> ()
|
||||
| '\\' -> () | '?' -> () | '\'' -> () | '"' -> ()
|
||||
| 'e' -> () (* linuxext: ? *)
|
||||
|
||||
(* old: "x" -> 10 gccext ? todo ugly, I put a fake value *)
|
||||
|
||||
(* cppext: can have \ for multiline in string too *)
|
||||
| '\n' -> ()
|
||||
| _ -> error ("unrecognised symbol in char:"^tok lexbuf);
|
||||
);
|
||||
x ^ char lexbuf
|
||||
}
|
||||
| eof { error "WEIRD end of file in char"; ""}
|
||||
|
||||
(*****************************************************************************)
|
||||
(* Rule string *)
|
||||
(*****************************************************************************)
|
||||
(* less? factorise code with char ? but not same ending token so hard. *)
|
||||
and string = parse
|
||||
| '"' { "" }
|
||||
| (_ as x)
|
||||
{ Common2.string_of_char x^string lexbuf}
|
||||
|
||||
| ("\\" (oct | oct oct | oct oct oct)) as x { x ^ string lexbuf }
|
||||
| ("\\x" (hex | hex hex)) as x { x ^ string lexbuf }
|
||||
(* unicode *)
|
||||
| ("\\u" (hex hex hex hex)) as x { x ^ string lexbuf }
|
||||
| ("\\U" (hex hex hex hex hex hex hex hex)) as x { x ^ string lexbuf }
|
||||
| ("\\" (_ as v)) as x
|
||||
{
|
||||
(match v with (* Machine specific ? *)
|
||||
| 'n' -> () | 't' -> () | 'v' -> () | 'b' -> () | 'r' -> ()
|
||||
| 'f' -> () | 'a' -> ()
|
||||
| '\\' -> () | '?' -> () | '\'' -> () | '"' -> ()
|
||||
| 'e' -> () (* linuxext: ? *)
|
||||
|
||||
(* old: "x" -> 10 gccext ? todo ugly, I put a fake value *)
|
||||
|
||||
(* cppext: can have \ for multiline in string too *)
|
||||
| '\n' -> ()
|
||||
| _ -> error ("unrecognised symbol in string:"^tok lexbuf);
|
||||
);
|
||||
x ^ string lexbuf
|
||||
}
|
||||
| eof { error "WEIRD end of file in string"; ""}
|
||||
|
||||
(* Bug if add following code, cos match also the '"' that is needed
|
||||
* to finish the string, and so go until end of file.
|
||||
*)
|
||||
(*
|
||||
| [^ '\\']+
|
||||
{ let cs = lexbuf +> tok +> list_of_string +> List.map Char.code in
|
||||
cs ++ string lexbuf
|
||||
}
|
||||
*)
|
||||
|
||||
(*****************************************************************************)
|
||||
(* Rule comment *)
|
||||
(*****************************************************************************)
|
||||
|
||||
(* less: allow only char-'*' ? *)
|
||||
and comment = parse
|
||||
| "*/" { tok lexbuf }
|
||||
(* noteopti: *)
|
||||
| [^ '*']+ { let s = tok lexbuf in s ^ comment lexbuf }
|
||||
| [ '*'] { let s = tok lexbuf in s ^ comment lexbuf }
|
||||
| _
|
||||
{ let s = tok lexbuf in
|
||||
error ("unrecognised symbol in comment:"^s);
|
||||
s ^ comment lexbuf
|
||||
}
|
||||
| eof { error "WEIRD end of file in comment"; ""}
|
||||
|
||||
(*****************************************************************************)
|
||||
(* Rule cpp_eat_until_nl *)
|
||||
(*****************************************************************************)
|
||||
|
||||
(* cpp recognize C comments, so when #define xx (yy) /* comment \n ... */
|
||||
* then he has already erased the /* comment. So:
|
||||
* - dont eat the start of the comment otherwise afterwards we are in the middle
|
||||
* of a comment and so we will problably get a parse error somewhere.
|
||||
* - have to recognize comments in cpp_eat_until_nl.
|
||||
*
|
||||
* note: I was using cpp_eat_until_nl for #define before, but now I
|
||||
* try also to parse define body so cpp_eat_until_nl is used only for the "body"
|
||||
* of other uninteresting directtives like #ifdef, #else where can have
|
||||
* stuff on the right on such directive.
|
||||
*)
|
||||
and cpp_eat_until_nl = parse
|
||||
(* bugfix: need to handle comments too *)
|
||||
| "/*"
|
||||
{ let s = tok lexbuf in
|
||||
let s2 = comment lexbuf in
|
||||
let s3 = cpp_eat_until_nl lexbuf in
|
||||
s ^ s2 ^ s3
|
||||
}
|
||||
| '\\' "\n" { let s = tok lexbuf in s ^ cpp_eat_until_nl lexbuf }
|
||||
|
||||
| "\n" { tok lexbuf }
|
||||
(* noteopti:
|
||||
* update: need also deal with comments chars now
|
||||
*)
|
||||
| [^ '\n' '\\' '/' '*' ]+
|
||||
{ let s = tok lexbuf in s ^ cpp_eat_until_nl lexbuf }
|
||||
|
||||
| eof { error "end of file in cpp_eat_until_nl"; ""}
|
||||
| _ { let s = tok lexbuf in s ^ cpp_eat_until_nl lexbuf }
|
||||
Loading…
Add table
Add a link
Reference in a new issue