jyapayne/cppast
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add function to tokenize strings

Browse source
This commit is contained in:
Jonathan Müller 2017-10-19 19:02:27 +02:00
commit 1572920650
14 changed files with 798 additions and 83 deletions
Download patch file Download diff file Expand all files Collapse all files

22
src/libclang/cxtokenizer.cpp
View file

@ -412,9 +412,9 @@ bool detail::skip_attribute(detail::cxtoken_stream& stream)
namespace
{
cpp_token_kind get_kind(CXTokenKind kind)
cpp_token_kind get_kind(const detail::cxtoken& token)
{
switch (kind)
switch (token.kind())
{
case CXToken_Punctuation:
return cpp_token_kind::punctuation;
@ -422,14 +422,26 @@ namespace
return cpp_token_kind::keyword;
case CXToken_Identifier:
return cpp_token_kind::identifier;
case CXToken_Literal:
return cpp_token_kind::literal;
{
auto spelling = token.value().std_str();
if (spelling.find('.') != std::string::npos)
return cpp_token_kind::float_literal;
else if (std::isdigit(spelling.front()))
return cpp_token_kind::int_literal;
else if (spelling.back() == '\'')
return cpp_token_kind::char_literal;
else
return cpp_token_kind::string_literal;
}
case CXToken_Comment:
break;
}
DEBUG_UNREACHABLE(detail::assert_handler{});
return cpp_token_kind ::literal;
return cpp_token_kind::punctuation;
}
}
@ -440,7 +452,7 @@ cpp_token_string detail::to_string(cxtoken_stream& stream, cxtoken_iterator end)
while (stream.cur() != end)
{
auto& token = stream.get();
builder.add_token(cpp_token(get_kind(token.kind()), token.c_str()));
builder.add_token(cpp_token(get_kind(token), token.c_str()));
}
if (stream.unmunch())

4
src/libclang/type_parser.cpp
View file

@ -246,7 +246,7 @@ namespace
return size_expr.empty() ?
nullptr :
cpp_unexposed_expression::build(cpp_builtin_type::build(cpp_ulonglong),
cpp_token_string::from_string(
cpp_token_string::tokenize(
std::string(size_expr.rbegin(),
size_expr.rend())));
}
@ -488,7 +488,7 @@ namespace
return cpp_decltype_type::build(
cpp_unexposed_expression::build(cpp_unexposed_type::build("<decltype>"),
cpp_token_string::from_string(spelling)));
cpp_token_string::tokenize(spelling)));
});
}