jyapayne/cppast
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Remove empty token check

Browse source 
Lead to an issue with semicolon at end of file, fixes #45.
This commit is contained in:
Jonathan Müller 2018-03-14 16:30:09 +01:00
commit 300854a6b0
1 changed files with 43 additions and 48 deletions
Download patch file Download diff file Expand all files Collapse all files

91
src/libclang/cxtokenizer.cpp
View file

@ -40,12 +40,9 @@ namespace
class simple_tokenizer class simple_tokenizer
{ {
public: public:
explicit simple_tokenizer(const CXTranslationUnit& tu, const CXSourceRange& range, explicit simple_tokenizer(const CXTranslationUnit& tu, const CXSourceRange& range) : tu_(tu)
const CXCursor& cur)
: tu_(tu)
{ {
clang_tokenize(tu, range, &tokens_, &no_); clang_tokenize(tu, range, &tokens_, &no_);
DEBUG_ASSERT(no_ >= 1u, detail::parse_error_handler{}, cur, "no tokens available");
} }
~simple_tokenizer() ~simple_tokenizer()
@ -72,17 +69,15 @@ namespace
unsigned no_; unsigned no_;
}; };
bool token_after_is(const CXTranslationUnit& tu, const CXFile& file, const CXCursor& cur, bool token_after_is(const CXTranslationUnit& tu, const CXFile& file,
const CXSourceLocation& loc, const char* token_str, int inc = 1) const CXSourceLocation& loc, const char* token_str, int inc)
{ {
auto loc_after = get_next_location(tu, file, loc, inc); auto loc_after = get_next_location(tu, file, loc, inc);
if (!clang_Location_isFromMainFile(loc_after)) if (!clang_Location_isFromMainFile(loc_after))
return false; return false;
simple_tokenizer tokenizer(tu, simple_tokenizer tokenizer(tu, inc > 0 ? clang_getRange(loc, loc_after) :
inc > 0 ? clang_getRange(loc, loc_after) : clang_getRange(loc_after, loc));
clang_getRange(loc_after, loc),
cur);
detail::cxstring spelling(clang_getTokenSpelling(tu, tokenizer[0u])); detail::cxstring spelling(clang_getTokenSpelling(tu, tokenizer[0u]));
return spelling == token_str; return spelling == token_str;
} }
@ -105,21 +100,21 @@ namespace
|| kind == CXCursor_VarDecl || kind == CXCursor_FieldDecl || kind == CXCursor_ParmDecl || kind == CXCursor_VarDecl || kind == CXCursor_FieldDecl || kind == CXCursor_ParmDecl
|| kind == CXCursor_NonTypeTemplateParameter) || kind == CXCursor_NonTypeTemplateParameter)
{ {
if (token_after_is(tu, file, cur, begin, "]", -2) if (token_after_is(tu, file, begin, "]", -2)
&& token_after_is(tu, file, cur, begin, "]", -3)) && token_after_is(tu, file, begin, "]", -3))
{ {
while (!token_after_is(tu, file, cur, begin, "[", -1) while (!token_after_is(tu, file, begin, "[", -1)
&& !token_after_is(tu, file, cur, begin, "[", -2)) && !token_after_is(tu, file, begin, "[", -2))
begin = get_next_location(tu, file, begin, -1); begin = get_next_location(tu, file, begin, -1);
begin = get_next_location(tu, file, begin, -3); begin = get_next_location(tu, file, begin, -3);
DEBUG_ASSERT(token_after_is(tu, file, cur, begin, "[") DEBUG_ASSERT(token_after_is(tu, file, begin, "[", 0)
&& token_after_is(tu, file, cur, && token_after_is(tu, file, get_next_location(tu, file, begin),
get_next_location(tu, file, begin), "["), "[", 0),
detail::parse_error_handler{}, cur, detail::parse_error_handler{}, cur,
"error in pre-function attribute parsing"); "error in pre-function attribute parsing");
} }
else if (token_after_is(tu, file, cur, begin, ")", -2)) else if (token_after_is(tu, file, begin, ")", -2))
{ {
// maybe alignas specifier // maybe alignas specifier
auto save_begin = begin; auto save_begin = begin;
@ -129,9 +124,9 @@ namespace
for (auto last_begin = begin; paren_count != 0; last_begin = begin) for (auto last_begin = begin; paren_count != 0; last_begin = begin)
{ {
begin = get_next_location(tu, file, begin, -1); begin = get_next_location(tu, file, begin, -1);
if (token_after_is(tu, file, cur, begin, "(", -1)) if (token_after_is(tu, file, begin, "(", -1))
--paren_count; --paren_count;
else if (token_after_is(tu, file, cur, begin, ")", -1)) else if (token_after_is(tu, file, begin, ")", -1))
++paren_count; ++paren_count;
DEBUG_ASSERT(!clang_equalLocations(last_begin, begin), DEBUG_ASSERT(!clang_equalLocations(last_begin, begin),
@ -140,7 +135,7 @@ namespace
} }
begin = get_next_location(tu, file, begin, -(int(std::strlen("alignas")) + 1)); begin = get_next_location(tu, file, begin, -(int(std::strlen("alignas")) + 1));
if (token_after_is(tu, file, cur, begin, "alignas")) if (token_after_is(tu, file, begin, "alignas", 0))
begin = get_next_location(tu, file, begin, -1); begin = get_next_location(tu, file, begin, -1);
else else
begin = save_begin; begin = save_begin;
@ -166,30 +161,30 @@ namespace
if (!is_definition) if (!is_definition)
{ {
// i have no idea why this is necessary // i have no idea why this is necessary
is_definition = token_after_is(tu, file, cur, end, "{") is_definition = token_after_is(tu, file, end, "{", 0)
|| token_after_is(tu, file, cur, end, "try") || token_after_is(tu, file, end, "try", 0)
|| token_after_is(tu, file, cur, end, ":"); || token_after_is(tu, file, end, ":", 0);
if (is_definition) if (is_definition)
// need to extend range here to include the token // need to extend range here to include the token
end = get_next_location(tu, file, end); end = get_next_location(tu, file, end);
} }
if (!is_definition && !token_after_is(tu, file, cur, end, ";")) if (!is_definition && !token_after_is(tu, file, end, ";", 0))
{ {
// we do not have a body, but it is not a declaration either // we do not have a body, but it is not a declaration either
do do
{ {
end = get_next_location(tu, file, end); end = get_next_location(tu, file, end);
} while (!token_after_is(tu, file, cur, end, ";")); } while (!token_after_is(tu, file, end, ";", 0));
} }
else if (kind == CXCursor_CXXMethod) else if (kind == CXCursor_CXXMethod)
// necessary for some reason // necessary for some reason
begin = get_next_location(tu, file, begin, -1); begin = get_next_location(tu, file, begin, -1);
else if (kind == CXCursor_Destructor && token_after_is(tu, file, cur, end, ")")) else if (kind == CXCursor_Destructor && token_after_is(tu, file, end, ")", 0))
// necessary for some other reason // necessary for some other reason
end = get_next_location(tu, file, end); end = get_next_location(tu, file, end);
} }
else if (kind == CXCursor_TemplateTypeParameter && token_after_is(tu, file, cur, end, "(")) else if (kind == CXCursor_TemplateTypeParameter && token_after_is(tu, file, end, "(", 0))
{ {
// if you have decltype as default argument for a type template parameter // if you have decltype as default argument for a type template parameter
// libclang doesn't include the parameters // libclang doesn't include the parameters
@ -197,9 +192,9 @@ namespace
auto prev = end; auto prev = end;
for (auto paren_count = 1; paren_count != 0; next = get_next_location(tu, file, next)) for (auto paren_count = 1; paren_count != 0; next = get_next_location(tu, file, next))
{ {
if (token_after_is(tu, file, cur, next, "(")) if (token_after_is(tu, file, next, "(", 0))
++paren_count; ++paren_count;
else if (token_after_is(tu, file, cur, next, ")")) else if (token_after_is(tu, file, next, ")", 0))
--paren_count; --paren_count;
prev = next; prev = next;
} }
@ -210,7 +205,7 @@ namespace
#endif #endif
} }
else if (kind == CXCursor_TemplateTemplateParameter else if (kind == CXCursor_TemplateTemplateParameter
&& token_after_is(tu, file, cur, end, "<")) && token_after_is(tu, file, end, "<", 0))
{ {
// if you have a template template parameter in a template template parameter, // if you have a template template parameter in a template template parameter,
// the tokens are all messed up, only contain the `template` // the tokens are all messed up, only contain the `template`
@ -220,18 +215,18 @@ namespace
auto next = get_next_location(tu, file, end, 2); auto next = get_next_location(tu, file, end, 2);
for (auto angle_count = 1; angle_count != 0; next = get_next_location(tu, file, next)) for (auto angle_count = 1; angle_count != 0; next = get_next_location(tu, file, next))
{ {
if (token_after_is(tu, file, cur, next, ">")) if (token_after_is(tu, file, next, ">", 0))
--angle_count; --angle_count;
else if (token_after_is(tu, file, cur, next, ">>")) else if (token_after_is(tu, file, next, ">>", 0))
angle_count -= 2; angle_count -= 2;
else if (token_after_is(tu, file, cur, next, "<")) else if (token_after_is(tu, file, next, "<", 0))
++angle_count; ++angle_count;
} }
// second: skip until end of parameter // second: skip until end of parameter
// no need to handle default, so look for '>' or ',' // no need to handle default, so look for '>' or ','
while (!token_after_is(tu, file, cur, next, ">") while (!token_after_is(tu, file, next, ">", 0)
&& !token_after_is(tu, file, cur, next, ",")) && !token_after_is(tu, file, next, ",", 0))
next = get_next_location(tu, file, next); next = get_next_location(tu, file, next);
// now we found the proper end of the token // now we found the proper end of the token
end = get_next_location(tu, file, next, -1); end = get_next_location(tu, file, next, -1);
@ -239,39 +234,39 @@ namespace
else if ((kind == CXCursor_TemplateTypeParameter else if ((kind == CXCursor_TemplateTypeParameter
|| kind == CXCursor_NonTypeTemplateParameter || kind == CXCursor_NonTypeTemplateParameter
|| kind == CXCursor_TemplateTemplateParameter) || kind == CXCursor_TemplateTemplateParameter)
&& token_after_is(tu, file, cur, end, "...")) && token_after_is(tu, file, end, "...", 0))
{ {
// variadic tokens in unnamed parameter not included // variadic tokens in unnamed parameter not included
end = get_next_location(tu, file, end, 3); end = get_next_location(tu, file, end, 3);
if (token_after_is(tu, file, cur, end, ".")) if (token_after_is(tu, file, end, ".", 0))
// extra whitespace, so bump again // extra whitespace, so bump again
// this should all go away once I redid the whole token thing... // this should all go away once I redid the whole token thing...
end = get_next_location(tu, file, end, 1); end = get_next_location(tu, file, end, 1);
DEBUG_ASSERT(token_after_is(tu, file, cur, end, ">") DEBUG_ASSERT(token_after_is(tu, file, end, ">", 0)
|| token_after_is(tu, file, cur, end, ","), || token_after_is(tu, file, end, ",", 0),
detail::parse_error_handler{}, cur, detail::parse_error_handler{}, cur,
"unexpected token in variadic parameter workaround"); "unexpected token in variadic parameter workaround");
} }
else if ((kind == CXCursor_TemplateTypeParameter else if ((kind == CXCursor_TemplateTypeParameter
|| kind == CXCursor_NonTypeTemplateParameter || kind == CXCursor_NonTypeTemplateParameter
|| kind == CXCursor_TemplateTemplateParameter) || kind == CXCursor_TemplateTemplateParameter)
&& !token_after_is(tu, file, cur, end, ">") && !token_after_is(tu, file, end, ">", 0)
&& !token_after_is(tu, file, cur, end, ",")) && !token_after_is(tu, file, end, ",", 0))
{ {
DEBUG_ASSERT(token_after_is(tu, file, cur, get_next_location(tu, file, end, -2), ">>"), DEBUG_ASSERT(token_after_is(tu, file, get_next_location(tu, file, end, -2), ">>", 0),
detail::parse_error_handler{}, cur, detail::parse_error_handler{}, cur,
"unexpected token in maximal munch workaround"); "unexpected token in maximal munch workaround");
unmunch = true; unmunch = true;
// need to shrink range anyway // need to shrink range anyway
end = get_next_location(tu, file, end, -1); end = get_next_location(tu, file, end, -1);
} }
else if (kind == CXCursor_EnumDecl && !token_after_is(tu, file, cur, end, ";")) else if (kind == CXCursor_EnumDecl && !token_after_is(tu, file, end, ";", 0))
{ {
while (!token_after_is(tu, file, cur, end, ";")) while (!token_after_is(tu, file, end, ";", 0))
end = get_next_location(tu, file, end); end = get_next_location(tu, file, end);
} }
else if (kind == CXCursor_EnumConstantDecl && !token_after_is(tu, file, cur, end, ",")) else if (kind == CXCursor_EnumConstantDecl && !token_after_is(tu, file, end, ",", 0))
{ {
// need to support attributes // need to support attributes
// just give up and extend the range to the range of the entire enum... // just give up and extend the range to the range of the entire enum...
@ -291,7 +286,7 @@ namespace
else if (kind == CXCursor_UnexposedDecl) else if (kind == CXCursor_UnexposedDecl)
{ {
// include semicolon, if necessary // include semicolon, if necessary
if (token_after_is(tu, file, cur, end, ";")) if (token_after_is(tu, file, end, ";", 0))
end = get_next_location(tu, file, end); end = get_next_location(tu, file, end);
} }
@ -304,7 +299,7 @@ detail::cxtokenizer::cxtokenizer(const CXTranslationUnit& tu, const CXFile& file
{ {
auto extent = get_extent(tu, file, cur, unmunch_); auto extent = get_extent(tu, file, cur, unmunch_);
simple_tokenizer tokenizer(tu, extent, cur); simple_tokenizer tokenizer(tu, extent);
tokens_.reserve(tokenizer.size()); tokens_.reserve(tokenizer.size());
for (auto i = 0u; i != tokenizer.size(); ++i) for (auto i = 0u; i != tokenizer.size(); ++i)
tokens_.emplace_back(tu, tokenizer[i]); tokens_.emplace_back(tu, tokenizer[i]);