jyapayne/cppast
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Remove tokenizer unmunch as it was never set

Browse source
This commit is contained in:
Jonathan Müller 2022-10-13 09:09:37 +02:00
commit 4c40bb7544
3 changed files with 23 additions and 46 deletions
Download patch file Download diff file Expand all files Collapse all files

17
src/libclang/cxtokenizer.cpp
View file

@ -70,7 +70,7 @@ public:
clang_disposeTokens(tu_, tokens_, no_); clang_disposeTokens(tu_, tokens_, no_);
} }
simple_tokenizer(const simple_tokenizer&) = delete; simple_tokenizer(const simple_tokenizer&) = delete;
simple_tokenizer& operator=(const simple_tokenizer&) = delete; simple_tokenizer& operator=(const simple_tokenizer&) = delete;
unsigned size() const noexcept unsigned size() const noexcept
@ -396,7 +396,6 @@ Extent get_extent(const CXTranslationUnit& tu, const CXFile& file, const CXCurso
detail::cxtokenizer::cxtokenizer(const CXTranslationUnit& tu, const CXFile& file, detail::cxtokenizer::cxtokenizer(const CXTranslationUnit& tu, const CXFile& file,
const CXCursor& cur) const CXCursor& cur)
: unmunch_(false)
{ {
auto extent = get_extent(tu, file, cur); auto extent = get_extent(tu, file, cur);
@ -532,19 +531,20 @@ void detail::skip_brackets(detail::cxtoken_stream& stream)
stream.set_cur(std::next(closing)); stream.set_cur(std::next(closing));
} }
detail::cxtoken_iterator detail::find_sequence(detail::cxtoken_stream stream, detail::cxtoken_iterator start, detail::cxtoken_iterator detail::find_sequence(detail::cxtoken_stream stream,
detail::cxtoken_iterator start,
detail::cxtoken_iterator end) detail::cxtoken_iterator end)
{ {
detail::cxtoken_iterator search_start = stream.cur(); detail::cxtoken_iterator search_start = stream.cur();
while(search_start != stream.end()) while (search_start != stream.end())
{ {
detail::cxtoken_iterator search_iter = search_start; detail::cxtoken_iterator search_iter = search_start;
detail::cxtoken_iterator seq_iter = start; detail::cxtoken_iterator seq_iter = start;
bool failed = false; bool failed = false;
while(!failed && search_iter != stream.end() && seq_iter != end) while (!failed && search_iter != stream.end() && seq_iter != end)
{ {
if(search_iter->value() != seq_iter->value() if (search_iter->value() != seq_iter->value()
|| search_iter->kind() != seq_iter->kind()) || search_iter->kind() != seq_iter->kind())
{ {
failed = true; failed = true;
@ -555,7 +555,7 @@ detail::cxtoken_iterator detail::find_sequence(detail::cxtoken_stream stream, de
++seq_iter; ++seq_iter;
} }
} }
if(!failed) if (!failed)
return search_start; return search_start;
++search_start; ++search_start;
@ -779,9 +779,6 @@ cpp_token_string detail::to_string(cxtoken_stream& stream, cxtoken_iterator end)
builder.add_token(cpp_token(get_kind(token), token.c_str())); builder.add_token(cpp_token(get_kind(token), token.c_str()));
} }
if (stream.unmunch())
builder.unmunch();
return builder.finish(); return builder.finish();
} }

21
src/libclang/cxtokenizer.hpp
View file

@ -78,25 +78,15 @@ namespace detail
return tokens_.end(); return tokens_.end();
} }
// if it returns true, the last token is ">>",
// but should haven been ">"
// only a problem for template parameters
bool unmunch() const noexcept
{
return unmunch_;
}
private: private:
std::vector<cxtoken> tokens_; std::vector<cxtoken> tokens_;
bool unmunch_;
}; };
class cxtoken_stream class cxtoken_stream
{ {
public: public:
explicit cxtoken_stream(const cxtokenizer& tokenizer, const CXCursor& cur) explicit cxtoken_stream(const cxtokenizer& tokenizer, const CXCursor& cur)
: cursor_(cur), begin_(tokenizer.begin()), cur_(begin_), end_(tokenizer.end()), : cursor_(cur), begin_(tokenizer.begin()), cur_(begin_), end_(tokenizer.end())
unmunch_(tokenizer.unmunch())
{} {}
const cxtoken& peek() const noexcept const cxtoken& peek() const noexcept
@ -155,15 +145,9 @@ namespace detail
cur_ = iter; cur_ = iter;
} }
bool unmunch() const noexcept
{
return unmunch_;
}
private: private:
CXCursor cursor_; CXCursor cursor_;
cxtoken_iterator begin_, cur_, end_; cxtoken_iterator begin_, cur_, end_;
bool unmunch_;
}; };
// skips the next token // skips the next token
@ -186,7 +170,8 @@ namespace detail
// finds the location of the given sequence in a stream // finds the location of the given sequence in a stream
// returns an iterator to the first token of the found sequence, or stream.end() if not found // returns an iterator to the first token of the found sequence, or stream.end() if not found
cxtoken_iterator find_sequence(cxtoken_stream stream, cxtoken_iterator start, cxtoken_iterator end); cxtoken_iterator find_sequence(cxtoken_stream stream, cxtoken_iterator start,
cxtoken_iterator end);
// parses attributes // parses attributes
// if skip_anyway is true it will bump even if no attributes have been parsed // if skip_anyway is true it will bump even if no attributes have been parsed

31
src/libclang/template_parser.cpp
View file

@ -43,16 +43,17 @@ type_safe::optional<typename TemplateT::builder> get_builder(const detail::parse
} }
cpp_token_string extract_parameter_constraint(const detail::parse_context& context, cpp_token_string extract_parameter_constraint(const detail::parse_context& context,
const CXCursor& parent, const CXCursor& parent,
detail::cxtoken_iterator target_range_start, detail::cxtoken_iterator target_range_start,
detail::cxtoken_iterator target_range_end) detail::cxtoken_iterator target_range_end)
{ {
//search the parent context for the *exact* sequence in it's entirety // search the parent context for the *exact* sequence in it's entirety
detail::cxtokenizer tokenizer(context.tu, context.file, parent); detail::cxtokenizer tokenizer(context.tu, context.file, parent);
detail::cxtoken_stream stream(tokenizer, parent); detail::cxtoken_stream stream(tokenizer, parent);
detail::cxtoken_iterator found_start = detail::find_sequence(stream, target_range_start, target_range_end); detail::cxtoken_iterator found_start
if(found_start == stream.end()) = detail::find_sequence(stream, target_range_start, target_range_end);
if (found_start == stream.end())
return detail::to_string(stream, stream.cur() + 1); return detail::to_string(stream, stream.cur() + 1);
stream.set_cur(found_start); stream.set_cur(found_start);
@ -62,8 +63,8 @@ cpp_token_string extract_parameter_constraint(const detail::parse_context& conte
detail::cxtoken_iterator constraint_end = stream.cur(); detail::cxtoken_iterator constraint_end = stream.cur();
stream.set_cur(found_start); stream.set_cur(found_start);
//seek backwards until we are at the start of the qualified name // seek backwards until we are at the start of the qualified name
while(stream.peek().value() != "<" && stream.peek().value() != ",") while (stream.peek().value() != "<" && stream.peek().value() != ",")
{ {
std::string str = stream.peek().value().std_str(); std::string str = stream.peek().value().std_str();
stream.bump_back(); stream.bump_back();
@ -79,7 +80,6 @@ std::unique_ptr<cpp_template_parameter> parse_type_parameter(const detail::parse
DEBUG_ASSERT(clang_getCursorKind(cur) == CXCursor_TemplateTypeParameter, DEBUG_ASSERT(clang_getCursorKind(cur) == CXCursor_TemplateTypeParameter,
detail::assert_handler{}); detail::assert_handler{});
detail::cxtokenizer tokenizer(context.tu, context.file, cur); detail::cxtokenizer tokenizer(context.tu, context.file, cur);
detail::cxtoken_stream stream(tokenizer, cur); detail::cxtoken_stream stream(tokenizer, cur);
auto name = detail::get_cursor_name(cur); auto name = detail::get_cursor_name(cur);
@ -94,7 +94,7 @@ std::unique_ptr<cpp_template_parameter> parse_type_parameter(const detail::parse
{ {
keyword = cpp_template_keyword::concept_contraint; keyword = cpp_template_keyword::concept_contraint;
//try to extract the constraint token string // try to extract the constraint token string
constraint = extract_parameter_constraint(context, parent, stream.cur(), stream.end()); constraint = extract_parameter_constraint(context, parent, stream.cur(), stream.end());
stream.bump(); stream.bump();
if (stream.peek() == "<") if (stream.peek() == "<")
@ -114,7 +114,8 @@ std::unique_ptr<cpp_template_parameter> parse_type_parameter(const detail::parse
def = detail::parse_raw_type(context, stream, stream.end()); def = detail::parse_raw_type(context, stream, stream.end());
return cpp_template_type_parameter::build(*context.idx, detail::get_entity_id(cur), return cpp_template_type_parameter::build(*context.idx, detail::get_entity_id(cur),
name.c_str(), keyword, variadic, std::move(def), constraint); name.c_str(), keyword, variadic, std::move(def),
constraint);
} }
std::unique_ptr<cpp_template_parameter> parse_non_type_parameter( std::unique_ptr<cpp_template_parameter> parse_non_type_parameter(
@ -205,12 +206,6 @@ std::unique_ptr<cpp_template_template_parameter> parse_template_parameter(
std::string spelling; std::string spelling;
while (!stream.done()) while (!stream.done())
spelling += stream.get().c_str(); spelling += stream.get().c_str();
if (stream.unmunch())
{
DEBUG_ASSERT(!spelling.empty() && spelling.back() == '>', detail::assert_handler{});
spelling.pop_back();
DEBUG_ASSERT(!spelling.empty() && spelling.back() == '>', detail::assert_handler{});
}
builder.default_template( builder.default_template(
cpp_template_ref(detail::get_entity_id(target), std::move(spelling))); cpp_template_ref(detail::get_entity_id(target), std::move(spelling)));