jyapayne/cppast
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Remove tokenizer unmunch as it was never set

Browse source
This commit is contained in:
Jonathan Müller 2022-10-13 09:09:37 +02:00
commit 4c40bb7544
3 changed files with 23 additions and 46 deletions
Download patch file Download diff file Expand all files Collapse all files

7
src/libclang/cxtokenizer.cpp
View file

@ -396,7 +396,6 @@ Extent get_extent(const CXTranslationUnit& tu, const CXFile& file, const CXCurso
detail::cxtokenizer::cxtokenizer(const CXTranslationUnit& tu, const CXFile& file, detail::cxtokenizer::cxtokenizer(const CXTranslationUnit& tu, const CXFile& file,
const CXCursor& cur) const CXCursor& cur)
: unmunch_(false)
{ {
auto extent = get_extent(tu, file, cur); auto extent = get_extent(tu, file, cur);
@ -532,7 +531,8 @@ void detail::skip_brackets(detail::cxtoken_stream& stream)
stream.set_cur(std::next(closing)); stream.set_cur(std::next(closing));
} }
detail::cxtoken_iterator detail::find_sequence(detail::cxtoken_stream stream, detail::cxtoken_iterator start, detail::cxtoken_iterator detail::find_sequence(detail::cxtoken_stream stream,
detail::cxtoken_iterator start,
detail::cxtoken_iterator end) detail::cxtoken_iterator end)
{ {
detail::cxtoken_iterator search_start = stream.cur(); detail::cxtoken_iterator search_start = stream.cur();
@ -779,9 +779,6 @@ cpp_token_string detail::to_string(cxtoken_stream& stream, cxtoken_iterator end)
builder.add_token(cpp_token(get_kind(token), token.c_str())); builder.add_token(cpp_token(get_kind(token), token.c_str()));
} }
if (stream.unmunch())
builder.unmunch();
return builder.finish(); return builder.finish();
} }

21
src/libclang/cxtokenizer.hpp
View file

@ -78,25 +78,15 @@ namespace detail
return tokens_.end(); return tokens_.end();
} }
// if it returns true, the last token is ">>",
// but should haven been ">"
// only a problem for template parameters
bool unmunch() const noexcept
{
return unmunch_;
}
private: private:
std::vector<cxtoken> tokens_; std::vector<cxtoken> tokens_;
bool unmunch_;
}; };
class cxtoken_stream class cxtoken_stream
{ {
public: public:
explicit cxtoken_stream(const cxtokenizer& tokenizer, const CXCursor& cur) explicit cxtoken_stream(const cxtokenizer& tokenizer, const CXCursor& cur)
: cursor_(cur), begin_(tokenizer.begin()), cur_(begin_), end_(tokenizer.end()), : cursor_(cur), begin_(tokenizer.begin()), cur_(begin_), end_(tokenizer.end())
unmunch_(tokenizer.unmunch())
{} {}
const cxtoken& peek() const noexcept const cxtoken& peek() const noexcept
@ -155,15 +145,9 @@ namespace detail
cur_ = iter; cur_ = iter;
} }
bool unmunch() const noexcept
{
return unmunch_;
}
private: private:
CXCursor cursor_; CXCursor cursor_;
cxtoken_iterator begin_, cur_, end_; cxtoken_iterator begin_, cur_, end_;
bool unmunch_;
}; };
// skips the next token // skips the next token
@ -186,7 +170,8 @@ namespace detail
// finds the location of the given sequence in a stream // finds the location of the given sequence in a stream
// returns an iterator to the first token of the found sequence, or stream.end() if not found // returns an iterator to the first token of the found sequence, or stream.end() if not found
cxtoken_iterator find_sequence(cxtoken_stream stream, cxtoken_iterator start, cxtoken_iterator end); cxtoken_iterator find_sequence(cxtoken_stream stream, cxtoken_iterator start,
cxtoken_iterator end);
// parses attributes // parses attributes
// if skip_anyway is true it will bump even if no attributes have been parsed // if skip_anyway is true it will bump even if no attributes have been parsed

13
src/libclang/template_parser.cpp
View file

@ -51,7 +51,8 @@ cpp_token_string extract_parameter_constraint(const detail::parse_context& conte
detail::cxtokenizer tokenizer(context.tu, context.file, parent); detail::cxtokenizer tokenizer(context.tu, context.file, parent);
detail::cxtoken_stream stream(tokenizer, parent); detail::cxtoken_stream stream(tokenizer, parent);
detail::cxtoken_iterator found_start = detail::find_sequence(stream, target_range_start, target_range_end); detail::cxtoken_iterator found_start
= detail::find_sequence(stream, target_range_start, target_range_end);
if (found_start == stream.end()) if (found_start == stream.end())
return detail::to_string(stream, stream.cur() + 1); return detail::to_string(stream, stream.cur() + 1);
@ -79,7 +80,6 @@ std::unique_ptr<cpp_template_parameter> parse_type_parameter(const detail::parse
DEBUG_ASSERT(clang_getCursorKind(cur) == CXCursor_TemplateTypeParameter, DEBUG_ASSERT(clang_getCursorKind(cur) == CXCursor_TemplateTypeParameter,
detail::assert_handler{}); detail::assert_handler{});
detail::cxtokenizer tokenizer(context.tu, context.file, cur); detail::cxtokenizer tokenizer(context.tu, context.file, cur);
detail::cxtoken_stream stream(tokenizer, cur); detail::cxtoken_stream stream(tokenizer, cur);
auto name = detail::get_cursor_name(cur); auto name = detail::get_cursor_name(cur);
@ -114,7 +114,8 @@ std::unique_ptr<cpp_template_parameter> parse_type_parameter(const detail::parse
def = detail::parse_raw_type(context, stream, stream.end()); def = detail::parse_raw_type(context, stream, stream.end());
return cpp_template_type_parameter::build(*context.idx, detail::get_entity_id(cur), return cpp_template_type_parameter::build(*context.idx, detail::get_entity_id(cur),
name.c_str(), keyword, variadic, std::move(def), constraint); name.c_str(), keyword, variadic, std::move(def),
constraint);
} }
std::unique_ptr<cpp_template_parameter> parse_non_type_parameter( std::unique_ptr<cpp_template_parameter> parse_non_type_parameter(
@ -205,12 +206,6 @@ std::unique_ptr<cpp_template_template_parameter> parse_template_parameter(
std::string spelling; std::string spelling;
while (!stream.done()) while (!stream.done())
spelling += stream.get().c_str(); spelling += stream.get().c_str();
if (stream.unmunch())
{
DEBUG_ASSERT(!spelling.empty() && spelling.back() == '>', detail::assert_handler{});
spelling.pop_back();
DEBUG_ASSERT(!spelling.empty() && spelling.back() == '>', detail::assert_handler{});
}
builder.default_template( builder.default_template(
cpp_template_ref(detail::get_entity_id(target), std::move(spelling))); cpp_template_ref(detail::get_entity_id(target), std::move(spelling)));