jyapayne/cppast
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Remove tokenizer unmunch as it was never set

Browse source
This commit is contained in:
Jonathan Müller 2022-10-13 09:09:37 +02:00
commit 4c40bb7544
3 changed files with 23 additions and 46 deletions
Download patch file Download diff file Expand all files Collapse all files

17
src/libclang/cxtokenizer.cpp
View file

@ -70,7 +70,7 @@ public:
clang_disposeTokens(tu_, tokens_, no_);
}
simple_tokenizer(const simple_tokenizer&) = delete;
simple_tokenizer(const simple_tokenizer&) = delete;
simple_tokenizer& operator=(const simple_tokenizer&) = delete;
unsigned size() const noexcept
@ -396,7 +396,6 @@ Extent get_extent(const CXTranslationUnit& tu, const CXFile& file, const CXCurso
detail::cxtokenizer::cxtokenizer(const CXTranslationUnit& tu, const CXFile& file,
const CXCursor& cur)
: unmunch_(false)
{
auto extent = get_extent(tu, file, cur);
@ -532,19 +531,20 @@ void detail::skip_brackets(detail::cxtoken_stream& stream)
stream.set_cur(std::next(closing));
}
detail::cxtoken_iterator detail::find_sequence(detail::cxtoken_stream stream, detail::cxtoken_iterator start,
detail::cxtoken_iterator detail::find_sequence(detail::cxtoken_stream stream,
detail::cxtoken_iterator start,
detail::cxtoken_iterator end)
{
detail::cxtoken_iterator search_start = stream.cur();
while(search_start != stream.end())
while (search_start != stream.end())
{
detail::cxtoken_iterator search_iter = search_start;
detail::cxtoken_iterator seq_iter = start;
bool failed = false;
while(!failed && search_iter != stream.end() && seq_iter != end)
while (!failed && search_iter != stream.end() && seq_iter != end)
{
if(search_iter->value() != seq_iter->value()
if (search_iter->value() != seq_iter->value()
|| search_iter->kind() != seq_iter->kind())
{
failed = true;
@ -555,7 +555,7 @@ detail::cxtoken_iterator detail::find_sequence(detail::cxtoken_stream stream, de
++seq_iter;
}
}
if(!failed)
if (!failed)
return search_start;
++search_start;
@ -779,9 +779,6 @@ cpp_token_string detail::to_string(cxtoken_stream& stream, cxtoken_iterator end)
builder.add_token(cpp_token(get_kind(token), token.c_str()));
}
if (stream.unmunch())
builder.unmunch();
return builder.finish();
}

21
src/libclang/cxtokenizer.hpp
View file

@ -78,25 +78,15 @@ namespace detail
return tokens_.end();
}
// if it returns true, the last token is ">>",
// but should haven been ">"
// only a problem for template parameters
bool unmunch() const noexcept
{
return unmunch_;
}
private:
std::vector<cxtoken> tokens_;
bool unmunch_;
};
class cxtoken_stream
{
public:
explicit cxtoken_stream(const cxtokenizer& tokenizer, const CXCursor& cur)
: cursor_(cur), begin_(tokenizer.begin()), cur_(begin_), end_(tokenizer.end()),
unmunch_(tokenizer.unmunch())
: cursor_(cur), begin_(tokenizer.begin()), cur_(begin_), end_(tokenizer.end())
{}
const cxtoken& peek() const noexcept
@ -155,15 +145,9 @@ namespace detail
cur_ = iter;
}
bool unmunch() const noexcept
{
return unmunch_;
}
private:
CXCursor cursor_;
cxtoken_iterator begin_, cur_, end_;
bool unmunch_;
};
// skips the next token
@ -186,7 +170,8 @@ namespace detail
// finds the location of the given sequence in a stream
// returns an iterator to the first token of the found sequence, or stream.end() if not found
cxtoken_iterator find_sequence(cxtoken_stream stream, cxtoken_iterator start, cxtoken_iterator end);
cxtoken_iterator find_sequence(cxtoken_stream stream, cxtoken_iterator start,
cxtoken_iterator end);
// parses attributes
// if skip_anyway is true it will bump even if no attributes have been parsed

31
src/libclang/template_parser.cpp
View file

@ -43,16 +43,17 @@ type_safe::optional<typename TemplateT::builder> get_builder(const detail::parse
}
cpp_token_string extract_parameter_constraint(const detail::parse_context& context,
const CXCursor& parent,
detail::cxtoken_iterator target_range_start,
detail::cxtoken_iterator target_range_end)
const CXCursor& parent,
detail::cxtoken_iterator target_range_start,
detail::cxtoken_iterator target_range_end)
{
//search the parent context for the *exact* sequence in it's entirety
// search the parent context for the *exact* sequence in it's entirety
detail::cxtokenizer tokenizer(context.tu, context.file, parent);
detail::cxtoken_stream stream(tokenizer, parent);
detail::cxtoken_iterator found_start = detail::find_sequence(stream, target_range_start, target_range_end);
if(found_start == stream.end())
detail::cxtoken_iterator found_start
= detail::find_sequence(stream, target_range_start, target_range_end);
if (found_start == stream.end())
return detail::to_string(stream, stream.cur() + 1);
stream.set_cur(found_start);
@ -62,8 +63,8 @@ cpp_token_string extract_parameter_constraint(const detail::parse_context& conte
detail::cxtoken_iterator constraint_end = stream.cur();
stream.set_cur(found_start);
//seek backwards until we are at the start of the qualified name
while(stream.peek().value() != "<" && stream.peek().value() != ",")
// seek backwards until we are at the start of the qualified name
while (stream.peek().value() != "<" && stream.peek().value() != ",")
{
std::string str = stream.peek().value().std_str();
stream.bump_back();
@ -79,7 +80,6 @@ std::unique_ptr<cpp_template_parameter> parse_type_parameter(const detail::parse
DEBUG_ASSERT(clang_getCursorKind(cur) == CXCursor_TemplateTypeParameter,
detail::assert_handler{});
detail::cxtokenizer tokenizer(context.tu, context.file, cur);
detail::cxtoken_stream stream(tokenizer, cur);
auto name = detail::get_cursor_name(cur);
@ -94,7 +94,7 @@ std::unique_ptr<cpp_template_parameter> parse_type_parameter(const detail::parse
{
keyword = cpp_template_keyword::concept_contraint;
//try to extract the constraint token string
// try to extract the constraint token string
constraint = extract_parameter_constraint(context, parent, stream.cur(), stream.end());
stream.bump();
if (stream.peek() == "<")
@ -114,7 +114,8 @@ std::unique_ptr<cpp_template_parameter> parse_type_parameter(const detail::parse
def = detail::parse_raw_type(context, stream, stream.end());
return cpp_template_type_parameter::build(*context.idx, detail::get_entity_id(cur),
name.c_str(), keyword, variadic, std::move(def), constraint);
name.c_str(), keyword, variadic, std::move(def),
constraint);
}
std::unique_ptr<cpp_template_parameter> parse_non_type_parameter(
@ -205,12 +206,6 @@ std::unique_ptr<cpp_template_template_parameter> parse_template_parameter(
std::string spelling;
while (!stream.done())
spelling += stream.get().c_str();
if (stream.unmunch())
{
DEBUG_ASSERT(!spelling.empty() && spelling.back() == '>', detail::assert_handler{});
spelling.pop_back();
DEBUG_ASSERT(!spelling.empty() && spelling.back() == '>', detail::assert_handler{});
}
builder.default_template(
cpp_template_ref(detail::get_entity_id(target), std::move(spelling)));