jyapayne/cppast
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Fix unmunch for template arguments

Browse source 
Fixes #147.
This commit is contained in:
Jonathan Müller 2022-10-13 09:20:33 +02:00
commit 34e7bb4bf4
10 changed files with 67 additions and 36 deletions
Download patch file Download diff file Expand all files Collapse all files

2
src/libclang/class_parser.cpp
View file

@ -115,7 +115,7 @@ void add_base_class(cpp_class::builder& builder, const detail::parse_context& co
else
detail::skip_if(stream, to_string(access));
auto name = detail::to_string(stream, stream.end()).as_string();
auto name = detail::to_string(stream, stream.end(), false).as_string();
auto type = detail::parse_type(context, class_cur, clang_getCursorType(cur));
auto& base = builder.base_class(std::move(name), std::move(type), access, is_virtual);

5
src/libclang/concept_parser.cpp
View file

@ -24,9 +24,8 @@ std::unique_ptr<cpp_entity> detail::try_parse_cpp_concept(const detail::parse_co
if (stream.peek() != "<")
return nullptr;
auto closing_bracket_iter = detail::find_closing_bracket(stream);
auto params = to_string(stream, closing_bracket_iter);
auto closing_bracket = detail::find_closing_bracket(stream);
auto params = to_string(stream, closing_bracket.bracket, closing_bracket.unmunch);
if (!detail::skip_if(stream, ">"))
return nullptr;

49
src/libclang/cxtokenizer.cpp
View file

@ -8,8 +8,6 @@
#include "libclang_visitor.hpp"
#include "parse_error.hpp"
#include <iostream> // TODO
using namespace cppast;
detail::cxtoken::cxtoken(const CXTranslationUnit& tu_unit, const CXToken& token)
@ -474,7 +472,7 @@ bool is_comparison(CXTokenKind last_kind, const detail::cxtoken& cur, CXTokenKin
}
} // namespace
detail::cxtoken_iterator detail::find_closing_bracket(detail::cxtoken_stream stream)
detail::closing_bracket_pos detail::find_closing_bracket(detail::cxtoken_stream stream)
{
auto template_bracket = false;
auto open_bracket = stream.peek().c_str();
@ -494,11 +492,14 @@ detail::cxtoken_iterator detail::find_closing_bracket(detail::cxtoken_stream str
DEBUG_UNREACHABLE(parse_error_handler{}, stream.cursor(),
format("expected a bracket, got '", stream.peek().c_str(), "'"));
auto bracket_count = 1;
auto paren_count = 0; // internal nested parenthesis
auto last_token = CXToken_Comment;
auto bracket_count = 1;
auto paren_count = 0; // internal nested parenthesis
auto last_token = CXToken_Comment;
auto last_was_double_angle = false;
while (!stream.done() && bracket_count != 0)
{
last_was_double_angle = false;
auto& cur = stream.get();
if (paren_count == 0 && cur == open_bracket
&& !is_comparison(last_token, cur, stream.peek().kind()))
@ -507,8 +508,11 @@ detail::cxtoken_iterator detail::find_closing_bracket(detail::cxtoken_stream str
&& !is_comparison(last_token, cur, stream.peek().kind()))
--bracket_count;
else if (paren_count == 0 && template_bracket && cur == ">>")
{
// maximal munch
bracket_count -= 2;
last_was_double_angle = true;
}
else if (cur == "(" || cur == "{" || cur == "[")
++paren_count;
else if (cur == ")" || cur == "}" || cur == "]")
@ -516,19 +520,25 @@ detail::cxtoken_iterator detail::find_closing_bracket(detail::cxtoken_stream str
last_token = cur.kind();
}
stream.bump_back();
// only check first parameter, token might be ">>"
DEBUG_ASSERT(bracket_count == 0 && paren_count == 0
&& stream.peek().value()[0] == close_bracket[0],
parse_error_handler{}, stream.cursor(),
DEBUG_ASSERT(bracket_count == 0 && paren_count == 0, parse_error_handler{}, stream.cursor(),
"find_closing_bracket() internal parse error");
return stream.cur();
if (last_was_double_angle)
{
return {stream.cur(), stream.cur(), true};
}
else
{
auto after = stream.cur();
stream.bump_back();
return {stream.cur(), after, false};
}
}
void detail::skip_brackets(detail::cxtoken_stream& stream)
{
auto closing = find_closing_bracket(stream);
stream.set_cur(std::next(closing));
stream.set_cur(closing.after);
}
detail::cxtoken_iterator detail::find_sequence(detail::cxtoken_stream stream,
@ -605,9 +615,9 @@ cpp_token_string parse_attribute_arguments(detail::cxtoken_stream& stream)
auto end = find_closing_bracket(stream);
skip(stream, "(");
auto arguments = detail::to_string(stream, end);
auto arguments = detail::to_string(stream, end.bracket, end.unmunch);
stream.set_cur(end.bracket);
stream.set_cur(end);
skip(stream, ")");
return arguments;
@ -769,7 +779,7 @@ cpp_token_kind get_kind(const detail::cxtoken& token)
}
} // namespace
cpp_token_string detail::to_string(cxtoken_stream& stream, cxtoken_iterator end)
cpp_token_string detail::to_string(cxtoken_stream& stream, cxtoken_iterator end, bool unmunch)
{
cpp_token_string::builder builder;
@ -779,6 +789,9 @@ cpp_token_string detail::to_string(cxtoken_stream& stream, cxtoken_iterator end)
builder.add_token(cpp_token(get_kind(token), token.c_str()));
}
if (unmunch)
builder.unmunch();
return builder.finish();
}
@ -800,8 +813,8 @@ bool detail::append_scope(detail::cxtoken_stream& stream, std::string& scope)
}
else if (stream.peek() == "<")
{
auto iter = detail::find_closing_bracket(stream);
scope += detail::to_string(stream, iter).as_string();
auto pos = detail::find_closing_bracket(stream);
scope += detail::to_string(stream, pos.bracket, pos.unmunch).as_string();
if (!detail::skip_if(stream, ">>"))
detail::skip(stream, ">");
scope += ">";

13
src/libclang/cxtokenizer.hpp
View file

@ -158,10 +158,19 @@ namespace detail
// if multi_token == true, str can consist of multiple tokens optionally separated by whitespace
bool skip_if(cxtoken_stream& stream, const char* str, bool multi_token = false);
struct closing_bracket_pos
{
// If unmunch == false: bracket points to the closing bracket, after is the iterator after
// that. If unmunch == true: bracket points to >>, after points to the same >>; only one
// bracket is part of the matching closing one.
cxtoken_iterator bracket, after;
bool unmunch;
};
// returns the location of the closing bracket
// the current token must be (,[,{ or <
// note: < might not work in the arguments of a template specialization
cxtoken_iterator find_closing_bracket(cxtoken_stream stream);
closing_bracket_pos find_closing_bracket(cxtoken_stream stream);
// skips brackets
// the current token must be (,[,{ or <
@ -178,7 +187,7 @@ namespace detail
cpp_attribute_list parse_attributes(cxtoken_stream& stream, bool skip_anyway = false);
// converts a token range to a string
cpp_token_string to_string(cxtoken_stream& stream, cxtoken_iterator end);
cpp_token_string to_string(cxtoken_stream& stream, cxtoken_iterator end, bool unmunch);
// appends token to scope, if it is still valid
// else clears it

4
src/libclang/expression_parser.cpp
View file

@ -17,7 +17,7 @@ std::unique_ptr<cpp_expression> detail::parse_expression(const detail::parse_con
detail::cxtoken_stream stream(tokenizer, cur);
auto type = parse_type(context, cur, clang_getCursorType(cur));
auto expr = to_string(stream, stream.end());
auto expr = to_string(stream, stream.end(), false);
if (kind == CXCursor_CallExpr && (expr.empty() || expr.back().spelling != ")"))
{
// we have a call expression that doesn't end in a closing parentheses
@ -42,6 +42,6 @@ std::unique_ptr<cpp_expression> detail::parse_raw_expression(const parse_context
if (stream.done())
return nullptr;
auto expr = to_string(stream, std::prev(end)->value() == ";" ? std::prev(end) : end);
auto expr = to_string(stream, std::prev(end)->value() == ";" ? std::prev(end) : end, false);
return cpp_unexposed_expression::build(std::move(type), std::move(expr));
}

4
src/libclang/function_parser.cpp
View file

@ -368,7 +368,7 @@ std::unique_ptr<cpp_expression> parse_noexcept(detail::cxtoken_stream& stre
auto closing = detail::find_closing_bracket(stream);
detail::skip(stream, "(");
auto expr = detail::parse_raw_expression(context, stream, closing, std::move(type));
auto expr = detail::parse_raw_expression(context, stream, closing.bracket, std::move(type));
detail::skip(stream, ")");
return expr;
@ -745,7 +745,7 @@ std::unique_ptr<cpp_entity> detail::parse_cpp_conversion_op(const detail::parse_
// read the type
stream.set_cur(type_start);
auto type_spelling = detail::to_string(stream, type_end).as_string();
auto type_spelling = detail::to_string(stream, type_end, false).as_string();
// parse arguments again
detail::skip(stream, "(");

2
src/libclang/parse_functions.cpp
View file

@ -210,7 +210,7 @@ try
// build unexposed entity
detail::cxtokenizer tokenizer(context.tu, context.file, cur);
detail::cxtoken_stream stream(tokenizer, cur);
auto spelling = detail::to_string(stream, stream.end());
auto spelling = detail::to_string(stream, stream.end(), false);
if (spelling.begin() + 1 == spelling.end() && spelling.front().spelling == ";")
// unnecessary semicolon
return nullptr;

8
src/libclang/template_parser.cpp
View file

@ -54,7 +54,7 @@ cpp_token_string extract_parameter_constraint(const detail::parse_context& conte
detail::cxtoken_iterator found_start
= detail::find_sequence(stream, target_range_start, target_range_end);
if (found_start == stream.end())
return detail::to_string(stream, stream.cur() + 1);
return detail::to_string(stream, stream.cur() + 1, false);
stream.set_cur(found_start);
stream.bump();
@ -70,7 +70,7 @@ cpp_token_string extract_parameter_constraint(const detail::parse_context& conte
stream.bump_back();
}
stream.bump();
return detail::to_string(stream, constraint_end);
return detail::to_string(stream, constraint_end, false);
}
std::unique_ptr<cpp_template_parameter> parse_type_parameter(const detail::parse_context& context,
@ -316,10 +316,10 @@ void parse_arguments(Builder& b, const detail::parse_context& context, const CXC
if (stream.peek() == "<")
{
auto iter = detail::find_closing_bracket(stream);
auto closing = detail::find_closing_bracket(stream);
stream.bump();
auto args = detail::to_string(stream, iter);
auto args = detail::to_string(stream, closing.bracket, closing.unmunch);
b.add_unexposed_arguments(std::move(args));
}
else

2
src/libclang/type_parser.cpp
View file

@ -724,7 +724,7 @@ std::unique_ptr<cpp_type> detail::parse_raw_type(const detail::parse_context&,
detail::cxtoken_stream& stream,
detail::cxtoken_iterator end)
{
auto result = detail::to_string(stream, end);
auto result = detail::to_string(stream, end, false);
return cpp_unexposed_type::build(result.as_string());
}

14
test/cpp_class_template.cpp
View file

@ -73,6 +73,13 @@ struct e
template <>
class a<int> {};
// full specialization with munch
/// template<>
/// class a<a<int>>{
/// };
template <>
class a<a<int>> {};
/// template<>
/// struct b<0,int>{
/// };
@ -245,7 +252,10 @@ template class a<int>;
if (templ.is_full_specialization())
{
check_template_parameters(templ, {});
REQUIRE(templ.unexposed_arguments().as_string() == "int");
auto args = templ.unexposed_arguments().as_string();
if (args != "int" && args != "a<int>")
FAIL("invalid args for specialization");
}
else
{
@ -273,5 +283,5 @@ template class a<int>;
else
REQUIRE(false);
});
REQUIRE(count == 6u);
REQUIRE(count == 7u);
}