Fix unmunch for template arguments

Fixes #147.

src/libclang/class_parser.cpp

@@ -115,7 +115,7 @@ void add_base_class(cpp_class::builder& builder, const detail::parse_context& co
     else
         detail::skip_if(stream, to_string(access));
 
-    auto name = detail::to_string(stream, stream.end()).as_string();
+    auto name = detail::to_string(stream, stream.end(), false).as_string();
 
     auto  type = detail::parse_type(context, class_cur, clang_getCursorType(cur));
     auto& base = builder.base_class(std::move(name), std::move(type), access, is_virtual);

src/libclang/concept_parser.cpp

@@ -24,9 +24,8 @@ std::unique_ptr<cpp_entity> detail::try_parse_cpp_concept(const detail::parse_co
     if (stream.peek() != "<")
         return nullptr;
 
-
+    auto closing_bracket = detail::find_closing_bracket(stream);
-    auto closing_bracket_iter = detail::find_closing_bracket(stream);
+    auto params          = to_string(stream, closing_bracket.bracket, closing_bracket.unmunch);
-    auto params               = to_string(stream, closing_bracket_iter);
 
     if (!detail::skip_if(stream, ">"))
         return nullptr;

src/libclang/cxtokenizer.cpp

@@ -8,8 +8,6 @@
 #include "libclang_visitor.hpp"
 #include "parse_error.hpp"
 
-#include <iostream> // TODO
-
 using namespace cppast;
 
 detail::cxtoken::cxtoken(const CXTranslationUnit& tu_unit, const CXToken& token)
@@ -474,7 +472,7 @@ bool is_comparison(CXTokenKind last_kind, const detail::cxtoken& cur, CXTokenKin
 }
 } // namespace
 
-detail::cxtoken_iterator detail::find_closing_bracket(detail::cxtoken_stream stream)
+detail::closing_bracket_pos detail::find_closing_bracket(detail::cxtoken_stream stream)
 {
     auto        template_bracket = false;
     auto        open_bracket     = stream.peek().c_str();
@@ -494,11 +492,14 @@ detail::cxtoken_iterator detail::find_closing_bracket(detail::cxtoken_stream str
         DEBUG_UNREACHABLE(parse_error_handler{}, stream.cursor(),
                           format("expected a bracket, got '", stream.peek().c_str(), "'"));
 
-    auto bracket_count = 1;
+    auto bracket_count         = 1;
-    auto paren_count   = 0; // internal nested parenthesis
+    auto paren_count           = 0; // internal nested parenthesis
-    auto last_token    = CXToken_Comment;
+    auto last_token            = CXToken_Comment;
+    auto last_was_double_angle = false;
     while (!stream.done() && bracket_count != 0)
     {
+        last_was_double_angle = false;
+
         auto& cur = stream.get();
         if (paren_count == 0 && cur == open_bracket
             && !is_comparison(last_token, cur, stream.peek().kind()))
@@ -507,8 +508,11 @@ detail::cxtoken_iterator detail::find_closing_bracket(detail::cxtoken_stream str
                  && !is_comparison(last_token, cur, stream.peek().kind()))
             --bracket_count;
         else if (paren_count == 0 && template_bracket && cur == ">>")
+        {
             // maximal munch
             bracket_count -= 2;
+            last_was_double_angle = true;
+        }
         else if (cur == "(" || cur == "{" || cur == "[")
             ++paren_count;
         else if (cur == ")" || cur == "}" || cur == "]")
@@ -516,19 +520,25 @@ detail::cxtoken_iterator detail::find_closing_bracket(detail::cxtoken_stream str
 
         last_token = cur.kind();
     }
-    stream.bump_back();
+    DEBUG_ASSERT(bracket_count == 0 && paren_count == 0, parse_error_handler{}, stream.cursor(),
-    // only check first parameter, token might be ">>"
-    DEBUG_ASSERT(bracket_count == 0 && paren_count == 0
-                     && stream.peek().value()[0] == close_bracket[0],
-                 parse_error_handler{}, stream.cursor(),
                  "find_closing_bracket() internal parse error");
-    return stream.cur();
+
+    if (last_was_double_angle)
+    {
+        return {stream.cur(), stream.cur(), true};
+    }
+    else
+    {
+        auto after = stream.cur();
+        stream.bump_back();
+        return {stream.cur(), after, false};
+    }
 }
 
 void detail::skip_brackets(detail::cxtoken_stream& stream)
 {
     auto closing = find_closing_bracket(stream);
-    stream.set_cur(std::next(closing));
+    stream.set_cur(closing.after);
 }
 
 detail::cxtoken_iterator detail::find_sequence(detail::cxtoken_stream   stream,
@@ -605,9 +615,9 @@ cpp_token_string parse_attribute_arguments(detail::cxtoken_stream& stream)
     auto end = find_closing_bracket(stream);
     skip(stream, "(");
 
-    auto arguments = detail::to_string(stream, end);
+    auto arguments = detail::to_string(stream, end.bracket, end.unmunch);
+    stream.set_cur(end.bracket);
 
-    stream.set_cur(end);
     skip(stream, ")");
 
     return arguments;
@@ -769,7 +779,7 @@ cpp_token_kind get_kind(const detail::cxtoken& token)
 }
 } // namespace
 
-cpp_token_string detail::to_string(cxtoken_stream& stream, cxtoken_iterator end)
+cpp_token_string detail::to_string(cxtoken_stream& stream, cxtoken_iterator end, bool unmunch)
 {
     cpp_token_string::builder builder;
 
@@ -779,6 +789,9 @@ cpp_token_string detail::to_string(cxtoken_stream& stream, cxtoken_iterator end)
         builder.add_token(cpp_token(get_kind(token), token.c_str()));
     }
 
+    if (unmunch)
+        builder.unmunch();
+
     return builder.finish();
 }
 
@@ -800,8 +813,8 @@ bool detail::append_scope(detail::cxtoken_stream& stream, std::string& scope)
     }
     else if (stream.peek() == "<")
     {
-        auto iter = detail::find_closing_bracket(stream);
+        auto pos = detail::find_closing_bracket(stream);
-        scope += detail::to_string(stream, iter).as_string();
+        scope += detail::to_string(stream, pos.bracket, pos.unmunch).as_string();
         if (!detail::skip_if(stream, ">>"))
             detail::skip(stream, ">");
         scope += ">";

src/libclang/cxtokenizer.hpp

@@ -158,10 +158,19 @@ namespace detail
     // if multi_token == true, str can consist of multiple tokens optionally separated by whitespace
     bool skip_if(cxtoken_stream& stream, const char* str, bool multi_token = false);
 
+    struct closing_bracket_pos
+    {
+        // If unmunch == false: bracket points to the closing bracket, after is the iterator after
+        // that. If unmunch == true: bracket points to >>, after points to the same >>; only one
+        // bracket is part of the matching closing one.
+        cxtoken_iterator bracket, after;
+        bool             unmunch;
+    };
+
     // returns the location of the closing bracket
     // the current token must be (,[,{ or <
     // note: < might not work in the arguments of a template specialization
-    cxtoken_iterator find_closing_bracket(cxtoken_stream stream);
+    closing_bracket_pos find_closing_bracket(cxtoken_stream stream);
 
     // skips brackets
     // the current token must be (,[,{ or <
@@ -178,7 +187,7 @@ namespace detail
     cpp_attribute_list parse_attributes(cxtoken_stream& stream, bool skip_anyway = false);
 
     // converts a token range to a string
-    cpp_token_string to_string(cxtoken_stream& stream, cxtoken_iterator end);
+    cpp_token_string to_string(cxtoken_stream& stream, cxtoken_iterator end, bool unmunch);
 
     // appends token to scope, if it is still valid
     // else clears it

src/libclang/expression_parser.cpp

@@ -17,7 +17,7 @@ std::unique_ptr<cpp_expression> detail::parse_expression(const detail::parse_con
     detail::cxtoken_stream stream(tokenizer, cur);
 
     auto type = parse_type(context, cur, clang_getCursorType(cur));
-    auto expr = to_string(stream, stream.end());
+    auto expr = to_string(stream, stream.end(), false);
     if (kind == CXCursor_CallExpr && (expr.empty() || expr.back().spelling != ")"))
     {
         // we have a call expression that doesn't end in a closing parentheses
@@ -42,6 +42,6 @@ std::unique_ptr<cpp_expression> detail::parse_raw_expression(const parse_context
     if (stream.done())
         return nullptr;
 
-    auto expr = to_string(stream, std::prev(end)->value() == ";" ? std::prev(end) : end);
+    auto expr = to_string(stream, std::prev(end)->value() == ";" ? std::prev(end) : end, false);
     return cpp_unexposed_expression::build(std::move(type), std::move(expr));
 }

src/libclang/function_parser.cpp

@@ -368,7 +368,7 @@ std::unique_ptr<cpp_expression> parse_noexcept(detail::cxtoken_stream&      stre
     auto closing = detail::find_closing_bracket(stream);
 
     detail::skip(stream, "(");
-    auto expr = detail::parse_raw_expression(context, stream, closing, std::move(type));
+    auto expr = detail::parse_raw_expression(context, stream, closing.bracket, std::move(type));
     detail::skip(stream, ")");
 
     return expr;
@@ -745,7 +745,7 @@ std::unique_ptr<cpp_entity> detail::parse_cpp_conversion_op(const detail::parse_
 
     // read the type
     stream.set_cur(type_start);
-    auto type_spelling = detail::to_string(stream, type_end).as_string();
+    auto type_spelling = detail::to_string(stream, type_end, false).as_string();
 
     // parse arguments again
     detail::skip(stream, "(");

src/libclang/parse_functions.cpp

@@ -210,7 +210,7 @@ try
         // build unexposed entity
         detail::cxtokenizer    tokenizer(context.tu, context.file, cur);
         detail::cxtoken_stream stream(tokenizer, cur);
-        auto                   spelling = detail::to_string(stream, stream.end());
+        auto                   spelling = detail::to_string(stream, stream.end(), false);
         if (spelling.begin() + 1 == spelling.end() && spelling.front().spelling == ";")
             // unnecessary semicolon
             return nullptr;

src/libclang/template_parser.cpp

@@ -54,7 +54,7 @@ cpp_token_string extract_parameter_constraint(const detail::parse_context& conte
     detail::cxtoken_iterator found_start
         = detail::find_sequence(stream, target_range_start, target_range_end);
     if (found_start == stream.end())
-        return detail::to_string(stream, stream.cur() + 1);
+        return detail::to_string(stream, stream.cur() + 1, false);
 
     stream.set_cur(found_start);
     stream.bump();
@@ -70,7 +70,7 @@ cpp_token_string extract_parameter_constraint(const detail::parse_context& conte
         stream.bump_back();
     }
     stream.bump();
-    return detail::to_string(stream, constraint_end);
+    return detail::to_string(stream, constraint_end, false);
 }
 
 std::unique_ptr<cpp_template_parameter> parse_type_parameter(const detail::parse_context& context,
@@ -316,10 +316,10 @@ void parse_arguments(Builder& b, const detail::parse_context& context, const CXC
 
     if (stream.peek() == "<")
     {
-        auto iter = detail::find_closing_bracket(stream);
+        auto closing = detail::find_closing_bracket(stream);
         stream.bump();
 
-        auto args = detail::to_string(stream, iter);
+        auto args = detail::to_string(stream, closing.bracket, closing.unmunch);
         b.add_unexposed_arguments(std::move(args));
     }
     else

src/libclang/type_parser.cpp

@@ -724,7 +724,7 @@ std::unique_ptr<cpp_type> detail::parse_raw_type(const detail::parse_context&,
                                                  detail::cxtoken_stream&  stream,
                                                  detail::cxtoken_iterator end)
 {
-    auto result = detail::to_string(stream, end);
+    auto result = detail::to_string(stream, end, false);
     return cpp_unexposed_type::build(result.as_string());
 }
 

test/cpp_class_template.cpp

@@ -73,6 +73,13 @@ struct e
 template <>
 class a<int> {};
 
+// full specialization with munch
+/// template<>
+/// class a<a<int>>{
+/// };
+template <>
+class a<a<int>> {};
+
 /// template<>
 /// struct b<0,int>{
 /// };
@@ -245,7 +252,10 @@ template class a<int>;
             if (templ.is_full_specialization())
             {
                 check_template_parameters(templ, {});
-                REQUIRE(templ.unexposed_arguments().as_string() == "int");
+
+                auto args = templ.unexposed_arguments().as_string();
+                if (args != "int" && args != "a<int>")
+                    FAIL("invalid args for specialization");
             }
             else
             {
@@ -273,5 +283,5 @@ template class a<int>;
         else
             REQUIRE(false);
     });
-    REQUIRE(count == 6u);
+    REQUIRE(count == 7u);
 }