Add token_stream

src/libclang/parse_error.hpp

@@ -6,6 +6,7 @@
 #define CPPAST_PARSE_ERROR_HPP_INCLUDED
 
 #include <stdexcept>
+#include <sstream>
 
 #include <debug_assert.hpp>
 #include <cppast/diagnostic.hpp>
@@ -45,11 +46,20 @@ namespace cppast
         struct parse_error_handler : debug_assert::set_level<1>, debug_assert::allow_exception
         {
             static void handle(const debug_assert::source_location&, const char*,
-                               const CXCursor& cur, const char* message)
+                               const CXCursor& cur, std::string message)
             {
-                throw parse_error(cur, message);
+                throw parse_error(cur, std::move(message));
             }
         };
+
+        template <typename... Args>
+        std::string format(Args&&... args)
+        {
+            std::ostringstream stream;
+            int                dummy[] = {(stream << std::forward<Args>(args), 0)...};
+            (void)dummy;
+            return stream.str();
+        }
     }
 } // namespace cppast::detail
 

src/libclang/tokenizer.cpp

@@ -167,3 +167,102 @@ detail::tokenizer::tokenizer(const detail::cxtranslation_unit& tu, const CXFile&
     for (auto i = 0u; i != tokenizer.size(); ++i)
         tokens_.emplace_back(tu, tokenizer[i]);
 }
+
+void detail::skip(detail::token_stream& stream, const char* str)
+{
+    auto& token = stream.peek();
+    DEBUG_ASSERT(token == str, parse_error_handler{}, stream.cursor(),
+                 format("expected '", str, "', got '", token.c_str(), "'"));
+    stream.bump();
+}
+
+bool detail::skip_if(detail::token_stream& stream, const char* str)
+{
+    auto& token = stream.peek();
+    if (token != str)
+        return false;
+    stream.bump();
+    return true;
+}
+
+detail::token_iterator detail::find_closing_bracket(detail::token_stream stream)
+{
+    auto        template_bracket = false;
+    auto        open_bracket     = stream.peek().c_str();
+    const char* close_bracket    = nullptr;
+    if (skip_if(stream, "("))
+        close_bracket = ")";
+    else if (skip_if(stream, "{"))
+        close_bracket = "}";
+    else if (skip_if(stream, "["))
+        close_bracket = "]";
+    else if (skip_if(stream, "<"))
+    {
+        close_bracket    = "<";
+        template_bracket = true;
+    }
+    else
+        DEBUG_UNREACHABLE(parse_error_handler{}, stream.cursor(),
+                          format("expected a bracket, got '", stream.peek().c_str(), "'"));
+
+    auto bracket_count = 1;
+    auto paren_count   = 0; // internal nested parenthesis
+    while (bracket_count != 0)
+    {
+        auto& cur = stream.get().value();
+        if (paren_count == 0 && cur == open_bracket)
+            ++bracket_count;
+        else if (paren_count == 0 && cur == close_bracket)
+            --bracket_count;
+        else if (paren_count == 0 && template_bracket && cur == ">>")
+            // maximal munch
+            bracket_count -= 2u;
+        else if (cur == "(")
+            ++paren_count;
+        else if (cur == ")")
+            --paren_count;
+    }
+    stream.bump_back();
+    DEBUG_ASSERT(paren_count == 0 && stream.peek().value() == close_bracket, parse_error_handler{},
+                 stream.cursor(), "find_closing_bracket() internal parse error");
+    return stream.cur();
+}
+
+void detail::skip_brackets(detail::token_stream& stream)
+{
+    auto closing = find_closing_bracket(stream);
+    stream.set_cur(std::next(closing));
+}
+
+bool detail::skip_attribute(detail::token_stream& stream)
+{
+    if (skip_if(stream, "[") && stream.peek() == "[")
+    {
+        // C++11 attribute
+        // [[<attribute>]]
+        //  ^
+        skip_brackets(stream);
+        // [[<attribute>]]
+        //               ^
+        skip(stream, "]");
+        return true;
+    }
+    else if (skip_if(stream, "__attribute__"))
+    {
+        // GCC/clang attributes
+        // __attribute__(<attribute>)
+        //              ^
+        skip_brackets(stream);
+        return true;
+    }
+    else if (skip_if(stream, "__declspec"))
+    {
+        // MSVC declspec
+        // __declspec(<attribute>)
+        //           ^
+        skip_brackets(stream);
+        return true;
+    }
+
+    return false;
+}

src/libclang/tokenizer.hpp

@@ -38,18 +38,40 @@ namespace cppast
             CXTokenKind kind_;
         };
 
+        inline bool operator==(const token& tok, const char* str) noexcept
+        {
+            return tok.value() == str;
+        }
+
+        inline bool operator==(const char* str, const token& tok) noexcept
+        {
+            return str == tok.value();
+        }
+
+        inline bool operator!=(const token& tok, const char* str) noexcept
+        {
+            return !(tok == str);
+        }
+
+        inline bool operator!=(const char* str, const token& tok) noexcept
+        {
+            return !(str == tok);
+        }
+
+        using token_iterator = std::vector<token>::const_iterator;
+
         class tokenizer
         {
         public:
             explicit tokenizer(const cxtranslation_unit& tu, const CXFile& file,
                                const CXCursor& cur);
 
-            std::vector<token>::const_iterator begin() const noexcept
+            token_iterator begin() const noexcept
             {
                 return tokens_.begin();
             }
 
-            std::vector<token>::const_iterator end() const noexcept
+            token_iterator end() const noexcept
             {
                 return tokens_.end();
             }
@@ -57,6 +79,85 @@ namespace cppast
         private:
             std::vector<token> tokens_;
         };
+
+        class token_stream
+        {
+        public:
+            explicit token_stream(const tokenizer& tokenizer, const CXCursor& cur)
+            : cursor_(cur), begin_(tokenizer.begin()), cur_(begin_), end_(tokenizer.end())
+            {
+            }
+
+            const token& peek() const noexcept
+            {
+                if (done())
+                    return *std::prev(end_);
+                return *cur_;
+            }
+
+            void bump() noexcept
+            {
+                if (cur_ != end_)
+                    ++cur_;
+            }
+
+            void bump_back() noexcept
+            {
+                if (cur_ != begin_)
+                    --cur_;
+            }
+
+            const token& get() noexcept
+            {
+                auto& result = peek();
+                bump();
+                return result;
+            }
+
+            bool done() const noexcept
+            {
+                return cur_ == end_;
+            }
+
+            const CXCursor& cursor() const noexcept
+            {
+                return cursor_;
+            }
+
+            token_iterator cur() const noexcept
+            {
+                return cur_;
+            }
+
+            void set_cur(token_iterator iter) noexcept
+            {
+                cur_ = iter;
+            }
+
+        private:
+            CXCursor       cursor_;
+            token_iterator begin_, cur_, end_;
+        };
+
+        // skips the next token
+        // asserts that it has the given string
+        void skip(token_stream& stream, const char* str);
+
+        // skips the next token if it has the given string
+        bool skip_if(token_stream& stream, const char* str);
+
+        // returns the location of the closing bracket
+        // the current token must be (,[,{ or <
+        // note: < might not work in the arguments of a template specialization
+        token_iterator find_closing_bracket(token_stream stream);
+
+        // skips brackets
+        // the current token must be (,[,{ or <
+        // note: < might not work in the arguments of a template specialization
+        void skip_brackets(token_stream& stream);
+
+        // skips an attribute
+        bool skip_attribute(token_stream& stream);
     }
 } // namespace cppast::detail