From cfac41c7b53facfa70c4b2ae73128336e8bff905 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonathan=20M=C3=BCller?= Date: Thu, 12 Oct 2017 18:43:12 +0200 Subject: [PATCH] Rename tokenizer stuff to include cx prefix --- src/CMakeLists.txt | 4 +- src/libclang/class_parser.cpp | 8 +-- .../{tokenizer.cpp => cxtokenizer.cpp} | 27 +++---- .../{tokenizer.hpp => cxtokenizer.hpp} | 70 +++++++++---------- src/libclang/debug_helper.cpp | 4 +- src/libclang/enum_parser.cpp | 10 +-- src/libclang/expression_parser.cpp | 8 +-- src/libclang/function_parser.cpp | 40 +++++------ src/libclang/language_linkage_parser.cpp | 4 +- src/libclang/libclang_parser.cpp | 2 +- src/libclang/namespace_parser.cpp | 16 ++--- src/libclang/parse_functions.cpp | 8 +-- src/libclang/parse_functions.hpp | 10 +-- src/libclang/template_parser.cpp | 20 +++--- src/libclang/type_parser.cpp | 4 +- src/libclang/variable_parser.cpp | 6 +- 16 files changed, 121 insertions(+), 120 deletions(-) rename src/libclang/{tokenizer.cpp => cxtokenizer.cpp} (94%) rename src/libclang/{tokenizer.hpp => cxtokenizer.hpp} (65%) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 04e1714..1e0ccd6 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -95,8 +95,8 @@ set(libclang_source libclang/preprocessor.hpp libclang/raii_wrapper.hpp libclang/template_parser.cpp - libclang/tokenizer.cpp - libclang/tokenizer.hpp + libclang/cxtokenizer.cpp + libclang/cxtokenizer.hpp libclang/type_parser.cpp libclang/variable_parser.cpp) diff --git a/src/libclang/class_parser.cpp b/src/libclang/class_parser.cpp index ea661ad..169a958 100644 --- a/src/libclang/class_parser.cpp +++ b/src/libclang/class_parser.cpp @@ -71,8 +71,8 @@ namespace auto access = convert_access(cur); auto is_virtual = clang_isVirtualBase(cur) != 0u; - detail::tokenizer tokenizer(context.tu, context.file, cur); - detail::token_stream stream(tokenizer, cur); + detail::cxtokenizer tokenizer(context.tu, context.file, cur); + detail::cxtoken_stream stream(tokenizer, cur); // [] [virtual] [] // can't use spelling to get the name @@ -108,8 +108,8 @@ std::unique_ptr detail::parse_cpp_class(const detail::parse_context& clang_getCursorLexicalParent(cur))) { // out-of-line definition - detail::tokenizer tokenizer(context.tu, context.file, cur); - detail::token_stream stream(tokenizer, cur); + detail::cxtokenizer tokenizer(context.tu, context.file, cur); + detail::cxtoken_stream stream(tokenizer, cur); std::string name = detail::get_cursor_name(cur).c_str(); auto pos = name.find('<'); diff --git a/src/libclang/tokenizer.cpp b/src/libclang/cxtokenizer.cpp similarity index 94% rename from src/libclang/tokenizer.cpp rename to src/libclang/cxtokenizer.cpp index f6895c5..51c3a87 100644 --- a/src/libclang/tokenizer.cpp +++ b/src/libclang/cxtokenizer.cpp @@ -2,7 +2,7 @@ // This file is subject to the license terms in the LICENSE file // found in the top-level directory of this distribution. -#include "tokenizer.hpp" +#include "cxtokenizer.hpp" #include @@ -11,7 +11,7 @@ using namespace cppast; -detail::token::token(const CXTranslationUnit& tu_unit, const CXToken& token) +detail::cxtoken::cxtoken(const CXTranslationUnit& tu_unit, const CXToken& token) : value_(clang_getTokenSpelling(tu_unit, token)), kind_(clang_getTokenKind(token)) { } @@ -237,7 +237,8 @@ namespace } } -detail::tokenizer::tokenizer(const CXTranslationUnit& tu, const CXFile& file, const CXCursor& cur) +detail::cxtokenizer::cxtokenizer(const CXTranslationUnit& tu, const CXFile& file, + const CXCursor& cur) { auto extent = get_extent(tu, file, cur, unmunch_); @@ -247,7 +248,7 @@ detail::tokenizer::tokenizer(const CXTranslationUnit& tu, const CXFile& file, co tokens_.emplace_back(tu, tokenizer[i]); } -void detail::skip(detail::token_stream& stream, const char* str) +void detail::skip(detail::cxtoken_stream& stream, const char* str) { if (*str) { @@ -263,7 +264,7 @@ void detail::skip(detail::token_stream& stream, const char* str) namespace { - bool starts_with(const char*& str, const detail::token& t) + bool starts_with(const char*& str, const detail::cxtoken& t) { if (std::strncmp(str, t.c_str(), t.value().length()) != 0) return false; @@ -274,7 +275,7 @@ namespace } } -bool detail::skip_if(detail::token_stream& stream, const char* str, bool multi_token) +bool detail::skip_if(detail::cxtoken_stream& stream, const char* str, bool multi_token) { if (!*str) return true; @@ -298,7 +299,7 @@ namespace { // whether or not the current angle bracket can be a comparison // note: this is a heuristic I hope works often enough - bool is_comparison(CXTokenKind last_kind, const detail::token& cur, CXTokenKind next_kind) + bool is_comparison(CXTokenKind last_kind, const detail::cxtoken& cur, CXTokenKind next_kind) { if (cur == "<") return last_kind == CXToken_Literal; @@ -308,7 +309,7 @@ namespace } } -detail::token_iterator detail::find_closing_bracket(detail::token_stream stream) +detail::cxtoken_iterator detail::find_closing_bracket(detail::cxtoken_stream stream) { auto template_bracket = false; auto open_bracket = stream.peek().c_str(); @@ -359,7 +360,7 @@ detail::token_iterator detail::find_closing_bracket(detail::token_stream stream) return stream.cur(); } -void detail::skip_brackets(detail::token_stream& stream) +void detail::skip_brackets(detail::cxtoken_stream& stream) { auto closing = find_closing_bracket(stream); stream.set_cur(std::next(closing)); @@ -367,7 +368,7 @@ void detail::skip_brackets(detail::token_stream& stream) namespace { - bool skip_attribute_impl(detail::token_stream& stream) + bool skip_attribute_impl(detail::cxtoken_stream& stream) { if (skip_if(stream, "[") && stream.peek() == "[") { @@ -401,7 +402,7 @@ namespace } } -bool detail::skip_attribute(detail::token_stream& stream) +bool detail::skip_attribute(detail::cxtoken_stream& stream) { auto any = false; while (skip_attribute_impl(stream)) @@ -432,7 +433,7 @@ namespace } } -cpp_token_string detail::to_string(token_stream& stream, token_iterator end) +cpp_token_string detail::to_string(cxtoken_stream& stream, cxtoken_iterator end) { cpp_token_string::builder builder; @@ -448,7 +449,7 @@ cpp_token_string detail::to_string(token_stream& stream, token_iterator end) return builder.finish(); } -bool detail::append_scope(detail::token_stream& stream, std::string& scope) +bool detail::append_scope(detail::cxtoken_stream& stream, std::string& scope) { // add identifiers and "::" to current scope name, // clear if there is any other token in between, or mismatched combination diff --git a/src/libclang/tokenizer.hpp b/src/libclang/cxtokenizer.hpp similarity index 65% rename from src/libclang/tokenizer.hpp rename to src/libclang/cxtokenizer.hpp index 78ac31b..bc311a2 100644 --- a/src/libclang/tokenizer.hpp +++ b/src/libclang/cxtokenizer.hpp @@ -2,8 +2,8 @@ // This file is subject to the license terms in the LICENSE file // found in the top-level directory of this distribution. -#ifndef CPPAST_TOKENIZER_HPP_INCLUDED -#define CPPAST_TOKENIZER_HPP_INCLUDED +#ifndef CPPAST_CXTOKENIZER_HPP_INCLUDED +#define CPPAST_CXTOKENIZER_HPP_INCLUDED #include #include @@ -16,10 +16,10 @@ namespace cppast { namespace detail { - class token + class cxtoken { public: - explicit token(const CXTranslationUnit& tu_unit, const CXToken& token); + explicit cxtoken(const CXTranslationUnit& tu_unit, const CXToken& token); const cxstring& value() const noexcept { @@ -41,40 +41,40 @@ namespace cppast CXTokenKind kind_; }; - inline bool operator==(const token& tok, const char* str) noexcept + inline bool operator==(const cxtoken& tok, const char* str) noexcept { return tok.value() == str; } - inline bool operator==(const char* str, const token& tok) noexcept + inline bool operator==(const char* str, const cxtoken& tok) noexcept { return str == tok.value(); } - inline bool operator!=(const token& tok, const char* str) noexcept + inline bool operator!=(const cxtoken& tok, const char* str) noexcept { return !(tok == str); } - inline bool operator!=(const char* str, const token& tok) noexcept + inline bool operator!=(const char* str, const cxtoken& tok) noexcept { return !(str == tok); } - using token_iterator = std::vector::const_iterator; + using cxtoken_iterator = std::vector::const_iterator; - class tokenizer + class cxtokenizer { public: - explicit tokenizer(const CXTranslationUnit& tu, const CXFile& file, - const CXCursor& cur); + explicit cxtokenizer(const CXTranslationUnit& tu, const CXFile& file, + const CXCursor& cur); - token_iterator begin() const noexcept + cxtoken_iterator begin() const noexcept { return tokens_.begin(); } - token_iterator end() const noexcept + cxtoken_iterator end() const noexcept { return tokens_.end(); } @@ -88,14 +88,14 @@ namespace cppast } private: - std::vector tokens_; - bool unmunch_; + std::vector tokens_; + bool unmunch_; }; - class token_stream + class cxtoken_stream { public: - explicit token_stream(const tokenizer& tokenizer, const CXCursor& cur) + explicit cxtoken_stream(const cxtokenizer& tokenizer, const CXCursor& cur) : cursor_(cur), begin_(tokenizer.begin()), cur_(begin_), @@ -104,7 +104,7 @@ namespace cppast { } - const token& peek() const noexcept + const cxtoken& peek() const noexcept { if (done()) return *std::prev(end_); @@ -123,7 +123,7 @@ namespace cppast --cur_; } - const token& get() noexcept + const cxtoken& get() noexcept { auto& result = peek(); bump(); @@ -140,22 +140,22 @@ namespace cppast return cursor_; } - token_iterator begin() const noexcept + cxtoken_iterator begin() const noexcept { return begin_; } - token_iterator cur() const noexcept + cxtoken_iterator cur() const noexcept { return cur_; } - token_iterator end() const noexcept + cxtoken_iterator end() const noexcept { return end_; } - void set_cur(token_iterator iter) noexcept + void set_cur(cxtoken_iterator iter) noexcept { cur_ = iter; } @@ -166,41 +166,41 @@ namespace cppast } private: - CXCursor cursor_; - token_iterator begin_, cur_, end_; - bool unmunch_; + CXCursor cursor_; + cxtoken_iterator begin_, cur_, end_; + bool unmunch_; }; // skips the next token // asserts that it has the given string - void skip(token_stream& stream, const char* str); + void skip(cxtoken_stream& stream, const char* str); // skips the next token if it has the given string // if multi_token == true, str can consist of multiple tokens optionally separated by whitespace - bool skip_if(token_stream& stream, const char* str, bool multi_token = false); + bool skip_if(cxtoken_stream& stream, const char* str, bool multi_token = false); // returns the location of the closing bracket // the current token must be (,[,{ or < // note: < might not work in the arguments of a template specialization - token_iterator find_closing_bracket(token_stream stream); + cxtoken_iterator find_closing_bracket(cxtoken_stream stream); // skips brackets // the current token must be (,[,{ or < // note: < might not work in the arguments of a template specialization - void skip_brackets(token_stream& stream); + void skip_brackets(cxtoken_stream& stream); // skips an attribute - bool skip_attribute(token_stream& stream); + bool skip_attribute(cxtoken_stream& stream); // converts a token range to a string - cpp_token_string to_string(token_stream& stream, token_iterator end); + cpp_token_string to_string(cxtoken_stream& stream, cxtoken_iterator end); // appends token to scope, if it is still valid // else clears it // note: does not consume the token if it is not valid, // returns false in that case - bool append_scope(token_stream& stream, std::string& scope); + bool append_scope(cxtoken_stream& stream, std::string& scope); } } // namespace cppast::detail -#endif // CPPAST_TOKENIZER_HPP_INCLUDED +#endif // CPPAST_CXTOKENIZER_HPP_INCLUDED diff --git a/src/libclang/debug_helper.cpp b/src/libclang/debug_helper.cpp index a3ab29a..ba4d38d 100644 --- a/src/libclang/debug_helper.cpp +++ b/src/libclang/debug_helper.cpp @@ -7,7 +7,7 @@ #include #include -#include "tokenizer.hpp" +#include "cxtokenizer.hpp" using namespace cppast; @@ -50,7 +50,7 @@ void detail::print_tokens(const CXTranslationUnit& tu, const CXFile& file, const CXCursor& cur) noexcept { std::lock_guard lock(mtx); - detail::tokenizer tokenizer(tu, file, cur); + detail::cxtokenizer tokenizer(tu, file, cur); for (auto& token : tokenizer) std::fprintf(stderr, "%s ", token.c_str()); std::fputs("\n", stderr); diff --git a/src/libclang/enum_parser.cpp b/src/libclang/enum_parser.cpp index 95eb886..9e7431f 100644 --- a/src/libclang/enum_parser.cpp +++ b/src/libclang/enum_parser.cpp @@ -20,8 +20,8 @@ namespace DEBUG_ASSERT(cur.kind == CXCursor_EnumConstantDecl, detail::parse_error_handler{}, cur, "unexpected child cursor of enum"); - detail::tokenizer tokenizer(context.tu, context.file, cur); - detail::token_stream stream(tokenizer, cur); + detail::cxtokenizer tokenizer(context.tu, context.file, cur); + detail::cxtoken_stream stream(tokenizer, cur); // [], // or: [] = , @@ -47,9 +47,9 @@ namespace cpp_enum::builder make_enum_builder(const detail::parse_context& context, const CXCursor& cur, type_safe::optional& semantic_parent) { - auto name = detail::get_cursor_name(cur); - detail::tokenizer tokenizer(context.tu, context.file, cur); - detail::token_stream stream(tokenizer, cur); + auto name = detail::get_cursor_name(cur); + detail::cxtokenizer tokenizer(context.tu, context.file, cur); + detail::cxtoken_stream stream(tokenizer, cur); // [] enum [class] [] name [: type] { detail::skip_attribute(stream); diff --git a/src/libclang/expression_parser.cpp b/src/libclang/expression_parser.cpp index 9950675..43079e0 100644 --- a/src/libclang/expression_parser.cpp +++ b/src/libclang/expression_parser.cpp @@ -14,8 +14,8 @@ std::unique_ptr detail::parse_expression(const detail::parse_con auto kind = clang_getCursorKind(cur); DEBUG_ASSERT(clang_isExpression(kind), detail::assert_handler{}); - detail::tokenizer tokenizer(context.tu, context.file, cur); - detail::token_stream stream(tokenizer, cur); + detail::cxtokenizer tokenizer(context.tu, context.file, cur); + detail::cxtoken_stream stream(tokenizer, cur); auto type = parse_type(context, cur, clang_getCursorType(cur)); auto expr = to_string(stream, stream.end()); @@ -36,8 +36,8 @@ std::unique_ptr detail::parse_expression(const detail::parse_con } std::unique_ptr detail::parse_raw_expression(const parse_context&, - token_stream& stream, - token_iterator end, + cxtoken_stream& stream, + cxtoken_iterator end, std::unique_ptr type) { if (stream.done()) diff --git a/src/libclang/function_parser.cpp b/src/libclang/function_parser.cpp index f13974d..5d20e9b 100644 --- a/src/libclang/function_parser.cpp +++ b/src/libclang/function_parser.cpp @@ -92,7 +92,7 @@ namespace } // precondition: after the name - void skip_parameters(detail::token_stream& stream) + void skip_parameters(detail::cxtoken_stream& stream) { if (stream.peek() == "<") // specialization arguments @@ -218,7 +218,7 @@ namespace bool is_friend = false; }; - bool prefix_end(detail::token_stream& stream, const char* name, bool is_ctor) + bool prefix_end(detail::cxtoken_stream& stream, const char* name, bool is_ctor) { auto cur = stream.cur(); // name can have multiple tokens if it is an operator @@ -262,7 +262,7 @@ namespace return true; } - prefix_info parse_prefix_info(detail::token_stream& stream, const char* name, bool is_ctor) + prefix_info parse_prefix_info(detail::cxtoken_stream& stream, const char* name, bool is_ctor) { prefix_info result; @@ -302,7 +302,7 @@ namespace } }; - cpp_cv parse_cv(detail::token_stream& stream) + cpp_cv parse_cv(detail::cxtoken_stream& stream) { if (detail::skip_if(stream, "const")) { @@ -322,7 +322,7 @@ namespace return cpp_cv_none; } - cpp_reference parse_ref(detail::token_stream& stream) + cpp_reference parse_ref(detail::cxtoken_stream& stream) { if (detail::skip_if(stream, "&")) return cpp_ref_lvalue; @@ -332,7 +332,7 @@ namespace return cpp_ref_none; } - std::unique_ptr parse_noexcept(detail::token_stream& stream, + std::unique_ptr parse_noexcept(detail::cxtoken_stream& stream, const detail::parse_context& context) { if (!detail::skip_if(stream, "noexcept")) @@ -351,7 +351,7 @@ namespace return expr; } - cpp_function_body_kind parse_body_kind(detail::token_stream& stream, bool& pure_virtual) + cpp_function_body_kind parse_body_kind(detail::cxtoken_stream& stream, bool& pure_virtual) { pure_virtual = false; if (detail::skip_if(stream, "default")) @@ -369,7 +369,7 @@ namespace return cpp_function_declaration; } - void parse_body(detail::token_stream& stream, suffix_info& result, bool allow_virtual) + void parse_body(detail::cxtoken_stream& stream, suffix_info& result, bool allow_virtual) { auto pure_virtual = false; result.body_kind = parse_body_kind(stream, pure_virtual); @@ -385,7 +385,7 @@ namespace } // precondition: we've skipped the function parameters - suffix_info parse_suffix_info(detail::token_stream& stream, + suffix_info parse_suffix_info(detail::cxtoken_stream& stream, const detail::parse_context& context, bool allow_qualifier, bool allow_virtual) { @@ -488,8 +488,8 @@ namespace { auto name = detail::get_cursor_name(cur); - detail::tokenizer tokenizer(context.tu, context.file, cur); - detail::token_stream stream(tokenizer, cur); + detail::cxtokenizer tokenizer(context.tu, context.file, cur); + detail::cxtoken_stream stream(tokenizer, cur); auto prefix = parse_prefix_info(stream, name.c_str(), false); DEBUG_ASSERT(!prefix.is_virtual && !prefix.is_explicit, detail::parse_error_handler{}, cur, @@ -611,7 +611,7 @@ namespace template std::unique_ptr handle_suffix(const detail::parse_context& context, const CXCursor& cur, Builder& builder, - detail::token_stream& stream, bool is_virtual, + detail::cxtoken_stream& stream, bool is_virtual, type_safe::optional semantic_parent) { auto allow_qualifiers = set_qualifier(0, builder, cpp_cv_none, cpp_ref_none); @@ -640,8 +640,8 @@ std::unique_ptr detail::parse_cpp_member_function(const detail::pars detail::assert_handler{}); auto name = detail::get_cursor_name(cur); - detail::tokenizer tokenizer(context.tu, context.file, cur); - detail::token_stream stream(tokenizer, cur); + detail::cxtokenizer tokenizer(context.tu, context.file, cur); + detail::cxtoken_stream stream(tokenizer, cur); auto prefix = parse_prefix_info(stream, name.c_str(), false); DEBUG_ASSERT(!prefix.is_explicit, detail::parse_error_handler{}, cur, @@ -670,8 +670,8 @@ std::unique_ptr detail::parse_cpp_conversion_op(const detail::parse_ || clang_getTemplateCursorKind(cur) == CXCursor_ConversionFunction, detail::assert_handler{}); - detail::tokenizer tokenizer(context.tu, context.file, cur); - detail::token_stream stream(tokenizer, cur); + detail::cxtokenizer tokenizer(context.tu, context.file, cur); + detail::cxtoken_stream stream(tokenizer, cur); auto prefix = parse_prefix_info(stream, "operator", false); // heuristic to find arguments tokens @@ -735,8 +735,8 @@ std::unique_ptr detail::parse_cpp_constructor(const detail::parse_co if (pos != std::string::npos) name.erase(pos); - detail::tokenizer tokenizer(context.tu, context.file, cur); - detail::token_stream stream(tokenizer, cur); + detail::cxtokenizer tokenizer(context.tu, context.file, cur); + detail::cxtoken_stream stream(tokenizer, cur); auto prefix = parse_prefix_info(stream, name.c_str(), true); DEBUG_ASSERT(!prefix.is_virtual, detail::parse_error_handler{}, cur, @@ -771,8 +771,8 @@ std::unique_ptr detail::parse_cpp_destructor(const detail::parse_con { DEBUG_ASSERT(clang_getCursorKind(cur) == CXCursor_Destructor, detail::assert_handler{}); - detail::tokenizer tokenizer(context.tu, context.file, cur); - detail::token_stream stream(tokenizer, cur); + detail::cxtokenizer tokenizer(context.tu, context.file, cur); + detail::cxtoken_stream stream(tokenizer, cur); auto prefix_info = parse_prefix_info(stream, "~", false); DEBUG_ASSERT(!prefix_info.is_constexpr && !prefix_info.is_explicit, detail::assert_handler{}); diff --git a/src/libclang/language_linkage_parser.cpp b/src/libclang/language_linkage_parser.cpp index d79f8cb..0477f21 100644 --- a/src/libclang/language_linkage_parser.cpp +++ b/src/libclang/language_linkage_parser.cpp @@ -17,8 +17,8 @@ std::unique_ptr detail::try_parse_cpp_language_linkage(const parse_c DEBUG_ASSERT(cur.kind == CXCursor_UnexposedDecl, detail::assert_handler{}); // not exposed currently - detail::tokenizer tokenizer(context.tu, context.file, cur); - detail::token_stream stream(tokenizer, cur); + detail::cxtokenizer tokenizer(context.tu, context.file, cur); + detail::cxtoken_stream stream(tokenizer, cur); // extern ... if (!detail::skip_if(stream, "extern")) diff --git a/src/libclang/libclang_parser.cpp b/src/libclang/libclang_parser.cpp index 12b602a..65ab58c 100644 --- a/src/libclang/libclang_parser.cpp +++ b/src/libclang/libclang_parser.cpp @@ -15,7 +15,7 @@ #include "parse_error.hpp" #include "parse_functions.hpp" #include "preprocessor.hpp" -#include "tokenizer.hpp" +#include "cxtokenizer.hpp" using namespace cppast; diff --git a/src/libclang/namespace_parser.cpp b/src/libclang/namespace_parser.cpp index f3b8797..1391565 100644 --- a/src/libclang/namespace_parser.cpp +++ b/src/libclang/namespace_parser.cpp @@ -16,8 +16,8 @@ namespace cpp_namespace::builder make_ns_builder(const detail::parse_context& context, const CXCursor& cur) { - detail::tokenizer tokenizer(context.tu, context.file, cur); - detail::token_stream stream(tokenizer, cur); + detail::cxtokenizer tokenizer(context.tu, context.file, cur); + detail::cxtoken_stream stream(tokenizer, cur); // [inline] namespace [] { auto is_inline = false; @@ -83,8 +83,8 @@ std::unique_ptr detail::parse_cpp_namespace_alias(const detail::pars { DEBUG_ASSERT(cur.kind == CXCursor_NamespaceAlias, detail::assert_handler{}); - detail::tokenizer tokenizer(context.tu, context.file, cur); - detail::token_stream stream(tokenizer, cur); + detail::cxtokenizer tokenizer(context.tu, context.file, cur); + detail::cxtoken_stream stream(tokenizer, cur); // namespace = ; detail::skip(stream, "namespace"); @@ -108,8 +108,8 @@ std::unique_ptr detail::parse_cpp_using_directive(const detail::pars { DEBUG_ASSERT(cur.kind == CXCursor_UsingDirective, detail::assert_handler{}); - detail::tokenizer tokenizer(context.tu, context.file, cur); - detail::token_stream stream(tokenizer, cur); + detail::cxtokenizer tokenizer(context.tu, context.file, cur); + detail::cxtoken_stream stream(tokenizer, cur); // using namespace ; detail::skip(stream, "using"); @@ -182,8 +182,8 @@ std::unique_ptr detail::parse_cpp_using_declaration( { DEBUG_ASSERT(cur.kind == CXCursor_UsingDeclaration, detail::assert_handler{}); - detail::tokenizer tokenizer(context.tu, context.file, cur); - detail::token_stream stream(tokenizer, cur); + detail::cxtokenizer tokenizer(context.tu, context.file, cur); + detail::cxtoken_stream stream(tokenizer, cur); // using ; detail::skip(stream, "using"); diff --git a/src/libclang/parse_functions.cpp b/src/libclang/parse_functions.cpp index 6b4b471..0b907a2 100644 --- a/src/libclang/parse_functions.cpp +++ b/src/libclang/parse_functions.cpp @@ -215,10 +215,10 @@ std::unique_ptr detail::parse_entity(const detail::parse_context& co detail::get_cursor_kind_spelling(cur).c_str(), "'")); // build unexposed entity - auto name = detail::get_cursor_name(cur); - detail::tokenizer tokenizer(context.tu, context.file, cur); - detail::token_stream stream(tokenizer, cur); - auto spelling = detail::to_string(stream, stream.end()); + auto name = detail::get_cursor_name(cur); + detail::cxtokenizer tokenizer(context.tu, context.file, cur); + detail::cxtoken_stream stream(tokenizer, cur); + auto spelling = detail::to_string(stream, stream.end()); std::unique_ptr entity; if (name.empty()) diff --git a/src/libclang/parse_functions.hpp b/src/libclang/parse_functions.hpp index 7ef94a1..b98d640 100644 --- a/src/libclang/parse_functions.hpp +++ b/src/libclang/parse_functions.hpp @@ -9,7 +9,7 @@ #include #include "raii_wrapper.hpp" -#include "tokenizer.hpp" // for convenience +#include "cxtokenizer.hpp" // for convenience #include "parse_error.hpp" // for convenience #include "preprocessor.hpp" @@ -76,8 +76,8 @@ namespace cppast // and ends at the given iterator // this is required for situations where there is no type exposed, // like default type of a template type parameter - std::unique_ptr parse_raw_type(const parse_context& context, token_stream& stream, - token_iterator end); + std::unique_ptr parse_raw_type(const parse_context& context, + cxtoken_stream& stream, cxtoken_iterator end); std::unique_ptr parse_expression(const parse_context& context, const CXCursor& cur); @@ -86,8 +86,8 @@ namespace cppast // this is required for situations where there is no expression cursor exposed, // like member initializers std::unique_ptr parse_raw_expression(const parse_context& context, - token_stream& stream, - token_iterator end, + cxtoken_stream& stream, + cxtoken_iterator end, std::unique_ptr type); // parse_entity() dispatches on the cursor type diff --git a/src/libclang/template_parser.cpp b/src/libclang/template_parser.cpp index b16bcee..de4ecef 100644 --- a/src/libclang/template_parser.cpp +++ b/src/libclang/template_parser.cpp @@ -50,9 +50,9 @@ namespace DEBUG_ASSERT(clang_getCursorKind(cur) == CXCursor_TemplateTypeParameter, detail::assert_handler{}); - detail::tokenizer tokenizer(context.tu, context.file, cur); - detail::token_stream stream(tokenizer, cur); - auto name = detail::get_cursor_name(cur); + detail::cxtokenizer tokenizer(context.tu, context.file, cur); + detail::cxtoken_stream stream(tokenizer, cur); + auto name = detail::get_cursor_name(cur); // syntax: typename/class [...] name [= ...] auto keyword = cpp_template_keyword::keyword_class; @@ -87,8 +87,8 @@ namespace auto type = clang_getCursorType(cur); auto def = detail::parse_default_value(context, cur, name.c_str()); - detail::tokenizer tokenizer(context.tu, context.file, cur); - detail::token_stream stream(tokenizer, cur); + detail::cxtokenizer tokenizer(context.tu, context.file, cur); + detail::cxtoken_stream stream(tokenizer, cur); // see if it is variadic // syntax a): some-tokens ... name some-tokens @@ -120,9 +120,9 @@ namespace DEBUG_ASSERT(clang_getCursorKind(cur) == CXCursor_TemplateTemplateParameter, detail::assert_handler{}); - detail::tokenizer tokenizer(context.tu, context.file, cur); - detail::token_stream stream(tokenizer, cur); - auto name = detail::get_cursor_name(cur); + detail::cxtokenizer tokenizer(context.tu, context.file, cur); + detail::cxtoken_stream stream(tokenizer, cur); + auto name = detail::get_cursor_name(cur); // syntax: template <…> class/typename [...] name [= …] detail::skip(stream, "template"); @@ -263,8 +263,8 @@ namespace template void parse_arguments(Builder& b, const detail::parse_context& context, const CXCursor& cur) { - detail::tokenizer tokenizer(context.tu, context.file, cur); - detail::token_stream stream(tokenizer, cur); + detail::cxtokenizer tokenizer(context.tu, context.file, cur); + detail::cxtoken_stream stream(tokenizer, cur); while (!stream.done() && !detail::skip_if(stream, detail::get_cursor_name(cur).c_str(), true)) diff --git a/src/libclang/type_parser.cpp b/src/libclang/type_parser.cpp index cdd841a..c850e8a 100644 --- a/src/libclang/type_parser.cpp +++ b/src/libclang/type_parser.cpp @@ -731,8 +731,8 @@ std::unique_ptr detail::parse_type(const detail::parse_context& contex } std::unique_ptr detail::parse_raw_type(const detail::parse_context&, - detail::token_stream& stream, - detail::token_iterator end) + detail::cxtoken_stream& stream, + detail::cxtoken_iterator end) { auto result = detail::to_string(stream, end); return cpp_unexposed_type::build(result.as_string()); diff --git a/src/libclang/variable_parser.cpp b/src/libclang/variable_parser.cpp index d4a65bf..f49e240 100644 --- a/src/libclang/variable_parser.cpp +++ b/src/libclang/variable_parser.cpp @@ -14,8 +14,8 @@ using namespace cppast; std::unique_ptr detail::parse_default_value(const detail::parse_context& context, const CXCursor& cur, const char* name) { - detail::tokenizer tokenizer(context.tu, context.file, cur); - detail::token_stream stream(tokenizer, cur); + detail::cxtokenizer tokenizer(context.tu, context.file, cur); + detail::cxtoken_stream stream(tokenizer, cur); auto has_default = false; auto got_name = *name == '\0'; @@ -57,7 +57,7 @@ std::unique_ptr detail::parse_cpp_variable(const detail::parse_conte // just look for thread local or constexpr // can't appear anywhere else, so good enough - detail::tokenizer tokenizer(context.tu, context.file, cur); + detail::cxtokenizer tokenizer(context.tu, context.file, cur); for (auto& token : tokenizer) if (token.value() == "thread_local") storage_class =