jyapayne/cppast
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Rename tokenizer stuff to include cx prefix

Browse source
This commit is contained in:
Jonathan Müller 2017-10-12 18:43:12 +02:00
commit cfac41c7b5
16 changed files with 121 additions and 120 deletions
Download patch file Download diff file Expand all files Collapse all files

4
src/CMakeLists.txt
View file

@ -95,8 +95,8 @@ set(libclang_source
libclang/preprocessor.hpp libclang/preprocessor.hpp
libclang/raii_wrapper.hpp libclang/raii_wrapper.hpp
libclang/template_parser.cpp libclang/template_parser.cpp
libclang/tokenizer.cpp libclang/cxtokenizer.cpp
libclang/tokenizer.hpp libclang/cxtokenizer.hpp
libclang/type_parser.cpp libclang/type_parser.cpp
libclang/variable_parser.cpp) libclang/variable_parser.cpp)

8
src/libclang/class_parser.cpp
View file

@ -71,8 +71,8 @@ namespace
auto access = convert_access(cur); auto access = convert_access(cur);
auto is_virtual = clang_isVirtualBase(cur) != 0u; auto is_virtual = clang_isVirtualBase(cur) != 0u;
detail::tokenizer tokenizer(context.tu, context.file, cur); detail::cxtokenizer tokenizer(context.tu, context.file, cur);
detail::token_stream stream(tokenizer, cur); detail::cxtoken_stream stream(tokenizer, cur);
// [<attribute>] [virtual] [<access>] <name> // [<attribute>] [virtual] [<access>] <name>
// can't use spelling to get the name // can't use spelling to get the name
@ -108,8 +108,8 @@ std::unique_ptr<cpp_entity> detail::parse_cpp_class(const detail::parse_context&
clang_getCursorLexicalParent(cur))) clang_getCursorLexicalParent(cur)))
{ {
// out-of-line definition // out-of-line definition
detail::tokenizer tokenizer(context.tu, context.file, cur); detail::cxtokenizer tokenizer(context.tu, context.file, cur);
detail::token_stream stream(tokenizer, cur); detail::cxtoken_stream stream(tokenizer, cur);
std::string name = detail::get_cursor_name(cur).c_str(); std::string name = detail::get_cursor_name(cur).c_str();
auto pos = name.find('<'); auto pos = name.find('<');

27
src/libclang/tokenizer.cpp → src/libclang/cxtokenizer.cpp
View file

@ -2,7 +2,7 @@
// This file is subject to the license terms in the LICENSE file // This file is subject to the license terms in the LICENSE file
// found in the top-level directory of this distribution. // found in the top-level directory of this distribution.
#include "tokenizer.hpp" #include "cxtokenizer.hpp"
#include <cctype> #include <cctype>
@ -11,7 +11,7 @@
using namespace cppast; using namespace cppast;
detail::token::token(const CXTranslationUnit& tu_unit, const CXToken& token) detail::cxtoken::cxtoken(const CXTranslationUnit& tu_unit, const CXToken& token)
: value_(clang_getTokenSpelling(tu_unit, token)), kind_(clang_getTokenKind(token)) : value_(clang_getTokenSpelling(tu_unit, token)), kind_(clang_getTokenKind(token))
{ {
} }
@ -237,7 +237,8 @@ namespace
} }
} }
detail::tokenizer::tokenizer(const CXTranslationUnit& tu, const CXFile& file, const CXCursor& cur) detail::cxtokenizer::cxtokenizer(const CXTranslationUnit& tu, const CXFile& file,
const CXCursor& cur)
{ {
auto extent = get_extent(tu, file, cur, unmunch_); auto extent = get_extent(tu, file, cur, unmunch_);
@ -247,7 +248,7 @@ detail::tokenizer::tokenizer(const CXTranslationUnit& tu, const CXFile& file, co
tokens_.emplace_back(tu, tokenizer[i]); tokens_.emplace_back(tu, tokenizer[i]);
} }
void detail::skip(detail::token_stream& stream, const char* str) void detail::skip(detail::cxtoken_stream& stream, const char* str)
{ {
if (*str) if (*str)
{ {
@ -263,7 +264,7 @@ void detail::skip(detail::token_stream& stream, const char* str)
namespace namespace
{ {
bool starts_with(const char*& str, const detail::token& t) bool starts_with(const char*& str, const detail::cxtoken& t)
{ {
if (std::strncmp(str, t.c_str(), t.value().length()) != 0) if (std::strncmp(str, t.c_str(), t.value().length()) != 0)
return false; return false;
@ -274,7 +275,7 @@ namespace
} }
} }
bool detail::skip_if(detail::token_stream& stream, const char* str, bool multi_token) bool detail::skip_if(detail::cxtoken_stream& stream, const char* str, bool multi_token)
{ {
if (!*str) if (!*str)
return true; return true;
@ -298,7 +299,7 @@ namespace
{ {
// whether or not the current angle bracket can be a comparison // whether or not the current angle bracket can be a comparison
// note: this is a heuristic I hope works often enough // note: this is a heuristic I hope works often enough
bool is_comparison(CXTokenKind last_kind, const detail::token& cur, CXTokenKind next_kind) bool is_comparison(CXTokenKind last_kind, const detail::cxtoken& cur, CXTokenKind next_kind)
{ {
if (cur == "<") if (cur == "<")
return last_kind == CXToken_Literal; return last_kind == CXToken_Literal;
@ -308,7 +309,7 @@ namespace
} }
} }
detail::token_iterator detail::find_closing_bracket(detail::token_stream stream) detail::cxtoken_iterator detail::find_closing_bracket(detail::cxtoken_stream stream)
{ {
auto template_bracket = false; auto template_bracket = false;
auto open_bracket = stream.peek().c_str(); auto open_bracket = stream.peek().c_str();
@ -359,7 +360,7 @@ detail::token_iterator detail::find_closing_bracket(detail::token_stream stream)
return stream.cur(); return stream.cur();
} }
void detail::skip_brackets(detail::token_stream& stream) void detail::skip_brackets(detail::cxtoken_stream& stream)
{ {
auto closing = find_closing_bracket(stream); auto closing = find_closing_bracket(stream);
stream.set_cur(std::next(closing)); stream.set_cur(std::next(closing));
@ -367,7 +368,7 @@ void detail::skip_brackets(detail::token_stream& stream)
namespace namespace
{ {
bool skip_attribute_impl(detail::token_stream& stream) bool skip_attribute_impl(detail::cxtoken_stream& stream)
{ {
if (skip_if(stream, "[") && stream.peek() == "[") if (skip_if(stream, "[") && stream.peek() == "[")
{ {
@ -401,7 +402,7 @@ namespace
} }
} }
bool detail::skip_attribute(detail::token_stream& stream) bool detail::skip_attribute(detail::cxtoken_stream& stream)
{ {
auto any = false; auto any = false;
while (skip_attribute_impl(stream)) while (skip_attribute_impl(stream))
@ -432,7 +433,7 @@ namespace
} }
} }
cpp_token_string detail::to_string(token_stream& stream, token_iterator end) cpp_token_string detail::to_string(cxtoken_stream& stream, cxtoken_iterator end)
{ {
cpp_token_string::builder builder; cpp_token_string::builder builder;
@ -448,7 +449,7 @@ cpp_token_string detail::to_string(token_stream& stream, token_iterator end)
return builder.finish(); return builder.finish();
} }
bool detail::append_scope(detail::token_stream& stream, std::string& scope) bool detail::append_scope(detail::cxtoken_stream& stream, std::string& scope)
{ {
// add identifiers and "::" to current scope name, // add identifiers and "::" to current scope name,
// clear if there is any other token in between, or mismatched combination // clear if there is any other token in between, or mismatched combination

70
src/libclang/tokenizer.hpp → src/libclang/cxtokenizer.hpp
View file

@ -2,8 +2,8 @@
// This file is subject to the license terms in the LICENSE file // This file is subject to the license terms in the LICENSE file
// found in the top-level directory of this distribution. // found in the top-level directory of this distribution.
#ifndef CPPAST_TOKENIZER_HPP_INCLUDED #ifndef CPPAST_CXTOKENIZER_HPP_INCLUDED
#define CPPAST_TOKENIZER_HPP_INCLUDED #define CPPAST_CXTOKENIZER_HPP_INCLUDED
#include <string> #include <string>
#include <vector> #include <vector>
@ -16,10 +16,10 @@ namespace cppast
{ {
namespace detail namespace detail
{ {
class token class cxtoken
{ {
public: public:
explicit token(const CXTranslationUnit& tu_unit, const CXToken& token); explicit cxtoken(const CXTranslationUnit& tu_unit, const CXToken& token);
const cxstring& value() const noexcept const cxstring& value() const noexcept
{ {
@ -41,40 +41,40 @@ namespace cppast
CXTokenKind kind_; CXTokenKind kind_;
}; };
inline bool operator==(const token& tok, const char* str) noexcept inline bool operator==(const cxtoken& tok, const char* str) noexcept
{ {
return tok.value() == str; return tok.value() == str;
} }
inline bool operator==(const char* str, const token& tok) noexcept inline bool operator==(const char* str, const cxtoken& tok) noexcept
{ {
return str == tok.value(); return str == tok.value();
} }
inline bool operator!=(const token& tok, const char* str) noexcept inline bool operator!=(const cxtoken& tok, const char* str) noexcept
{ {
return !(tok == str); return !(tok == str);
} }
inline bool operator!=(const char* str, const token& tok) noexcept inline bool operator!=(const char* str, const cxtoken& tok) noexcept
{ {
return !(str == tok); return !(str == tok);
} }
using token_iterator = std::vector<token>::const_iterator; using cxtoken_iterator = std::vector<cxtoken>::const_iterator;
class tokenizer class cxtokenizer
{ {
public: public:
explicit tokenizer(const CXTranslationUnit& tu, const CXFile& file, explicit cxtokenizer(const CXTranslationUnit& tu, const CXFile& file,
const CXCursor& cur); const CXCursor& cur);
token_iterator begin() const noexcept cxtoken_iterator begin() const noexcept
{ {
return tokens_.begin(); return tokens_.begin();
} }
token_iterator end() const noexcept cxtoken_iterator end() const noexcept
{ {
return tokens_.end(); return tokens_.end();
} }
@ -88,14 +88,14 @@ namespace cppast
} }
private: private:
std::vector<token> tokens_; std::vector<cxtoken> tokens_;
bool unmunch_; bool unmunch_;
}; };
class token_stream class cxtoken_stream
{ {
public: public:
explicit token_stream(const tokenizer& tokenizer, const CXCursor& cur) explicit cxtoken_stream(const cxtokenizer& tokenizer, const CXCursor& cur)
: cursor_(cur), : cursor_(cur),
begin_(tokenizer.begin()), begin_(tokenizer.begin()),
cur_(begin_), cur_(begin_),
@ -104,7 +104,7 @@ namespace cppast
{ {
} }
const token& peek() const noexcept const cxtoken& peek() const noexcept
{ {
if (done()) if (done())
return *std::prev(end_); return *std::prev(end_);
@ -123,7 +123,7 @@ namespace cppast
--cur_; --cur_;
} }
const token& get() noexcept const cxtoken& get() noexcept
{ {
auto& result = peek(); auto& result = peek();
bump(); bump();
@ -140,22 +140,22 @@ namespace cppast
return cursor_; return cursor_;
} }
token_iterator begin() const noexcept cxtoken_iterator begin() const noexcept
{ {
return begin_; return begin_;
} }
token_iterator cur() const noexcept cxtoken_iterator cur() const noexcept
{ {
return cur_; return cur_;
} }
token_iterator end() const noexcept cxtoken_iterator end() const noexcept
{ {
return end_; return end_;
} }
void set_cur(token_iterator iter) noexcept void set_cur(cxtoken_iterator iter) noexcept
{ {
cur_ = iter; cur_ = iter;
} }
@ -166,41 +166,41 @@ namespace cppast
} }
private: private:
CXCursor cursor_; CXCursor cursor_;
token_iterator begin_, cur_, end_; cxtoken_iterator begin_, cur_, end_;
bool unmunch_; bool unmunch_;
}; };
// skips the next token // skips the next token
// asserts that it has the given string // asserts that it has the given string
void skip(token_stream& stream, const char* str); void skip(cxtoken_stream& stream, const char* str);
// skips the next token if it has the given string // skips the next token if it has the given string
// if multi_token == true, str can consist of multiple tokens optionally separated by whitespace // if multi_token == true, str can consist of multiple tokens optionally separated by whitespace
bool skip_if(token_stream& stream, const char* str, bool multi_token = false); bool skip_if(cxtoken_stream& stream, const char* str, bool multi_token = false);
// returns the location of the closing bracket // returns the location of the closing bracket
// the current token must be (,[,{ or < // the current token must be (,[,{ or <
// note: < might not work in the arguments of a template specialization // note: < might not work in the arguments of a template specialization
token_iterator find_closing_bracket(token_stream stream); cxtoken_iterator find_closing_bracket(cxtoken_stream stream);
// skips brackets // skips brackets
// the current token must be (,[,{ or < // the current token must be (,[,{ or <
// note: < might not work in the arguments of a template specialization // note: < might not work in the arguments of a template specialization
void skip_brackets(token_stream& stream); void skip_brackets(cxtoken_stream& stream);
// skips an attribute // skips an attribute
bool skip_attribute(token_stream& stream); bool skip_attribute(cxtoken_stream& stream);
// converts a token range to a string // converts a token range to a string
cpp_token_string to_string(token_stream& stream, token_iterator end); cpp_token_string to_string(cxtoken_stream& stream, cxtoken_iterator end);
// appends token to scope, if it is still valid // appends token to scope, if it is still valid
// else clears it // else clears it
// note: does not consume the token if it is not valid, // note: does not consume the token if it is not valid,
// returns false in that case // returns false in that case
bool append_scope(token_stream& stream, std::string& scope); bool append_scope(cxtoken_stream& stream, std::string& scope);
} }
} // namespace cppast::detail } // namespace cppast::detail
#endif // CPPAST_TOKENIZER_HPP_INCLUDED #endif // CPPAST_CXTOKENIZER_HPP_INCLUDED

4
src/libclang/debug_helper.cpp
View file

@ -7,7 +7,7 @@
#include <cstdio> #include <cstdio>
#include <mutex> #include <mutex>
#include "tokenizer.hpp" #include "cxtokenizer.hpp"
using namespace cppast; using namespace cppast;
@ -50,7 +50,7 @@ void detail::print_tokens(const CXTranslationUnit& tu, const CXFile& file,
const CXCursor& cur) noexcept const CXCursor& cur) noexcept
{ {
std::lock_guard<std::mutex> lock(mtx); std::lock_guard<std::mutex> lock(mtx);
detail::tokenizer tokenizer(tu, file, cur); detail::cxtokenizer tokenizer(tu, file, cur);
for (auto& token : tokenizer) for (auto& token : tokenizer)
std::fprintf(stderr, "%s ", token.c_str()); std::fprintf(stderr, "%s ", token.c_str());
std::fputs("\n", stderr); std::fputs("\n", stderr);

10
src/libclang/enum_parser.cpp
View file

@ -20,8 +20,8 @@ namespace
DEBUG_ASSERT(cur.kind == CXCursor_EnumConstantDecl, detail::parse_error_handler{}, cur, DEBUG_ASSERT(cur.kind == CXCursor_EnumConstantDecl, detail::parse_error_handler{}, cur,
"unexpected child cursor of enum"); "unexpected child cursor of enum");
detail::tokenizer tokenizer(context.tu, context.file, cur); detail::cxtokenizer tokenizer(context.tu, context.file, cur);
detail::token_stream stream(tokenizer, cur); detail::cxtoken_stream stream(tokenizer, cur);
// <identifier> [<attribute>], // <identifier> [<attribute>],
// or: <identifier> [<attribute>] = <expression>, // or: <identifier> [<attribute>] = <expression>,
@ -47,9 +47,9 @@ namespace
cpp_enum::builder make_enum_builder(const detail::parse_context& context, const CXCursor& cur, cpp_enum::builder make_enum_builder(const detail::parse_context& context, const CXCursor& cur,
type_safe::optional<cpp_entity_ref>& semantic_parent) type_safe::optional<cpp_entity_ref>& semantic_parent)
{ {
auto name = detail::get_cursor_name(cur); auto name = detail::get_cursor_name(cur);
detail::tokenizer tokenizer(context.tu, context.file, cur); detail::cxtokenizer tokenizer(context.tu, context.file, cur);
detail::token_stream stream(tokenizer, cur); detail::cxtoken_stream stream(tokenizer, cur);
// [<attribute>] enum [class] [<attribute>] name [: type] { // [<attribute>] enum [class] [<attribute>] name [: type] {
detail::skip_attribute(stream); detail::skip_attribute(stream);

8
src/libclang/expression_parser.cpp
View file

@ -14,8 +14,8 @@ std::unique_ptr<cpp_expression> detail::parse_expression(const detail::parse_con
auto kind = clang_getCursorKind(cur); auto kind = clang_getCursorKind(cur);
DEBUG_ASSERT(clang_isExpression(kind), detail::assert_handler{}); DEBUG_ASSERT(clang_isExpression(kind), detail::assert_handler{});
detail::tokenizer tokenizer(context.tu, context.file, cur); detail::cxtokenizer tokenizer(context.tu, context.file, cur);
detail::token_stream stream(tokenizer, cur); detail::cxtoken_stream stream(tokenizer, cur);
auto type = parse_type(context, cur, clang_getCursorType(cur)); auto type = parse_type(context, cur, clang_getCursorType(cur));
auto expr = to_string(stream, stream.end()); auto expr = to_string(stream, stream.end());
@ -36,8 +36,8 @@ std::unique_ptr<cpp_expression> detail::parse_expression(const detail::parse_con
} }
std::unique_ptr<cpp_expression> detail::parse_raw_expression(const parse_context&, std::unique_ptr<cpp_expression> detail::parse_raw_expression(const parse_context&,
token_stream& stream, cxtoken_stream& stream,
token_iterator end, cxtoken_iterator end,
std::unique_ptr<cpp_type> type) std::unique_ptr<cpp_type> type)
{ {
if (stream.done()) if (stream.done())

40
src/libclang/function_parser.cpp
View file

@ -92,7 +92,7 @@ namespace
} }
// precondition: after the name // precondition: after the name
void skip_parameters(detail::token_stream& stream) void skip_parameters(detail::cxtoken_stream& stream)
{ {
if (stream.peek() == "<") if (stream.peek() == "<")
// specialization arguments // specialization arguments
@ -218,7 +218,7 @@ namespace
bool is_friend = false; bool is_friend = false;
}; };
bool prefix_end(detail::token_stream& stream, const char* name, bool is_ctor) bool prefix_end(detail::cxtoken_stream& stream, const char* name, bool is_ctor)
{ {
auto cur = stream.cur(); auto cur = stream.cur();
// name can have multiple tokens if it is an operator // name can have multiple tokens if it is an operator
@ -262,7 +262,7 @@ namespace
return true; return true;
} }
prefix_info parse_prefix_info(detail::token_stream& stream, const char* name, bool is_ctor) prefix_info parse_prefix_info(detail::cxtoken_stream& stream, const char* name, bool is_ctor)
{ {
prefix_info result; prefix_info result;
@ -302,7 +302,7 @@ namespace
} }
}; };
cpp_cv parse_cv(detail::token_stream& stream) cpp_cv parse_cv(detail::cxtoken_stream& stream)
{ {
if (detail::skip_if(stream, "const")) if (detail::skip_if(stream, "const"))
{ {
@ -322,7 +322,7 @@ namespace
return cpp_cv_none; return cpp_cv_none;
} }
cpp_reference parse_ref(detail::token_stream& stream) cpp_reference parse_ref(detail::cxtoken_stream& stream)
{ {
if (detail::skip_if(stream, "&")) if (detail::skip_if(stream, "&"))
return cpp_ref_lvalue; return cpp_ref_lvalue;
@ -332,7 +332,7 @@ namespace
return cpp_ref_none; return cpp_ref_none;
} }
std::unique_ptr<cpp_expression> parse_noexcept(detail::token_stream& stream, std::unique_ptr<cpp_expression> parse_noexcept(detail::cxtoken_stream& stream,
const detail::parse_context& context) const detail::parse_context& context)
{ {
if (!detail::skip_if(stream, "noexcept")) if (!detail::skip_if(stream, "noexcept"))
@ -351,7 +351,7 @@ namespace
return expr; return expr;
} }
cpp_function_body_kind parse_body_kind(detail::token_stream& stream, bool& pure_virtual) cpp_function_body_kind parse_body_kind(detail::cxtoken_stream& stream, bool& pure_virtual)
{ {
pure_virtual = false; pure_virtual = false;
if (detail::skip_if(stream, "default")) if (detail::skip_if(stream, "default"))
@ -369,7 +369,7 @@ namespace
return cpp_function_declaration; return cpp_function_declaration;
} }
void parse_body(detail::token_stream& stream, suffix_info& result, bool allow_virtual) void parse_body(detail::cxtoken_stream& stream, suffix_info& result, bool allow_virtual)
{ {
auto pure_virtual = false; auto pure_virtual = false;
result.body_kind = parse_body_kind(stream, pure_virtual); result.body_kind = parse_body_kind(stream, pure_virtual);
@ -385,7 +385,7 @@ namespace
} }
// precondition: we've skipped the function parameters // precondition: we've skipped the function parameters
suffix_info parse_suffix_info(detail::token_stream& stream, suffix_info parse_suffix_info(detail::cxtoken_stream& stream,
const detail::parse_context& context, bool allow_qualifier, const detail::parse_context& context, bool allow_qualifier,
bool allow_virtual) bool allow_virtual)
{ {
@ -488,8 +488,8 @@ namespace
{ {
auto name = detail::get_cursor_name(cur); auto name = detail::get_cursor_name(cur);
detail::tokenizer tokenizer(context.tu, context.file, cur); detail::cxtokenizer tokenizer(context.tu, context.file, cur);
detail::token_stream stream(tokenizer, cur); detail::cxtoken_stream stream(tokenizer, cur);
auto prefix = parse_prefix_info(stream, name.c_str(), false); auto prefix = parse_prefix_info(stream, name.c_str(), false);
DEBUG_ASSERT(!prefix.is_virtual && !prefix.is_explicit, detail::parse_error_handler{}, cur, DEBUG_ASSERT(!prefix.is_virtual && !prefix.is_explicit, detail::parse_error_handler{}, cur,
@ -611,7 +611,7 @@ namespace
template <class Builder> template <class Builder>
std::unique_ptr<cpp_entity> handle_suffix(const detail::parse_context& context, std::unique_ptr<cpp_entity> handle_suffix(const detail::parse_context& context,
const CXCursor& cur, Builder& builder, const CXCursor& cur, Builder& builder,
detail::token_stream& stream, bool is_virtual, detail::cxtoken_stream& stream, bool is_virtual,
type_safe::optional<cpp_entity_ref> semantic_parent) type_safe::optional<cpp_entity_ref> semantic_parent)
{ {
auto allow_qualifiers = set_qualifier(0, builder, cpp_cv_none, cpp_ref_none); auto allow_qualifiers = set_qualifier(0, builder, cpp_cv_none, cpp_ref_none);
@ -640,8 +640,8 @@ std::unique_ptr<cpp_entity> detail::parse_cpp_member_function(const detail::pars
detail::assert_handler{}); detail::assert_handler{});
auto name = detail::get_cursor_name(cur); auto name = detail::get_cursor_name(cur);
detail::tokenizer tokenizer(context.tu, context.file, cur); detail::cxtokenizer tokenizer(context.tu, context.file, cur);
detail::token_stream stream(tokenizer, cur); detail::cxtoken_stream stream(tokenizer, cur);
auto prefix = parse_prefix_info(stream, name.c_str(), false); auto prefix = parse_prefix_info(stream, name.c_str(), false);
DEBUG_ASSERT(!prefix.is_explicit, detail::parse_error_handler{}, cur, DEBUG_ASSERT(!prefix.is_explicit, detail::parse_error_handler{}, cur,
@ -670,8 +670,8 @@ std::unique_ptr<cpp_entity> detail::parse_cpp_conversion_op(const detail::parse_
|| clang_getTemplateCursorKind(cur) == CXCursor_ConversionFunction, || clang_getTemplateCursorKind(cur) == CXCursor_ConversionFunction,
detail::assert_handler{}); detail::assert_handler{});
detail::tokenizer tokenizer(context.tu, context.file, cur); detail::cxtokenizer tokenizer(context.tu, context.file, cur);
detail::token_stream stream(tokenizer, cur); detail::cxtoken_stream stream(tokenizer, cur);
auto prefix = parse_prefix_info(stream, "operator", false); auto prefix = parse_prefix_info(stream, "operator", false);
// heuristic to find arguments tokens // heuristic to find arguments tokens
@ -735,8 +735,8 @@ std::unique_ptr<cpp_entity> detail::parse_cpp_constructor(const detail::parse_co
if (pos != std::string::npos) if (pos != std::string::npos)
name.erase(pos); name.erase(pos);
detail::tokenizer tokenizer(context.tu, context.file, cur); detail::cxtokenizer tokenizer(context.tu, context.file, cur);
detail::token_stream stream(tokenizer, cur); detail::cxtoken_stream stream(tokenizer, cur);
auto prefix = parse_prefix_info(stream, name.c_str(), true); auto prefix = parse_prefix_info(stream, name.c_str(), true);
DEBUG_ASSERT(!prefix.is_virtual, detail::parse_error_handler{}, cur, DEBUG_ASSERT(!prefix.is_virtual, detail::parse_error_handler{}, cur,
@ -771,8 +771,8 @@ std::unique_ptr<cpp_entity> detail::parse_cpp_destructor(const detail::parse_con
{ {
DEBUG_ASSERT(clang_getCursorKind(cur) == CXCursor_Destructor, detail::assert_handler{}); DEBUG_ASSERT(clang_getCursorKind(cur) == CXCursor_Destructor, detail::assert_handler{});
detail::tokenizer tokenizer(context.tu, context.file, cur); detail::cxtokenizer tokenizer(context.tu, context.file, cur);
detail::token_stream stream(tokenizer, cur); detail::cxtoken_stream stream(tokenizer, cur);
auto prefix_info = parse_prefix_info(stream, "~", false); auto prefix_info = parse_prefix_info(stream, "~", false);
DEBUG_ASSERT(!prefix_info.is_constexpr && !prefix_info.is_explicit, detail::assert_handler{}); DEBUG_ASSERT(!prefix_info.is_constexpr && !prefix_info.is_explicit, detail::assert_handler{});

4
src/libclang/language_linkage_parser.cpp
View file

@ -17,8 +17,8 @@ std::unique_ptr<cpp_entity> detail::try_parse_cpp_language_linkage(const parse_c
DEBUG_ASSERT(cur.kind == CXCursor_UnexposedDecl, DEBUG_ASSERT(cur.kind == CXCursor_UnexposedDecl,
detail::assert_handler{}); // not exposed currently detail::assert_handler{}); // not exposed currently
detail::tokenizer tokenizer(context.tu, context.file, cur); detail::cxtokenizer tokenizer(context.tu, context.file, cur);
detail::token_stream stream(tokenizer, cur); detail::cxtoken_stream stream(tokenizer, cur);
// extern <name> ... // extern <name> ...
if (!detail::skip_if(stream, "extern")) if (!detail::skip_if(stream, "extern"))

2
src/libclang/libclang_parser.cpp
View file

@ -15,7 +15,7 @@
#include "parse_error.hpp" #include "parse_error.hpp"
#include "parse_functions.hpp" #include "parse_functions.hpp"
#include "preprocessor.hpp" #include "preprocessor.hpp"
#include "tokenizer.hpp" #include "cxtokenizer.hpp"
using namespace cppast; using namespace cppast;

16
src/libclang/namespace_parser.cpp
View file

@ -16,8 +16,8 @@ namespace
cpp_namespace::builder make_ns_builder(const detail::parse_context& context, cpp_namespace::builder make_ns_builder(const detail::parse_context& context,
const CXCursor& cur) const CXCursor& cur)
{ {
detail::tokenizer tokenizer(context.tu, context.file, cur); detail::cxtokenizer tokenizer(context.tu, context.file, cur);
detail::token_stream stream(tokenizer, cur); detail::cxtoken_stream stream(tokenizer, cur);
// [inline] namespace [<attribute>] <identifier> { // [inline] namespace [<attribute>] <identifier> {
auto is_inline = false; auto is_inline = false;
@ -83,8 +83,8 @@ std::unique_ptr<cpp_entity> detail::parse_cpp_namespace_alias(const detail::pars
{ {
DEBUG_ASSERT(cur.kind == CXCursor_NamespaceAlias, detail::assert_handler{}); DEBUG_ASSERT(cur.kind == CXCursor_NamespaceAlias, detail::assert_handler{});
detail::tokenizer tokenizer(context.tu, context.file, cur); detail::cxtokenizer tokenizer(context.tu, context.file, cur);
detail::token_stream stream(tokenizer, cur); detail::cxtoken_stream stream(tokenizer, cur);
// namespace <identifier> = <nested identifier>; // namespace <identifier> = <nested identifier>;
detail::skip(stream, "namespace"); detail::skip(stream, "namespace");
@ -108,8 +108,8 @@ std::unique_ptr<cpp_entity> detail::parse_cpp_using_directive(const detail::pars
{ {
DEBUG_ASSERT(cur.kind == CXCursor_UsingDirective, detail::assert_handler{}); DEBUG_ASSERT(cur.kind == CXCursor_UsingDirective, detail::assert_handler{});
detail::tokenizer tokenizer(context.tu, context.file, cur); detail::cxtokenizer tokenizer(context.tu, context.file, cur);
detail::token_stream stream(tokenizer, cur); detail::cxtoken_stream stream(tokenizer, cur);
// using namespace <nested identifier>; // using namespace <nested identifier>;
detail::skip(stream, "using"); detail::skip(stream, "using");
@ -182,8 +182,8 @@ std::unique_ptr<cpp_entity> detail::parse_cpp_using_declaration(
{ {
DEBUG_ASSERT(cur.kind == CXCursor_UsingDeclaration, detail::assert_handler{}); DEBUG_ASSERT(cur.kind == CXCursor_UsingDeclaration, detail::assert_handler{});
detail::tokenizer tokenizer(context.tu, context.file, cur); detail::cxtokenizer tokenizer(context.tu, context.file, cur);
detail::token_stream stream(tokenizer, cur); detail::cxtoken_stream stream(tokenizer, cur);
// using <nested identifier>; // using <nested identifier>;
detail::skip(stream, "using"); detail::skip(stream, "using");

8
src/libclang/parse_functions.cpp
View file

@ -215,10 +215,10 @@ std::unique_ptr<cpp_entity> detail::parse_entity(const detail::parse_context& co
detail::get_cursor_kind_spelling(cur).c_str(), "'")); detail::get_cursor_kind_spelling(cur).c_str(), "'"));
// build unexposed entity // build unexposed entity
auto name = detail::get_cursor_name(cur); auto name = detail::get_cursor_name(cur);
detail::tokenizer tokenizer(context.tu, context.file, cur); detail::cxtokenizer tokenizer(context.tu, context.file, cur);
detail::token_stream stream(tokenizer, cur); detail::cxtoken_stream stream(tokenizer, cur);
auto spelling = detail::to_string(stream, stream.end()); auto spelling = detail::to_string(stream, stream.end());
std::unique_ptr<cppast::cpp_entity> entity; std::unique_ptr<cppast::cpp_entity> entity;
if (name.empty()) if (name.empty())

10
src/libclang/parse_functions.hpp
View file

@ -9,7 +9,7 @@
#include <cppast/parser.hpp> #include <cppast/parser.hpp>
#include "raii_wrapper.hpp" #include "raii_wrapper.hpp"
#include "tokenizer.hpp" // for convenience #include "cxtokenizer.hpp" // for convenience
#include "parse_error.hpp" // for convenience #include "parse_error.hpp" // for convenience
#include "preprocessor.hpp" #include "preprocessor.hpp"
@ -76,8 +76,8 @@ namespace cppast
// and ends at the given iterator // and ends at the given iterator
// this is required for situations where there is no type exposed, // this is required for situations where there is no type exposed,
// like default type of a template type parameter // like default type of a template type parameter
std::unique_ptr<cpp_type> parse_raw_type(const parse_context& context, token_stream& stream, std::unique_ptr<cpp_type> parse_raw_type(const parse_context& context,
token_iterator end); cxtoken_stream& stream, cxtoken_iterator end);
std::unique_ptr<cpp_expression> parse_expression(const parse_context& context, std::unique_ptr<cpp_expression> parse_expression(const parse_context& context,
const CXCursor& cur); const CXCursor& cur);
@ -86,8 +86,8 @@ namespace cppast
// this is required for situations where there is no expression cursor exposed, // this is required for situations where there is no expression cursor exposed,
// like member initializers // like member initializers
std::unique_ptr<cpp_expression> parse_raw_expression(const parse_context& context, std::unique_ptr<cpp_expression> parse_raw_expression(const parse_context& context,
token_stream& stream, cxtoken_stream& stream,
token_iterator end, cxtoken_iterator end,
std::unique_ptr<cpp_type> type); std::unique_ptr<cpp_type> type);
// parse_entity() dispatches on the cursor type // parse_entity() dispatches on the cursor type

20
src/libclang/template_parser.cpp
View file

@ -50,9 +50,9 @@ namespace
DEBUG_ASSERT(clang_getCursorKind(cur) == CXCursor_TemplateTypeParameter, DEBUG_ASSERT(clang_getCursorKind(cur) == CXCursor_TemplateTypeParameter,
detail::assert_handler{}); detail::assert_handler{});
detail::tokenizer tokenizer(context.tu, context.file, cur); detail::cxtokenizer tokenizer(context.tu, context.file, cur);
detail::token_stream stream(tokenizer, cur); detail::cxtoken_stream stream(tokenizer, cur);
auto name = detail::get_cursor_name(cur); auto name = detail::get_cursor_name(cur);
// syntax: typename/class [...] name [= ...] // syntax: typename/class [...] name [= ...]
auto keyword = cpp_template_keyword::keyword_class; auto keyword = cpp_template_keyword::keyword_class;
@ -87,8 +87,8 @@ namespace
auto type = clang_getCursorType(cur); auto type = clang_getCursorType(cur);
auto def = detail::parse_default_value(context, cur, name.c_str()); auto def = detail::parse_default_value(context, cur, name.c_str());
detail::tokenizer tokenizer(context.tu, context.file, cur); detail::cxtokenizer tokenizer(context.tu, context.file, cur);
detail::token_stream stream(tokenizer, cur); detail::cxtoken_stream stream(tokenizer, cur);
// see if it is variadic // see if it is variadic
// syntax a): some-tokens ... name some-tokens // syntax a): some-tokens ... name some-tokens
@ -120,9 +120,9 @@ namespace
DEBUG_ASSERT(clang_getCursorKind(cur) == CXCursor_TemplateTemplateParameter, DEBUG_ASSERT(clang_getCursorKind(cur) == CXCursor_TemplateTemplateParameter,
detail::assert_handler{}); detail::assert_handler{});
detail::tokenizer tokenizer(context.tu, context.file, cur); detail::cxtokenizer tokenizer(context.tu, context.file, cur);
detail::token_stream stream(tokenizer, cur); detail::cxtoken_stream stream(tokenizer, cur);
auto name = detail::get_cursor_name(cur); auto name = detail::get_cursor_name(cur);
// syntax: template <…> class/typename [...] name [= …] // syntax: template <…> class/typename [...] name [= …]
detail::skip(stream, "template"); detail::skip(stream, "template");
@ -263,8 +263,8 @@ namespace
template <class Builder> template <class Builder>
void parse_arguments(Builder& b, const detail::parse_context& context, const CXCursor& cur) void parse_arguments(Builder& b, const detail::parse_context& context, const CXCursor& cur)
{ {
detail::tokenizer tokenizer(context.tu, context.file, cur); detail::cxtokenizer tokenizer(context.tu, context.file, cur);
detail::token_stream stream(tokenizer, cur); detail::cxtoken_stream stream(tokenizer, cur);
while (!stream.done() while (!stream.done()
&& !detail::skip_if(stream, detail::get_cursor_name(cur).c_str(), true)) && !detail::skip_if(stream, detail::get_cursor_name(cur).c_str(), true))

4
src/libclang/type_parser.cpp
View file

@ -731,8 +731,8 @@ std::unique_ptr<cpp_type> detail::parse_type(const detail::parse_context& contex
} }
std::unique_ptr<cpp_type> detail::parse_raw_type(const detail::parse_context&, std::unique_ptr<cpp_type> detail::parse_raw_type(const detail::parse_context&,
detail::token_stream& stream, detail::cxtoken_stream& stream,
detail::token_iterator end) detail::cxtoken_iterator end)
{ {
auto result = detail::to_string(stream, end); auto result = detail::to_string(stream, end);
return cpp_unexposed_type::build(result.as_string()); return cpp_unexposed_type::build(result.as_string());

6
src/libclang/variable_parser.cpp
View file

@ -14,8 +14,8 @@ using namespace cppast;
std::unique_ptr<cpp_expression> detail::parse_default_value(const detail::parse_context& context, std::unique_ptr<cpp_expression> detail::parse_default_value(const detail::parse_context& context,
const CXCursor& cur, const char* name) const CXCursor& cur, const char* name)
{ {
detail::tokenizer tokenizer(context.tu, context.file, cur); detail::cxtokenizer tokenizer(context.tu, context.file, cur);
detail::token_stream stream(tokenizer, cur); detail::cxtoken_stream stream(tokenizer, cur);
auto has_default = false; auto has_default = false;
auto got_name = *name == '\0'; auto got_name = *name == '\0';
@ -57,7 +57,7 @@ std::unique_ptr<cpp_entity> detail::parse_cpp_variable(const detail::parse_conte
// just look for thread local or constexpr // just look for thread local or constexpr
// can't appear anywhere else, so good enough // can't appear anywhere else, so good enough
detail::tokenizer tokenizer(context.tu, context.file, cur); detail::cxtokenizer tokenizer(context.tu, context.file, cur);
for (auto& token : tokenizer) for (auto& token : tokenizer)
if (token.value() == "thread_local") if (token.value() == "thread_local")
storage_class = storage_class =