jyapayne/cppast
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Rename tokenizer stuff to include cx prefix

Browse source
This commit is contained in:
Jonathan Müller 2017-10-12 18:43:12 +02:00
commit cfac41c7b5
16 changed files with 121 additions and 120 deletions
Download patch file Download diff file Expand all files Collapse all files

4
src/CMakeLists.txt
View file

@ -95,8 +95,8 @@ set(libclang_source
libclang/preprocessor.hpp
libclang/raii_wrapper.hpp
libclang/template_parser.cpp
libclang/tokenizer.cpp
libclang/tokenizer.hpp
libclang/cxtokenizer.cpp
libclang/cxtokenizer.hpp
libclang/type_parser.cpp
libclang/variable_parser.cpp)

8
src/libclang/class_parser.cpp
View file

@ -71,8 +71,8 @@ namespace
auto access = convert_access(cur);
auto is_virtual = clang_isVirtualBase(cur) != 0u;
detail::tokenizer tokenizer(context.tu, context.file, cur);
detail::token_stream stream(tokenizer, cur);
detail::cxtokenizer tokenizer(context.tu, context.file, cur);
detail::cxtoken_stream stream(tokenizer, cur);
// [<attribute>] [virtual] [<access>] <name>
// can't use spelling to get the name
@ -108,8 +108,8 @@ std::unique_ptr<cpp_entity> detail::parse_cpp_class(const detail::parse_context&
clang_getCursorLexicalParent(cur)))
{
// out-of-line definition
detail::tokenizer tokenizer(context.tu, context.file, cur);
detail::token_stream stream(tokenizer, cur);
detail::cxtokenizer tokenizer(context.tu, context.file, cur);
detail::cxtoken_stream stream(tokenizer, cur);
std::string name = detail::get_cursor_name(cur).c_str();
auto pos = name.find('<');

27
src/libclang/tokenizer.cpp → src/libclang/cxtokenizer.cpp
View file

@ -2,7 +2,7 @@
// This file is subject to the license terms in the LICENSE file
// found in the top-level directory of this distribution.
#include "tokenizer.hpp"
#include "cxtokenizer.hpp"
#include <cctype>
@ -11,7 +11,7 @@
using namespace cppast;
detail::token::token(const CXTranslationUnit& tu_unit, const CXToken& token)
detail::cxtoken::cxtoken(const CXTranslationUnit& tu_unit, const CXToken& token)
: value_(clang_getTokenSpelling(tu_unit, token)), kind_(clang_getTokenKind(token))
{
}
@ -237,7 +237,8 @@ namespace
}
}
detail::tokenizer::tokenizer(const CXTranslationUnit& tu, const CXFile& file, const CXCursor& cur)
detail::cxtokenizer::cxtokenizer(const CXTranslationUnit& tu, const CXFile& file,
const CXCursor& cur)
{
auto extent = get_extent(tu, file, cur, unmunch_);
@ -247,7 +248,7 @@ detail::tokenizer::tokenizer(const CXTranslationUnit& tu, const CXFile& file, co
tokens_.emplace_back(tu, tokenizer[i]);
}
void detail::skip(detail::token_stream& stream, const char* str)
void detail::skip(detail::cxtoken_stream& stream, const char* str)
{
if (*str)
{
@ -263,7 +264,7 @@ void detail::skip(detail::token_stream& stream, const char* str)
namespace
{
bool starts_with(const char*& str, const detail::token& t)
bool starts_with(const char*& str, const detail::cxtoken& t)
{
if (std::strncmp(str, t.c_str(), t.value().length()) != 0)
return false;
@ -274,7 +275,7 @@ namespace
}
}
bool detail::skip_if(detail::token_stream& stream, const char* str, bool multi_token)
bool detail::skip_if(detail::cxtoken_stream& stream, const char* str, bool multi_token)
{
if (!*str)
return true;
@ -298,7 +299,7 @@ namespace
{
// whether or not the current angle bracket can be a comparison
// note: this is a heuristic I hope works often enough
bool is_comparison(CXTokenKind last_kind, const detail::token& cur, CXTokenKind next_kind)
bool is_comparison(CXTokenKind last_kind, const detail::cxtoken& cur, CXTokenKind next_kind)
{
if (cur == "<")
return last_kind == CXToken_Literal;
@ -308,7 +309,7 @@ namespace
}
}
detail::token_iterator detail::find_closing_bracket(detail::token_stream stream)
detail::cxtoken_iterator detail::find_closing_bracket(detail::cxtoken_stream stream)
{
auto template_bracket = false;
auto open_bracket = stream.peek().c_str();
@ -359,7 +360,7 @@ detail::token_iterator detail::find_closing_bracket(detail::token_stream stream)
return stream.cur();
}
void detail::skip_brackets(detail::token_stream& stream)
void detail::skip_brackets(detail::cxtoken_stream& stream)
{
auto closing = find_closing_bracket(stream);
stream.set_cur(std::next(closing));
@ -367,7 +368,7 @@ void detail::skip_brackets(detail::token_stream& stream)
namespace
{
bool skip_attribute_impl(detail::token_stream& stream)
bool skip_attribute_impl(detail::cxtoken_stream& stream)
{
if (skip_if(stream, "[") && stream.peek() == "[")
{
@ -401,7 +402,7 @@ namespace
}
}
bool detail::skip_attribute(detail::token_stream& stream)
bool detail::skip_attribute(detail::cxtoken_stream& stream)
{
auto any = false;
while (skip_attribute_impl(stream))
@ -432,7 +433,7 @@ namespace
}
}
cpp_token_string detail::to_string(token_stream& stream, token_iterator end)
cpp_token_string detail::to_string(cxtoken_stream& stream, cxtoken_iterator end)
{
cpp_token_string::builder builder;
@ -448,7 +449,7 @@ cpp_token_string detail::to_string(token_stream& stream, token_iterator end)
return builder.finish();
}
bool detail::append_scope(detail::token_stream& stream, std::string& scope)
bool detail::append_scope(detail::cxtoken_stream& stream, std::string& scope)
{
// add identifiers and "::" to current scope name,
// clear if there is any other token in between, or mismatched combination

70
src/libclang/tokenizer.hpp → src/libclang/cxtokenizer.hpp
View file

@ -2,8 +2,8 @@
// This file is subject to the license terms in the LICENSE file
// found in the top-level directory of this distribution.
#ifndef CPPAST_TOKENIZER_HPP_INCLUDED
#define CPPAST_TOKENIZER_HPP_INCLUDED
#ifndef CPPAST_CXTOKENIZER_HPP_INCLUDED
#define CPPAST_CXTOKENIZER_HPP_INCLUDED
#include <string>
#include <vector>
@ -16,10 +16,10 @@ namespace cppast
{
namespace detail
{
class token
class cxtoken
{
public:
explicit token(const CXTranslationUnit& tu_unit, const CXToken& token);
explicit cxtoken(const CXTranslationUnit& tu_unit, const CXToken& token);
const cxstring& value() const noexcept
{
@ -41,40 +41,40 @@ namespace cppast
CXTokenKind kind_;
};
inline bool operator==(const token& tok, const char* str) noexcept
inline bool operator==(const cxtoken& tok, const char* str) noexcept
{
return tok.value() == str;
}
inline bool operator==(const char* str, const token& tok) noexcept
inline bool operator==(const char* str, const cxtoken& tok) noexcept
{
return str == tok.value();
}
inline bool operator!=(const token& tok, const char* str) noexcept
inline bool operator!=(const cxtoken& tok, const char* str) noexcept
{
return !(tok == str);
}
inline bool operator!=(const char* str, const token& tok) noexcept
inline bool operator!=(const char* str, const cxtoken& tok) noexcept
{
return !(str == tok);
}
using token_iterator = std::vector<token>::const_iterator;
using cxtoken_iterator = std::vector<cxtoken>::const_iterator;
class tokenizer
class cxtokenizer
{
public:
explicit tokenizer(const CXTranslationUnit& tu, const CXFile& file,
const CXCursor& cur);
explicit cxtokenizer(const CXTranslationUnit& tu, const CXFile& file,
const CXCursor& cur);
token_iterator begin() const noexcept
cxtoken_iterator begin() const noexcept
{
return tokens_.begin();
}
token_iterator end() const noexcept
cxtoken_iterator end() const noexcept
{
return tokens_.end();
}
@ -88,14 +88,14 @@ namespace cppast
}
private:
std::vector<token> tokens_;
bool unmunch_;
std::vector<cxtoken> tokens_;
bool unmunch_;
};
class token_stream
class cxtoken_stream
{
public:
explicit token_stream(const tokenizer& tokenizer, const CXCursor& cur)
explicit cxtoken_stream(const cxtokenizer& tokenizer, const CXCursor& cur)
: cursor_(cur),
begin_(tokenizer.begin()),
cur_(begin_),
@ -104,7 +104,7 @@ namespace cppast
{
}
const token& peek() const noexcept
const cxtoken& peek() const noexcept
{
if (done())
return *std::prev(end_);
@ -123,7 +123,7 @@ namespace cppast
--cur_;
}
const token& get() noexcept
const cxtoken& get() noexcept
{
auto& result = peek();
bump();
@ -140,22 +140,22 @@ namespace cppast
return cursor_;
}
token_iterator begin() const noexcept
cxtoken_iterator begin() const noexcept
{
return begin_;
}
token_iterator cur() const noexcept
cxtoken_iterator cur() const noexcept
{
return cur_;
}
token_iterator end() const noexcept
cxtoken_iterator end() const noexcept
{
return end_;
}
void set_cur(token_iterator iter) noexcept
void set_cur(cxtoken_iterator iter) noexcept
{
cur_ = iter;
}
@ -166,41 +166,41 @@ namespace cppast
}
private:
CXCursor cursor_;
token_iterator begin_, cur_, end_;
bool unmunch_;
CXCursor cursor_;
cxtoken_iterator begin_, cur_, end_;
bool unmunch_;
};
// skips the next token
// asserts that it has the given string
void skip(token_stream& stream, const char* str);
void skip(cxtoken_stream& stream, const char* str);
// skips the next token if it has the given string
// if multi_token == true, str can consist of multiple tokens optionally separated by whitespace
bool skip_if(token_stream& stream, const char* str, bool multi_token = false);
bool skip_if(cxtoken_stream& stream, const char* str, bool multi_token = false);
// returns the location of the closing bracket
// the current token must be (,[,{ or <
// note: < might not work in the arguments of a template specialization
token_iterator find_closing_bracket(token_stream stream);
cxtoken_iterator find_closing_bracket(cxtoken_stream stream);
// skips brackets
// the current token must be (,[,{ or <
// note: < might not work in the arguments of a template specialization
void skip_brackets(token_stream& stream);
void skip_brackets(cxtoken_stream& stream);
// skips an attribute
bool skip_attribute(token_stream& stream);
bool skip_attribute(cxtoken_stream& stream);
// converts a token range to a string
cpp_token_string to_string(token_stream& stream, token_iterator end);
cpp_token_string to_string(cxtoken_stream& stream, cxtoken_iterator end);
// appends token to scope, if it is still valid
// else clears it
// note: does not consume the token if it is not valid,
// returns false in that case
bool append_scope(token_stream& stream, std::string& scope);
bool append_scope(cxtoken_stream& stream, std::string& scope);
}
} // namespace cppast::detail
#endif // CPPAST_TOKENIZER_HPP_INCLUDED
#endif // CPPAST_CXTOKENIZER_HPP_INCLUDED

4
src/libclang/debug_helper.cpp
View file

@ -7,7 +7,7 @@
#include <cstdio>
#include <mutex>
#include "tokenizer.hpp"
#include "cxtokenizer.hpp"
using namespace cppast;
@ -50,7 +50,7 @@ void detail::print_tokens(const CXTranslationUnit& tu, const CXFile& file,
const CXCursor& cur) noexcept
{
std::lock_guard<std::mutex> lock(mtx);
detail::tokenizer tokenizer(tu, file, cur);
detail::cxtokenizer tokenizer(tu, file, cur);
for (auto& token : tokenizer)
std::fprintf(stderr, "%s ", token.c_str());
std::fputs("\n", stderr);

10
src/libclang/enum_parser.cpp
View file

@ -20,8 +20,8 @@ namespace
DEBUG_ASSERT(cur.kind == CXCursor_EnumConstantDecl, detail::parse_error_handler{}, cur,
"unexpected child cursor of enum");
detail::tokenizer tokenizer(context.tu, context.file, cur);
detail::token_stream stream(tokenizer, cur);
detail::cxtokenizer tokenizer(context.tu, context.file, cur);
detail::cxtoken_stream stream(tokenizer, cur);
// <identifier> [<attribute>],
// or: <identifier> [<attribute>] = <expression>,
@ -47,9 +47,9 @@ namespace
cpp_enum::builder make_enum_builder(const detail::parse_context& context, const CXCursor& cur,
type_safe::optional<cpp_entity_ref>& semantic_parent)
{
auto name = detail::get_cursor_name(cur);
detail::tokenizer tokenizer(context.tu, context.file, cur);
detail::token_stream stream(tokenizer, cur);
auto name = detail::get_cursor_name(cur);
detail::cxtokenizer tokenizer(context.tu, context.file, cur);
detail::cxtoken_stream stream(tokenizer, cur);
// [<attribute>] enum [class] [<attribute>] name [: type] {
detail::skip_attribute(stream);

8
src/libclang/expression_parser.cpp
View file

@ -14,8 +14,8 @@ std::unique_ptr<cpp_expression> detail::parse_expression(const detail::parse_con
auto kind = clang_getCursorKind(cur);
DEBUG_ASSERT(clang_isExpression(kind), detail::assert_handler{});
detail::tokenizer tokenizer(context.tu, context.file, cur);
detail::token_stream stream(tokenizer, cur);
detail::cxtokenizer tokenizer(context.tu, context.file, cur);
detail::cxtoken_stream stream(tokenizer, cur);
auto type = parse_type(context, cur, clang_getCursorType(cur));
auto expr = to_string(stream, stream.end());
@ -36,8 +36,8 @@ std::unique_ptr<cpp_expression> detail::parse_expression(const detail::parse_con
}
std::unique_ptr<cpp_expression> detail::parse_raw_expression(const parse_context&,
token_stream& stream,
token_iterator end,
cxtoken_stream& stream,
cxtoken_iterator end,
std::unique_ptr<cpp_type> type)
{
if (stream.done())

40
src/libclang/function_parser.cpp
View file

@ -92,7 +92,7 @@ namespace
}
// precondition: after the name
void skip_parameters(detail::token_stream& stream)
void skip_parameters(detail::cxtoken_stream& stream)
{
if (stream.peek() == "<")
// specialization arguments
@ -218,7 +218,7 @@ namespace
bool is_friend = false;
};
bool prefix_end(detail::token_stream& stream, const char* name, bool is_ctor)
bool prefix_end(detail::cxtoken_stream& stream, const char* name, bool is_ctor)
{
auto cur = stream.cur();
// name can have multiple tokens if it is an operator
@ -262,7 +262,7 @@ namespace
return true;
}
prefix_info parse_prefix_info(detail::token_stream& stream, const char* name, bool is_ctor)
prefix_info parse_prefix_info(detail::cxtoken_stream& stream, const char* name, bool is_ctor)
{
prefix_info result;
@ -302,7 +302,7 @@ namespace
}
};
cpp_cv parse_cv(detail::token_stream& stream)
cpp_cv parse_cv(detail::cxtoken_stream& stream)
{
if (detail::skip_if(stream, "const"))
{
@ -322,7 +322,7 @@ namespace
return cpp_cv_none;
}
cpp_reference parse_ref(detail::token_stream& stream)
cpp_reference parse_ref(detail::cxtoken_stream& stream)
{
if (detail::skip_if(stream, "&"))
return cpp_ref_lvalue;
@ -332,7 +332,7 @@ namespace
return cpp_ref_none;
}
std::unique_ptr<cpp_expression> parse_noexcept(detail::token_stream& stream,
std::unique_ptr<cpp_expression> parse_noexcept(detail::cxtoken_stream& stream,
const detail::parse_context& context)
{
if (!detail::skip_if(stream, "noexcept"))
@ -351,7 +351,7 @@ namespace
return expr;
}
cpp_function_body_kind parse_body_kind(detail::token_stream& stream, bool& pure_virtual)
cpp_function_body_kind parse_body_kind(detail::cxtoken_stream& stream, bool& pure_virtual)
{
pure_virtual = false;
if (detail::skip_if(stream, "default"))
@ -369,7 +369,7 @@ namespace
return cpp_function_declaration;
}
void parse_body(detail::token_stream& stream, suffix_info& result, bool allow_virtual)
void parse_body(detail::cxtoken_stream& stream, suffix_info& result, bool allow_virtual)
{
auto pure_virtual = false;
result.body_kind = parse_body_kind(stream, pure_virtual);
@ -385,7 +385,7 @@ namespace
}
// precondition: we've skipped the function parameters
suffix_info parse_suffix_info(detail::token_stream& stream,
suffix_info parse_suffix_info(detail::cxtoken_stream& stream,
const detail::parse_context& context, bool allow_qualifier,
bool allow_virtual)
{
@ -488,8 +488,8 @@ namespace
{
auto name = detail::get_cursor_name(cur);
detail::tokenizer tokenizer(context.tu, context.file, cur);
detail::token_stream stream(tokenizer, cur);
detail::cxtokenizer tokenizer(context.tu, context.file, cur);
detail::cxtoken_stream stream(tokenizer, cur);
auto prefix = parse_prefix_info(stream, name.c_str(), false);
DEBUG_ASSERT(!prefix.is_virtual && !prefix.is_explicit, detail::parse_error_handler{}, cur,
@ -611,7 +611,7 @@ namespace
template <class Builder>
std::unique_ptr<cpp_entity> handle_suffix(const detail::parse_context& context,
const CXCursor& cur, Builder& builder,
detail::token_stream& stream, bool is_virtual,
detail::cxtoken_stream& stream, bool is_virtual,
type_safe::optional<cpp_entity_ref> semantic_parent)
{
auto allow_qualifiers = set_qualifier(0, builder, cpp_cv_none, cpp_ref_none);
@ -640,8 +640,8 @@ std::unique_ptr<cpp_entity> detail::parse_cpp_member_function(const detail::pars
detail::assert_handler{});
auto name = detail::get_cursor_name(cur);
detail::tokenizer tokenizer(context.tu, context.file, cur);
detail::token_stream stream(tokenizer, cur);
detail::cxtokenizer tokenizer(context.tu, context.file, cur);
detail::cxtoken_stream stream(tokenizer, cur);
auto prefix = parse_prefix_info(stream, name.c_str(), false);
DEBUG_ASSERT(!prefix.is_explicit, detail::parse_error_handler{}, cur,
@ -670,8 +670,8 @@ std::unique_ptr<cpp_entity> detail::parse_cpp_conversion_op(const detail::parse_
|| clang_getTemplateCursorKind(cur) == CXCursor_ConversionFunction,
detail::assert_handler{});
detail::tokenizer tokenizer(context.tu, context.file, cur);
detail::token_stream stream(tokenizer, cur);
detail::cxtokenizer tokenizer(context.tu, context.file, cur);
detail::cxtoken_stream stream(tokenizer, cur);
auto prefix = parse_prefix_info(stream, "operator", false);
// heuristic to find arguments tokens
@ -735,8 +735,8 @@ std::unique_ptr<cpp_entity> detail::parse_cpp_constructor(const detail::parse_co
if (pos != std::string::npos)
name.erase(pos);
detail::tokenizer tokenizer(context.tu, context.file, cur);
detail::token_stream stream(tokenizer, cur);
detail::cxtokenizer tokenizer(context.tu, context.file, cur);
detail::cxtoken_stream stream(tokenizer, cur);
auto prefix = parse_prefix_info(stream, name.c_str(), true);
DEBUG_ASSERT(!prefix.is_virtual, detail::parse_error_handler{}, cur,
@ -771,8 +771,8 @@ std::unique_ptr<cpp_entity> detail::parse_cpp_destructor(const detail::parse_con
{
DEBUG_ASSERT(clang_getCursorKind(cur) == CXCursor_Destructor, detail::assert_handler{});
detail::tokenizer tokenizer(context.tu, context.file, cur);
detail::token_stream stream(tokenizer, cur);
detail::cxtokenizer tokenizer(context.tu, context.file, cur);
detail::cxtoken_stream stream(tokenizer, cur);
auto prefix_info = parse_prefix_info(stream, "~", false);
DEBUG_ASSERT(!prefix_info.is_constexpr && !prefix_info.is_explicit, detail::assert_handler{});

4
src/libclang/language_linkage_parser.cpp
View file

@ -17,8 +17,8 @@ std::unique_ptr<cpp_entity> detail::try_parse_cpp_language_linkage(const parse_c
DEBUG_ASSERT(cur.kind == CXCursor_UnexposedDecl,
detail::assert_handler{}); // not exposed currently
detail::tokenizer tokenizer(context.tu, context.file, cur);
detail::token_stream stream(tokenizer, cur);
detail::cxtokenizer tokenizer(context.tu, context.file, cur);
detail::cxtoken_stream stream(tokenizer, cur);
// extern <name> ...
if (!detail::skip_if(stream, "extern"))

2
src/libclang/libclang_parser.cpp
View file

@ -15,7 +15,7 @@
#include "parse_error.hpp"
#include "parse_functions.hpp"
#include "preprocessor.hpp"
#include "tokenizer.hpp"
#include "cxtokenizer.hpp"
using namespace cppast;

16
src/libclang/namespace_parser.cpp
View file

@ -16,8 +16,8 @@ namespace
cpp_namespace::builder make_ns_builder(const detail::parse_context& context,
const CXCursor& cur)
{
detail::tokenizer tokenizer(context.tu, context.file, cur);
detail::token_stream stream(tokenizer, cur);
detail::cxtokenizer tokenizer(context.tu, context.file, cur);
detail::cxtoken_stream stream(tokenizer, cur);
// [inline] namespace [<attribute>] <identifier> {
auto is_inline = false;
@ -83,8 +83,8 @@ std::unique_ptr<cpp_entity> detail::parse_cpp_namespace_alias(const detail::pars
{
DEBUG_ASSERT(cur.kind == CXCursor_NamespaceAlias, detail::assert_handler{});
detail::tokenizer tokenizer(context.tu, context.file, cur);
detail::token_stream stream(tokenizer, cur);
detail::cxtokenizer tokenizer(context.tu, context.file, cur);
detail::cxtoken_stream stream(tokenizer, cur);
// namespace <identifier> = <nested identifier>;
detail::skip(stream, "namespace");
@ -108,8 +108,8 @@ std::unique_ptr<cpp_entity> detail::parse_cpp_using_directive(const detail::pars
{
DEBUG_ASSERT(cur.kind == CXCursor_UsingDirective, detail::assert_handler{});
detail::tokenizer tokenizer(context.tu, context.file, cur);
detail::token_stream stream(tokenizer, cur);
detail::cxtokenizer tokenizer(context.tu, context.file, cur);
detail::cxtoken_stream stream(tokenizer, cur);
// using namespace <nested identifier>;
detail::skip(stream, "using");
@ -182,8 +182,8 @@ std::unique_ptr<cpp_entity> detail::parse_cpp_using_declaration(
{
DEBUG_ASSERT(cur.kind == CXCursor_UsingDeclaration, detail::assert_handler{});
detail::tokenizer tokenizer(context.tu, context.file, cur);
detail::token_stream stream(tokenizer, cur);
detail::cxtokenizer tokenizer(context.tu, context.file, cur);
detail::cxtoken_stream stream(tokenizer, cur);
// using <nested identifier>;
detail::skip(stream, "using");

8
src/libclang/parse_functions.cpp
View file

@ -215,10 +215,10 @@ std::unique_ptr<cpp_entity> detail::parse_entity(const detail::parse_context& co
detail::get_cursor_kind_spelling(cur).c_str(), "'"));
// build unexposed entity
auto name = detail::get_cursor_name(cur);
detail::tokenizer tokenizer(context.tu, context.file, cur);
detail::token_stream stream(tokenizer, cur);
auto spelling = detail::to_string(stream, stream.end());
auto name = detail::get_cursor_name(cur);
detail::cxtokenizer tokenizer(context.tu, context.file, cur);
detail::cxtoken_stream stream(tokenizer, cur);
auto spelling = detail::to_string(stream, stream.end());
std::unique_ptr<cppast::cpp_entity> entity;
if (name.empty())

10
src/libclang/parse_functions.hpp
View file

@ -9,7 +9,7 @@
#include <cppast/parser.hpp>
#include "raii_wrapper.hpp"
#include "tokenizer.hpp" // for convenience
#include "cxtokenizer.hpp" // for convenience
#include "parse_error.hpp" // for convenience
#include "preprocessor.hpp"
@ -76,8 +76,8 @@ namespace cppast
// and ends at the given iterator
// this is required for situations where there is no type exposed,
// like default type of a template type parameter
std::unique_ptr<cpp_type> parse_raw_type(const parse_context& context, token_stream& stream,
token_iterator end);
std::unique_ptr<cpp_type> parse_raw_type(const parse_context& context,
cxtoken_stream& stream, cxtoken_iterator end);
std::unique_ptr<cpp_expression> parse_expression(const parse_context& context,
const CXCursor& cur);
@ -86,8 +86,8 @@ namespace cppast
// this is required for situations where there is no expression cursor exposed,
// like member initializers
std::unique_ptr<cpp_expression> parse_raw_expression(const parse_context& context,
token_stream& stream,
token_iterator end,
cxtoken_stream& stream,
cxtoken_iterator end,
std::unique_ptr<cpp_type> type);
// parse_entity() dispatches on the cursor type

20
src/libclang/template_parser.cpp
View file

@ -50,9 +50,9 @@ namespace
DEBUG_ASSERT(clang_getCursorKind(cur) == CXCursor_TemplateTypeParameter,
detail::assert_handler{});
detail::tokenizer tokenizer(context.tu, context.file, cur);
detail::token_stream stream(tokenizer, cur);
auto name = detail::get_cursor_name(cur);
detail::cxtokenizer tokenizer(context.tu, context.file, cur);
detail::cxtoken_stream stream(tokenizer, cur);
auto name = detail::get_cursor_name(cur);
// syntax: typename/class [...] name [= ...]
auto keyword = cpp_template_keyword::keyword_class;
@ -87,8 +87,8 @@ namespace
auto type = clang_getCursorType(cur);
auto def = detail::parse_default_value(context, cur, name.c_str());
detail::tokenizer tokenizer(context.tu, context.file, cur);
detail::token_stream stream(tokenizer, cur);
detail::cxtokenizer tokenizer(context.tu, context.file, cur);
detail::cxtoken_stream stream(tokenizer, cur);
// see if it is variadic
// syntax a): some-tokens ... name some-tokens
@ -120,9 +120,9 @@ namespace
DEBUG_ASSERT(clang_getCursorKind(cur) == CXCursor_TemplateTemplateParameter,
detail::assert_handler{});
detail::tokenizer tokenizer(context.tu, context.file, cur);
detail::token_stream stream(tokenizer, cur);
auto name = detail::get_cursor_name(cur);
detail::cxtokenizer tokenizer(context.tu, context.file, cur);
detail::cxtoken_stream stream(tokenizer, cur);
auto name = detail::get_cursor_name(cur);
// syntax: template <…> class/typename [...] name [= …]
detail::skip(stream, "template");
@ -263,8 +263,8 @@ namespace
template <class Builder>
void parse_arguments(Builder& b, const detail::parse_context& context, const CXCursor& cur)
{
detail::tokenizer tokenizer(context.tu, context.file, cur);
detail::token_stream stream(tokenizer, cur);
detail::cxtokenizer tokenizer(context.tu, context.file, cur);
detail::cxtoken_stream stream(tokenizer, cur);
while (!stream.done()
&& !detail::skip_if(stream, detail::get_cursor_name(cur).c_str(), true))

4
src/libclang/type_parser.cpp
View file

@ -731,8 +731,8 @@ std::unique_ptr<cpp_type> detail::parse_type(const detail::parse_context& contex
}
std::unique_ptr<cpp_type> detail::parse_raw_type(const detail::parse_context&,
detail::token_stream& stream,
detail::token_iterator end)
detail::cxtoken_stream& stream,
detail::cxtoken_iterator end)
{
auto result = detail::to_string(stream, end);
return cpp_unexposed_type::build(result.as_string());

6
src/libclang/variable_parser.cpp
View file

@ -14,8 +14,8 @@ using namespace cppast;
std::unique_ptr<cpp_expression> detail::parse_default_value(const detail::parse_context& context,
const CXCursor& cur, const char* name)
{
detail::tokenizer tokenizer(context.tu, context.file, cur);
detail::token_stream stream(tokenizer, cur);
detail::cxtokenizer tokenizer(context.tu, context.file, cur);
detail::cxtoken_stream stream(tokenizer, cur);
auto has_default = false;
auto got_name = *name == '\0';
@ -57,7 +57,7 @@ std::unique_ptr<cpp_entity> detail::parse_cpp_variable(const detail::parse_conte
// just look for thread local or constexpr
// can't appear anywhere else, so good enough
detail::tokenizer tokenizer(context.tu, context.file, cur);
detail::cxtokenizer tokenizer(context.tu, context.file, cur);
for (auto& token : tokenizer)
if (token.value() == "thread_local")
storage_class =