jyapayne/cppast
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Rename tokenizer stuff to include cx prefix

Browse source
This commit is contained in:
Jonathan Müller 2017-10-12 18:43:12 +02:00
commit cfac41c7b5
16 changed files with 121 additions and 120 deletions
Download patch file Download diff file Expand all files Collapse all files

206
src/libclang/tokenizer.hpp
View file

@ -1,206 +0,0 @@
// Copyright (C) 2017 Jonathan Müller <jonathanmueller.dev@gmail.com>
// This file is subject to the license terms in the LICENSE file
// found in the top-level directory of this distribution.
#ifndef CPPAST_TOKENIZER_HPP_INCLUDED
#define CPPAST_TOKENIZER_HPP_INCLUDED
#include <string>
#include <vector>
#include <cppast/cpp_token.hpp>
#include "raii_wrapper.hpp"
namespace cppast
{
namespace detail
{
class token
{
public:
explicit token(const CXTranslationUnit& tu_unit, const CXToken& token);
const cxstring& value() const noexcept
{
return value_;
}
const char* c_str() const noexcept
{
return value_.c_str();
}
CXTokenKind kind() const noexcept
{
return kind_;
}
private:
cxstring value_;
CXTokenKind kind_;
};
inline bool operator==(const token& tok, const char* str) noexcept
{
return tok.value() == str;
}
inline bool operator==(const char* str, const token& tok) noexcept
{
return str == tok.value();
}
inline bool operator!=(const token& tok, const char* str) noexcept
{
return !(tok == str);
}
inline bool operator!=(const char* str, const token& tok) noexcept
{
return !(str == tok);
}
using token_iterator = std::vector<token>::const_iterator;
class tokenizer
{
public:
explicit tokenizer(const CXTranslationUnit& tu, const CXFile& file,
const CXCursor& cur);
token_iterator begin() const noexcept
{
return tokens_.begin();
}
token_iterator end() const noexcept
{
return tokens_.end();
}
// if it returns true, the last token is ">>",
// but should haven been ">"
// only a problem for template parameters
bool unmunch() const noexcept
{
return unmunch_;
}
private:
std::vector<token> tokens_;
bool unmunch_;
};
class token_stream
{
public:
explicit token_stream(const tokenizer& tokenizer, const CXCursor& cur)
: cursor_(cur),
begin_(tokenizer.begin()),
cur_(begin_),
end_(tokenizer.end()),
unmunch_(tokenizer.unmunch())
{
}
const token& peek() const noexcept
{
if (done())
return *std::prev(end_);
return *cur_;
}
void bump() noexcept
{
if (cur_ != end_)
++cur_;
}
void bump_back() noexcept
{
if (cur_ != begin_)
--cur_;
}
const token& get() noexcept
{
auto& result = peek();
bump();
return result;
}
bool done() const noexcept
{
return cur_ == end_;
}
const CXCursor& cursor() const noexcept
{
return cursor_;
}
token_iterator begin() const noexcept
{
return begin_;
}
token_iterator cur() const noexcept
{
return cur_;
}
token_iterator end() const noexcept
{
return end_;
}
void set_cur(token_iterator iter) noexcept
{
cur_ = iter;
}
bool unmunch() const noexcept
{
return unmunch_;
}
private:
CXCursor cursor_;
token_iterator begin_, cur_, end_;
bool unmunch_;
};
// skips the next token
// asserts that it has the given string
void skip(token_stream& stream, const char* str);
// skips the next token if it has the given string
// if multi_token == true, str can consist of multiple tokens optionally separated by whitespace
bool skip_if(token_stream& stream, const char* str, bool multi_token = false);
// returns the location of the closing bracket
// the current token must be (,[,{ or <
// note: < might not work in the arguments of a template specialization
token_iterator find_closing_bracket(token_stream stream);
// skips brackets
// the current token must be (,[,{ or <
// note: < might not work in the arguments of a template specialization
void skip_brackets(token_stream& stream);
// skips an attribute
bool skip_attribute(token_stream& stream);
// converts a token range to a string
cpp_token_string to_string(token_stream& stream, token_iterator end);
// appends token to scope, if it is still valid
// else clears it
// note: does not consume the token if it is not valid,
// returns false in that case
bool append_scope(token_stream& stream, std::string& scope);
}
} // namespace cppast::detail
#endif // CPPAST_TOKENIZER_HPP_INCLUDED