jyapayne/cppast
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Add and use cpp_token_string for unexposed entities

Browse source
This commit is contained in:
Jonathan Müller 2017-10-11 14:09:50 +02:00
commit 0672e85186
29 changed files with 789 additions and 481 deletions
Download patch file Download diff file Expand all files Collapse all files

2
src/CMakeLists.txt
View file

@ -36,6 +36,7 @@ set(header
../include/cppast/cpp_storage_class_specifiers.hpp
../include/cppast/cpp_template.hpp
../include/cppast/cpp_template_parameter.hpp
../include/cppast/cpp_token.hpp
../include/cppast/cpp_type.hpp
../include/cppast/cpp_type_alias.hpp
../include/cppast/cpp_variable.hpp
@ -68,6 +69,7 @@ set(source
cpp_preprocessor.cpp
cpp_static_assert.cpp
cpp_template_parameter.cpp
cpp_token.cpp
cpp_type.cpp
cpp_type_alias.cpp
cpp_variable.cpp

73
src/code_generator.cpp
View file

@ -20,6 +20,7 @@
#include <cppast/cpp_preprocessor.hpp>
#include <cppast/cpp_static_assert.hpp>
#include <cppast/cpp_template_parameter.hpp>
#include <cppast/cpp_token.hpp>
#include <cppast/cpp_type_alias.hpp>
#include <cppast/cpp_variable.hpp>
#include <cppast/cpp_variable_template.hpp>
@ -343,8 +344,11 @@ namespace
if (spec.arguments_exposed())
detail::write_template_arguments(output, spec.arguments());
else if (!spec.unexposed_arguments().empty())
output << punctuation("<") << bracket_ws << token_seq(spec.unexposed_arguments())
<< bracket_ws << punctuation(">");
{
output << punctuation("<") << bracket_ws;
detail::write_token_string(output, spec.unexposed_arguments());
output << bracket_ws << punctuation(">");
}
}
void write_bases(code_generator& generator, code_generator::output& output, const cpp_class& c)
@ -537,7 +541,8 @@ namespace
output << keyword("noexcept") << punctuation("(") << bracket_ws;
// update check when expression gets exposed
if (cond.kind() == cpp_expression_kind::unexposed_t
&& static_cast<const cpp_unexposed_expression&>(cond).expression() == "false")
&& static_cast<const cpp_unexposed_expression&>(cond).expression().front().spelling
== "false")
output << keyword("false");
else if (output.options().is_set(code_generator::exclude_noexcept_condition))
output.excluded(base);
@ -1029,7 +1034,7 @@ namespace
code_generator::output output(type_safe::ref(generator), type_safe::ref(entity),
cur_access);
if (output)
output << token_seq(entity.spelling());
detail::write_token_string(output, entity.spelling());
return static_cast<bool>(output);
}
@ -1147,3 +1152,63 @@ void detail::write_template_arguments(
output << bracket_ws << punctuation(">");
}
}
void detail::write_token_string(code_generator::output& output, const cpp_token_string& tokens)
{
auto last_kind = cpp_token_kind::unknown;
for (auto& token : tokens)
{
switch (token.kind)
{
case cpp_token_kind::identifier:
if (last_kind == cpp_token_kind::identifier || last_kind == cpp_token_kind::keyword)
output << whitespace;
output << keyword(token.spelling);
break;
case cpp_token_kind::keyword:
if (last_kind == cpp_token_kind::identifier || last_kind == cpp_token_kind::keyword)
output << whitespace;
output << identifier(token.spelling);
if (token.spelling == "template")
output << operator_ws;
break;
case cpp_token_kind::literal:
// determine kind of literal
if (token.spelling.front() == '\"')
output << string_literal(token.spelling);
else if (token.spelling.find('.') != std::string::npos)
output << float_literal(token.spelling);
else
output << int_literal(token.spelling);
break;
case cpp_token_kind::punctuation:
if (token.spelling == ",")
output << comma;
// print brackets
// don't treat <> special as they might be operators...
else if (token.spelling == "(" || token.spelling == "[" || token.spelling == "{")
output << punctuation(token.spelling) << bracket_ws;
else if (token.spelling == ")" || token.spelling == "]" || token.spelling == "}")
output << bracket_ws << punctuation(token.spelling);
// operators that are always binary operators in all contexts
else if (token.spelling.back() == '=' // all assignment operators
|| token.spelling == "/" || token.spelling == "%" || token.spelling == "=="
|| token.spelling == "!=" || token.spelling == "<" || token.spelling == ">"
|| token.spelling == "<=" || token.spelling == ">=" || token.spelling == "&&"
|| token.spelling == "||" || token.spelling == "|" || token.spelling == "^"
|| token.spelling == "?")
output << operator_ws << punctuation(token.spelling) << operator_ws;
else
output << punctuation(token.spelling);
break;
case cpp_token_kind::unknown:
output << token_seq(token.spelling);
}
last_kind = token.kind;
}
}

4
src/cpp_entity.cpp
View file

@ -45,7 +45,7 @@ cpp_entity_kind cpp_unexposed_entity::kind() noexcept
std::unique_ptr<cpp_entity> cpp_unexposed_entity::build(const cpp_entity_index& index,
cpp_entity_id id, std::string name,
std::string spelling)
cpp_token_string spelling)
{
std::unique_ptr<cpp_entity> result(
new cpp_unexposed_entity(std::move(name), std::move(spelling)));
@ -53,7 +53,7 @@ std::unique_ptr<cpp_entity> cpp_unexposed_entity::build(const cpp_entity_index&
return result;
}
std::unique_ptr<cpp_entity> cpp_unexposed_entity::build(std::string spelling)
std::unique_ptr<cpp_entity> cpp_unexposed_entity::build(cpp_token_string spelling)
{
return std::unique_ptr<cpp_entity>(new cpp_unexposed_entity("", std::move(spelling)));
}

2
src/cpp_expression.cpp
View file

@ -75,7 +75,7 @@ namespace
void write_unexposed(code_generator::output& output, const cpp_unexposed_expression& expr)
{
output << token_seq(expr.expression());
detail::write_token_string(output, expr.expression());
}
}

45
src/cpp_token.cpp Normal file
View file

@ -0,0 +1,45 @@
// Copyright (C) 2017 Jonathan Müller <jonathanmueller.dev@gmail.com>
// This file is subject to the license terms in the LICENSE file
// found in the top-level directory of this distribution.
#include <cppast/cpp_token.hpp>
#include <algorithm>
#include <cctype>
#include <cppast/detail/assert.hpp>
using namespace cppast;
void cpp_token_string::builder::unmunch()
{
DEBUG_ASSERT(!tokens_.empty() && tokens_.back().spelling == ">>", detail::assert_handler{});
tokens_.back().spelling = ">";
}
namespace
{
bool is_identifier(char c)
{
return std::isalnum(c) || c == '_';
}
}
std::string cpp_token_string::as_string() const
{
std::string result;
for (auto& token : tokens_)
{
DEBUG_ASSERT(!token.spelling.empty(), detail::assert_handler{});
if (!result.empty() && is_identifier(result.back()) && is_identifier(token.spelling[0u]))
result += ' ';
result += token.spelling;
}
return result;
}
bool cppast::operator==(const cpp_token_string& lhs, const cpp_token_string& rhs)
{
if (lhs.tokens_.size() != rhs.tokens_.size())
return false;
return std::equal(lhs.tokens_.begin(), lhs.tokens_.end(), rhs.tokens_.begin());
}

2
src/libclang/class_parser.cpp
View file

@ -81,7 +81,7 @@ namespace
detail::skip(stream, "virtual");
detail::skip_if(stream, to_string(access));
auto name = detail::to_string(stream, stream.end());
auto name = detail::to_string(stream, stream.end()).as_string();
auto type = detail::parse_type(context, class_cur, clang_getCursorType(cur));
builder.base_class(std::move(name), std::move(type), access, is_virtual);

9
src/libclang/expression_parser.cpp
View file

@ -19,7 +19,7 @@ std::unique_ptr<cpp_expression> detail::parse_expression(const detail::parse_con
auto type = parse_type(context, cur, clang_getCursorType(cur));
auto expr = to_string(stream, stream.end());
if (kind == CXCursor_CallExpr && (expr.empty() || expr.back() != ')'))
if (kind == CXCursor_CallExpr && (expr.empty() || expr.back().spelling != ")"))
{
// we have a call expression that doesn't end in a closing parentheses
// this means default constructor, don't parse it at all
@ -30,7 +30,7 @@ std::unique_ptr<cpp_expression> detail::parse_expression(const detail::parse_con
|| kind == CXCursor_FloatingLiteral || kind == CXCursor_ImaginaryLiteral
|| kind == CXCursor_IntegerLiteral || kind == CXCursor_StringLiteral
|| kind == CXCursor_CXXBoolLiteralExpr || kind == CXCursor_CXXNullPtrLiteralExpr)
return cpp_literal_expression::build(std::move(type), std::move(expr));
return cpp_literal_expression::build(std::move(type), expr.as_string());
else
return cpp_unexposed_expression::build(std::move(type), std::move(expr));
}
@ -42,8 +42,7 @@ std::unique_ptr<cpp_expression> detail::parse_raw_expression(const parse_context
{
if (stream.done())
return nullptr;
auto expr = to_string(stream, end);
if (!expr.empty() && expr.back() == ';')
expr.pop_back();
auto expr = to_string(stream, std::prev(end)->value() == ";" ? std::prev(end) : end);
return cpp_unexposed_expression::build(std::move(type), std::move(expr));
}

2
src/libclang/function_parser.cpp
View file

@ -705,7 +705,7 @@ std::unique_ptr<cpp_entity> detail::parse_cpp_conversion_op(const detail::parse_
// read the type
stream.set_cur(type_start);
auto type_spelling = detail::to_string(stream, type_end);
auto type_spelling = detail::to_string(stream, type_end).as_string();
// parse arguments again
detail::skip(stream, "(");

2
src/libclang/template_parser.cpp
View file

@ -279,7 +279,7 @@ namespace
b.add_unexposed_arguments(std::move(args));
}
else
b.add_unexposed_arguments("");
b.add_unexposed_arguments(cpp_token_string::builder().finish());
}
}

40
src/libclang/tokenizer.cpp
View file

@ -411,29 +411,41 @@ bool detail::skip_attribute(detail::token_stream& stream)
namespace
{
bool is_identifier(char c)
cpp_token_kind get_kind(CXTokenKind kind)
{
return std::isalnum(c) || c == '_';
switch (kind)
{
case CXToken_Punctuation:
return cpp_token_kind::punctuation;
case CXToken_Keyword:
return cpp_token_kind::keyword;
case CXToken_Identifier:
return cpp_token_kind::identifier;
case CXToken_Literal:
return cpp_token_kind::literal;
case CXToken_Comment:
break;
}
DEBUG_UNREACHABLE(detail::assert_handler{});
return cpp_token_kind ::literal;
}
}
std::string detail::to_string(token_stream& stream, token_iterator end)
cpp_token_string detail::to_string(token_stream& stream, token_iterator end)
{
std::string result;
cpp_token_string::builder builder;
while (stream.cur() != end)
{
auto& token = stream.get();
if (!result.empty() && is_identifier(result.back()) && is_identifier(token.value()[0u]))
result += ' ';
result += token.c_str();
builder.add_token(cpp_token(get_kind(token.kind()), token.c_str()));
}
if (stream.unmunch())
{
DEBUG_ASSERT(!result.empty() && result.back() == '>', detail::assert_handler{});
result.pop_back();
DEBUG_ASSERT(!result.empty() && result.back() == '>', detail::assert_handler{});
}
return result;
builder.unmunch();
return builder.finish();
}
bool detail::append_scope(detail::token_stream& stream, std::string& scope)
@ -455,7 +467,7 @@ bool detail::append_scope(detail::token_stream& stream, std::string& scope)
else if (stream.peek() == "<")
{
auto iter = detail::find_closing_bracket(stream);
scope += detail::to_string(stream, iter);
scope += detail::to_string(stream, iter).as_string();
if (!detail::skip_if(stream, ">>"))
detail::skip(stream, ">");
scope += ">";

6
src/libclang/tokenizer.hpp
View file

@ -8,6 +8,8 @@
#include <string>
#include <vector>
#include <cppast/cpp_token.hpp>
#include "raii_wrapper.hpp"
namespace cppast
@ -190,8 +192,8 @@ namespace cppast
// skips an attribute
bool skip_attribute(token_stream& stream);
// converts a token range to a string, adding whitespace where necessary
std::string to_string(token_stream& stream, token_iterator end);
// converts a token range to a string
cpp_token_string to_string(token_stream& stream, token_iterator end);
// appends token to scope, if it is still valid
// else clears it

10
src/libclang/type_parser.cpp
View file

@ -246,8 +246,9 @@ namespace
return size_expr.empty() ?
nullptr :
cpp_unexposed_expression::build(cpp_builtin_type::build(cpp_ulonglong),
std::string(size_expr.rbegin(),
size_expr.rend()));
cpp_token_string::from_string(
std::string(size_expr.rbegin(),
size_expr.rend())));
}
std::unique_ptr<cpp_type> try_parse_array_type(const detail::parse_context& context,
@ -486,7 +487,8 @@ namespace
spelling.pop_back();
return cpp_decltype_type::build(
cpp_unexposed_expression::build(cpp_unexposed_type::build("<decltype>"), spelling));
cpp_unexposed_expression::build(cpp_unexposed_type::build("<decltype>"),
cpp_token_string::from_string(spelling)));
});
}
@ -734,7 +736,7 @@ std::unique_ptr<cpp_type> detail::parse_raw_type(const detail::parse_context&,
detail::token_iterator end)
{
auto result = detail::to_string(stream, end);
return cpp_unexposed_type::build(std::move(result));
return cpp_unexposed_type::build(result.as_string());
}
std::unique_ptr<cpp_entity> detail::parse_cpp_type_alias(const detail::parse_context& context,