From fd449dde4a7db53969e295e8c920cedb5c6ec06c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonathan=20M=C3=BCller?= Date: Sun, 11 Jun 2017 22:16:56 +0200 Subject: [PATCH 01/20] Store full path of header as well --- include/cppast/cpp_preprocessor.hpp | 21 ++- src/libclang/libclang_parser.cpp | 49 +++++-- src/libclang/parse_functions.cpp | 5 + src/libclang/preprocessor.cpp | 206 +++++++++++++--------------- src/libclang/preprocessor.hpp | 16 ++- test/cpp_preprocessor.cpp | 3 + 6 files changed, 169 insertions(+), 131 deletions(-) diff --git a/include/cppast/cpp_preprocessor.hpp b/include/cppast/cpp_preprocessor.hpp index 53a9b77..bcdb62f 100644 --- a/include/cppast/cpp_preprocessor.hpp +++ b/include/cppast/cpp_preprocessor.hpp @@ -80,9 +80,11 @@ namespace cppast /// \notes It is not meant to be registered in the [cppast::cpp_entity_index](), /// as no other [cppast::cpp_entity]() can refer to it. static std::unique_ptr build(const cpp_file_ref& target, - cpp_include_kind kind) + cpp_include_kind kind, + std::string full_path) { - return std::unique_ptr(new cpp_include_directive(target, kind)); + return std::unique_ptr( + new cpp_include_directive(target, kind, std::move(full_path))); } /// \returns A reference to the [cppast::cpp_file]() it includes. @@ -97,17 +99,28 @@ namespace cppast return kind_; } + /// \returns The full path of the included file. + const std::string& full_path() const noexcept + { + return full_path_; + } + private: cpp_entity_kind do_get_entity_kind() const noexcept override; - cpp_include_directive(const cpp_file_ref& target, cpp_include_kind kind) - : cpp_entity(target.name()), target_(target.id()[0u]), kind_(kind) + cpp_include_directive(const cpp_file_ref& target, cpp_include_kind kind, + std::string full_path) + : cpp_entity(target.name()), + target_(target.id()[0u]), + kind_(kind), + full_path_(std::move(full_path)) { DEBUG_ASSERT(!target.is_overloaded(), detail::precondition_error_handler{}); } cpp_entity_id target_; cpp_include_kind kind_; + std::string full_path_; }; } // namespace cppast diff --git a/src/libclang/libclang_parser.cpp b/src/libclang/libclang_parser.cpp index 1174c09..4718e97 100644 --- a/src/libclang/libclang_parser.cpp +++ b/src/libclang/libclang_parser.cpp @@ -205,14 +205,16 @@ namespace auto args = get_arguments(config); CXTranslationUnit tu; - auto error = + auto flags = CXTranslationUnit_Incomplete | CXTranslationUnit_KeepGoing; + if (detail::libclang_compile_config_access::clang_version(config) >= 40000) + flags |= CXTranslationUnit_DetailedPreprocessingRecord; + + auto error = clang_parseTranslationUnit2(idx.get(), path, // index and path args.data(), static_cast(args.size()), // arguments (ptr + size) &file, 1, // unsaved files (ptr + size) - CXTranslationUnit_Incomplete - | CXTranslationUnit_KeepGoing, // flags - &tu); + unsigned(flags), &tu); if (error != CXError_Success) { switch (error) @@ -259,25 +261,42 @@ std::unique_ptr libclang_parser::do_parse(const cpp_entity_index& idx, auto file = clang_getFile(tu.get(), path.c_str()); cpp_file::builder builder(path); - auto preprocessed_iter = preprocessed.entities.begin(); + auto macro_iter = preprocessed.macros.begin(); + auto include_iter = preprocessed.includes.begin(); // convert entity hierachies detail::parse_context context{tu.get(), file, type_safe::ref(logger()), type_safe::ref(idx), detail::comment_context(preprocessed.comments)}; detail::visit_tu(tu, path.c_str(), [&](const CXCursor& cur) { - // add macro if needed - for (auto line = get_line_no(cur); - preprocessed_iter != preprocessed.entities.end() && preprocessed_iter->line <= line; - ++preprocessed_iter) - builder.add_child(std::move(preprocessed_iter->entity)); + if (clang_getCursorKind(cur) == CXCursor_InclusionDirective) + { + DEBUG_ASSERT(include_iter != preprocessed.includes.end() + && get_line_no(cur) >= include_iter->line, + detail::assert_handler{}); - auto entity = detail::parse_entity(context, cur); - if (entity) - builder.add_child(std::move(entity)); + auto include = + cpp_include_directive::build(std::move(include_iter->file), include_iter->kind, + detail::get_cursor_name(cur).c_str()); + context.comments.match(*include, include_iter->line); + builder.add_child(std::move(include)); + + ++include_iter; + } + else if (clang_getCursorKind(cur) != CXCursor_MacroDefinition) + { + // add macro if needed + for (auto line = get_line_no(cur); + macro_iter != preprocessed.macros.end() && macro_iter->line <= line; ++macro_iter) + builder.add_child(std::move(macro_iter->macro)); + + auto entity = detail::parse_entity(context, cur); + if (entity) + builder.add_child(std::move(entity)); + } }); - for (; preprocessed_iter != preprocessed.entities.end(); ++preprocessed_iter) - builder.add_child(std::move(preprocessed_iter->entity)); + for (; macro_iter != preprocessed.macros.end(); ++macro_iter) + builder.add_child(std::move(macro_iter->macro)); for (auto& c : preprocessed.comments) { diff --git a/src/libclang/parse_functions.cpp b/src/libclang/parse_functions.cpp index 3e0af7e..6a5e6a4 100644 --- a/src/libclang/parse_functions.cpp +++ b/src/libclang/parse_functions.cpp @@ -104,6 +104,11 @@ std::unique_ptr detail::parse_entity(const detail::parse_context& co return entity; break; + case CXCursor_MacroDefinition: + case CXCursor_InclusionDirective: + DEBUG_UNREACHABLE(detail::assert_handler{}, "handle preprocessor in parser callback"); + break; + case CXCursor_Namespace: return parse_cpp_namespace(context, cur); case CXCursor_NamespaceAlias: diff --git a/src/libclang/preprocessor.cpp b/src/libclang/preprocessor.cpp index 1a83dd5..4559307 100644 --- a/src/libclang/preprocessor.cpp +++ b/src/libclang/preprocessor.cpp @@ -268,6 +268,89 @@ namespace p.skip(std::strlen(str)); } + void skip_spaces(position& p, bool bump = false) + { + while (starts_with(p, " ")) + if (bump) + p.bump(); + else + p.skip(); + } + + struct linemarker + { + std::string file; + unsigned line; + enum + { + line_directive, // no change in file + enter_new, // open a new file + enter_old, // return to an old file + } flag = line_directive; + bool is_system = false; + }; + + ts::optional parse_linemarker(position& p) + { + // format (at new line): # "" + // flag 1: enter_new + // flag 2: enter_old + // flag 3: system file + // flag 4: ignored + if (!p.was_newl() || !starts_with(p, "#")) + return ts::nullopt; + p.skip(); + DEBUG_ASSERT(!starts_with(p, "define") && !starts_with(p, "undef") + && !starts_with(p, "pragma"), + detail::assert_handler{}, "handle macros first"); + + linemarker result; + + std::string line; + for (skip_spaces(p); std::isdigit(*p.ptr()); p.skip()) + line += *p.ptr(); + result.line = unsigned(std::stoi(line)); + + skip_spaces(p); + DEBUG_ASSERT(*p.ptr() == '"', detail::assert_handler{}); + p.skip(); + + std::string file_name; + for (; !starts_with(p, "\""); p.skip()) + file_name += *p.ptr(); + p.skip(); + result.file = std::move(file_name); + + for (; !starts_with(p, "\n"); p.skip()) + { + skip_spaces(p); + + switch (*p.ptr()) + { + case '1': + DEBUG_ASSERT(result.flag == linemarker::line_directive, detail::assert_handler{}); + result.flag = linemarker::enter_new; + break; + case '2': + DEBUG_ASSERT(result.flag == linemarker::line_directive, detail::assert_handler{}); + result.flag = linemarker::enter_old; + break; + case '3': + result.is_system = true; + break; + case '4': + break; // ignored + + default: + DEBUG_UNREACHABLE(detail::assert_handler{}, "invalid line marker"); + break; + } + } + p.skip(); + + return result; + } + detail::pp_doc_comment parse_c_doc_comment(position& p) { detail::pp_doc_comment result; @@ -423,15 +506,6 @@ namespace return true; } - void skip_spaces(position& p, bool bump = false) - { - while (starts_with(p, " ")) - if (bump) - p.bump(); - else - p.skip(); - } - std::unique_ptr parse_macro(position& p, detail::preprocessor_output& output, bool in_main_file) @@ -504,15 +578,13 @@ namespace return result; } - std::unique_ptr parse_include(position& p, - detail::preprocessor_output& output, - bool in_main_file) + type_safe::optional parse_include(position& p, bool in_main_file) { // format (at new line, literal <>): #include // or: #include "filename" // note: don't write include back if (!p.was_newl() || !starts_with(p, "#include")) - return nullptr; + return type_safe::nullopt; p.skip(std::strlen("#include")); if (starts_with(p, "_next")) p.skip(std::strlen("_next")); @@ -544,16 +616,10 @@ namespace // don't skip newline if (!in_main_file) - return nullptr; + return type_safe::nullopt; - auto result = cpp_include_directive::build(cpp_file_ref(cpp_entity_id(filename), filename), - include_kind); - if (!output.comments.empty() && output.comments.back().matches(*result, p.cur_line())) - { - result->set_comment(std::move(output.comments.back().comment)); - output.comments.pop_back(); - } - return result; + return detail::pp_include{cpp_file_ref(cpp_entity_id(filename), filename), include_kind, + p.cur_line()}; } bool bump_pragma(position& p) @@ -568,80 +634,6 @@ namespace return true; } - - struct linemarker - { - std::string file; - unsigned line; - enum - { - line_directive, // no change in file - enter_new, // open a new file - enter_old, // return to an old file - } flag = line_directive; - bool is_system = false; - }; - - ts::optional parse_linemarker(position& p) - { - // format (at new line): # "" - // flag 1: enter_new - // flag 2: enter_old - // flag 3: system file - // flag 4: ignored - if (!p.was_newl() || !starts_with(p, "#")) - return ts::nullopt; - p.skip(); - DEBUG_ASSERT(!starts_with(p, "define") && !starts_with(p, "undef") - && !starts_with(p, "pragma"), - detail::assert_handler{}, "handle macros first"); - - linemarker result; - - std::string line; - for (skip_spaces(p); std::isdigit(*p.ptr()); p.skip()) - line += *p.ptr(); - result.line = unsigned(std::stoi(line)); - - skip_spaces(p); - DEBUG_ASSERT(*p.ptr() == '"', detail::assert_handler{}); - p.skip(); - - std::string file_name; - for (; !starts_with(p, "\""); p.skip()) - file_name += *p.ptr(); - p.skip(); - result.file = std::move(file_name); - - for (; !starts_with(p, "\n"); p.skip()) - { - skip_spaces(p); - - switch (*p.ptr()) - { - case '1': - DEBUG_ASSERT(result.flag == linemarker::line_directive, detail::assert_handler{}); - result.flag = linemarker::enter_new; - break; - case '2': - DEBUG_ASSERT(result.flag == linemarker::line_directive, detail::assert_handler{}); - result.flag = linemarker::enter_old; - break; - case '3': - result.is_system = true; - break; - case '4': - break; // ignored - - default: - DEBUG_UNREACHABLE(detail::assert_handler{}, "invalid line marker"); - break; - } - } - p.skip(); - - return result; - } } detail::preprocessor_output detail::preprocess(const libclang_compile_config& config, @@ -682,7 +674,7 @@ detail::preprocessor_output detail::preprocess(const libclang_compile_config& co severity::debug}); } - result.entities.push_back({std::move(macro), p.cur_line()}); + result.macros.push_back({std::move(macro), p.cur_line()}); } else if (auto undef = parse_undef(p)) { @@ -695,26 +687,24 @@ detail::preprocessor_output detail::preprocess(const libclang_compile_config& co diagnostic{std::move(message), source_location::make_file(path), severity::debug}); } - result.entities - .erase(std::remove_if(result.entities.begin(), result.entities.end(), - [&](const pp_entity& e) { - return e.entity->kind() - == cpp_entity_kind::macro_definition_t - && e.entity->name() == undef.value(); - }), - result.entities.end()); + result.macros.erase(std::remove_if(result.macros.begin(), result.macros.end(), + [&](const pp_macro& e) { + return e.macro->name() == undef.value(); + }), + result.macros.end()); } } - else if (auto include = parse_include(p, result, file_depth == 0u)) + else if (auto include = parse_include(p, file_depth == 0u)) { if (logger.is_verbose()) { - auto message = detail::format("parsing include '", include->name(), "'"); + auto message = + detail::format("parsing include '", include.value().file.name(), "'"); logger.log("preprocessor", diagnostic{std::move(message), source_location::make_file(path), severity::debug}); } - result.entities.push_back({std::move(include), p.cur_line()}); + result.includes.push_back(std::move(include.value())); } else if (bump_pragma(p)) continue; diff --git a/src/libclang/preprocessor.hpp b/src/libclang/preprocessor.hpp index 130ab54..6fa3640 100644 --- a/src/libclang/preprocessor.hpp +++ b/src/libclang/preprocessor.hpp @@ -12,10 +12,17 @@ namespace cppast { namespace detail { - struct pp_entity + struct pp_macro { - std::unique_ptr entity; - unsigned line; + std::unique_ptr macro; + unsigned line; + }; + + struct pp_include + { + cpp_file_ref file; + cpp_include_kind kind; + unsigned line; }; struct pp_doc_comment @@ -35,7 +42,8 @@ namespace cppast struct preprocessor_output { std::string source; - std::vector entities; + std::vector includes; + std::vector macros; std::vector comments; }; diff --git a/test/cpp_preprocessor.cpp b/test/cpp_preprocessor.cpp index 8c8945d..a96188f 100644 --- a/test/cpp_preprocessor.cpp +++ b/test/cpp_preprocessor.cpp @@ -107,12 +107,14 @@ TEST_CASE("cpp_include_directive", "[!hide][clang4]") REQUIRE(include.target().name() == include.name()); REQUIRE(include.include_kind() == cppast::cpp_include_kind::system); REQUIRE(include.target().get(idx).empty()); + REQUIRE_THAT(include.full_path(), Catch::EndsWith("iostream")); } else if (include.name() == "cpp_include_directive-header.hpp") { REQUIRE(include.target().name() == include.name()); REQUIRE(include.include_kind() == cppast::cpp_include_kind::local); REQUIRE(include.target().get(idx).empty()); + REQUIRE(include.full_path() == "./cpp_include_directive-header.hpp"); } else REQUIRE(false); @@ -126,6 +128,7 @@ TEST_CASE("cpp_include_directive", "[!hide][clang4]") REQUIRE(include.include_kind() == cppast::cpp_include_kind::local); REQUIRE( equal_ref(idx, include.target(), cpp_file_ref(cpp_entity_id(""), "header_a.hpp"))); + REQUIRE(include.full_path() == "./header_a.hpp"); } else REQUIRE(false); From d6e993a7b2ccedcc7f05bf0535a451fa2b94b753 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonathan=20M=C3=BCller?= Date: Wed, 14 Jun 2017 22:38:14 +0200 Subject: [PATCH 02/20] Optimize preprocessing string searching/comparing --- src/libclang/preprocessor.cpp | 33 ++++++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/src/libclang/preprocessor.cpp b/src/libclang/preprocessor.cpp index 4559307..e80ab64 100644 --- a/src/libclang/preprocessor.cpp +++ b/src/libclang/preprocessor.cpp @@ -210,8 +210,13 @@ namespace void bump(std::size_t offset) noexcept { - for (std::size_t i = 0u; i != offset; ++i) - bump(); + if (write_ == true) + { + for (std::size_t i = 0u; i != offset; ++i) + bump(); + } + else + skip(offset); } // no write, no newline detection @@ -257,14 +262,20 @@ namespace ts::flag write_; }; - bool starts_with(const position& p, const char* str) + bool starts_with(const position& p, const char* str, std::size_t len) { - return std::strncmp(p.ptr(), str, std::strlen(str)) == 0; + return std::strncmp(p.ptr(), str, len) == 0; + } + + template + bool starts_with(const position& p, const char (&str)[N]) + { + return std::strncmp(p.ptr(), str, N - 1) == 0; } void skip(position& p, const char* str) { - DEBUG_ASSERT(starts_with(p, str), detail::assert_handler{}); + DEBUG_ASSERT(starts_with(p, str, std::strlen(str)), detail::assert_handler{}); p.skip(std::strlen(str)); } @@ -498,9 +509,8 @@ namespace } else { - while (!starts_with(p, "\n")) - p.skip(); - // don't skip newline + auto newline = std::strchr(p.ptr(), '\n'); + p.skip(std::size_t(newline - p.ptr())); // don't skip newline } return true; @@ -609,7 +619,8 @@ namespace std::string filename; for (; !starts_with(p, "\"") && !starts_with(p, ">"); p.skip()) filename += *p.ptr(); - DEBUG_ASSERT(starts_with(p, end_str), detail::assert_handler{}, "bad termination"); + DEBUG_ASSERT(starts_with(p, end_str, std::strlen(end_str)), detail::assert_handler{}, + "bad termination"); p.skip(); skip(p, " /* clang -E -dI */"); DEBUG_ASSERT(starts_with(p, "\n"), detail::assert_handler{}); @@ -648,6 +659,10 @@ detail::preprocessor_output detail::preprocess(const libclang_compile_config& co ts::flag in_string(false), in_char(false); while (p) { + auto next = std::strpbrk(p.ptr(), "\"'#/"); // look for ", ', # or / + if (next && next > p.ptr()) + p.bump(std::size_t(next - p.ptr() - 1)); // subtract one to get before that character + if (starts_with(p, "\\\"")) // starts with \" p.bump(2u); else if (starts_with(p, "\\'")) // starts with \' From 084d8050b97a95ddf48606352aca57884d1f0083 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonathan=20M=C3=BCller?= Date: Tue, 6 Jun 2017 17:33:34 +0200 Subject: [PATCH 03/20] Allow attribute after namespace name --- src/libclang/namespace_parser.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/libclang/namespace_parser.cpp b/src/libclang/namespace_parser.cpp index 12b3e55..77f4615 100644 --- a/src/libclang/namespace_parser.cpp +++ b/src/libclang/namespace_parser.cpp @@ -32,6 +32,7 @@ namespace return cpp_namespace::builder("", is_inline); auto& name = stream.get().value(); + skip_attribute(stream); skip(stream, "{"); return cpp_namespace::builder(name.c_str(), is_inline); } From b0cd79583696180154aad88fa8ea7e4691db84a4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonathan=20M=C3=BCller?= Date: Tue, 6 Jun 2017 17:34:52 +0200 Subject: [PATCH 04/20] Hide wrong warnings if header files are parsed directly --- src/libclang/preprocessor.cpp | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/libclang/preprocessor.cpp b/src/libclang/preprocessor.cpp index e80ab64..69292bb 100644 --- a/src/libclang/preprocessor.cpp +++ b/src/libclang/preprocessor.cpp @@ -41,8 +41,9 @@ namespace // -fno-show-column: don't show the column number // -fdiagnostics-format msvc: use easier to parse MSVC format flags += " -fno-caret-diagnostics -fno-show-column -fdiagnostics-format=msvc"; - // -Wno-pragma-once-outside-header: hide wrong warning - flags += " -Wno-pragma-once-outside-header"; + // -Wno-*: hide wrong warnings if header file is directly parsed + flags += " -Wno-pragma-once-outside-header -Wno-pragma-system-header-outside-header " + "-Wno-include-next-outside-header"; std::string cmd(detail::libclang_compile_config_access::clang_binary(c) + " " + std::move(flags) + " "); From 13d9ac614712f124d40f37fb2b31b8dc8deb4063 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonathan=20M=C3=BCller?= Date: Tue, 6 Jun 2017 17:36:27 +0200 Subject: [PATCH 05/20] Skip attribute cursors --- src/libclang/enum_parser.cpp | 7 ++++++- src/libclang/parse_functions.cpp | 32 +++++++++++++++++++------------- 2 files changed, 25 insertions(+), 14 deletions(-) diff --git a/src/libclang/enum_parser.cpp b/src/libclang/enum_parser.cpp index b65ceb5..973d4a1 100644 --- a/src/libclang/enum_parser.cpp +++ b/src/libclang/enum_parser.cpp @@ -14,6 +14,9 @@ namespace std::unique_ptr parse_enum_value(const detail::parse_context& context, const CXCursor& cur) { + if (clang_isAttribute(clang_getCursorKind(cur))) + return nullptr; + DEBUG_ASSERT(cur.kind == CXCursor_EnumConstantDecl, detail::parse_error_handler{}, cur, "unexpected child cursor of enum"); @@ -84,8 +87,10 @@ std::unique_ptr detail::parse_cpp_enum(const detail::parse_context& { auto entity = parse_enum_value(context, child); if (entity) + { context.comments.match(*entity, child); - builder.add_value(std::move(entity)); + builder.add_value(std::move(entity)); + } } catch (parse_error& ex) { diff --git a/src/libclang/parse_functions.cpp b/src/libclang/parse_functions.cpp index 6a5e6a4..0633708 100644 --- a/src/libclang/parse_functions.cpp +++ b/src/libclang/parse_functions.cpp @@ -177,21 +177,27 @@ std::unique_ptr detail::parse_entity(const detail::parse_context& co break; } - auto msg = detail::format("unhandled cursor of kind '", - detail::get_cursor_kind_spelling(cur).c_str(), "'"); - context.logger->log("libclang parser", - diagnostic{std::move(msg), detail::make_location(cur), severity::warning}); + if (!clang_isAttribute(clang_getCursorKind(cur))) + { + auto msg = detail::format("unhandled cursor of kind '", + detail::get_cursor_kind_spelling(cur).c_str(), "'"); + context.logger->log("libclang parser", + diagnostic{std::move(msg), detail::make_location(cur), + severity::warning}); - // build unexposed entity - auto name = detail::get_cursor_name(cur); - detail::tokenizer tokenizer(context.tu, context.file, cur); - detail::token_stream stream(tokenizer, cur); - auto spelling = detail::to_string(stream, stream.end()); - if (name.empty()) - return cpp_unexposed_entity::build(std::move(spelling)); + // build unexposed entity + auto name = detail::get_cursor_name(cur); + detail::tokenizer tokenizer(context.tu, context.file, cur); + detail::token_stream stream(tokenizer, cur); + auto spelling = detail::to_string(stream, stream.end()); + if (name.empty()) + return cpp_unexposed_entity::build(std::move(spelling)); + else + return cpp_unexposed_entity::build(*context.idx, detail::get_entity_id(cur), + name.c_str(), std::move(spelling)); + } else - return cpp_unexposed_entity::build(*context.idx, detail::get_entity_id(cur), name.c_str(), - std::move(spelling)); + return nullptr; } catch (parse_error& ex) { From bf35018bdf692eb2f09c10428104d4b34b840fd0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonathan=20M=C3=BCller?= Date: Thu, 15 Jun 2017 20:23:03 +0200 Subject: [PATCH 06/20] Fix libclang issue with USR, partial class template specialization and ref qualifiers --- src/libclang/parse_functions.cpp | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/libclang/parse_functions.cpp b/src/libclang/parse_functions.cpp index 0633708..d17aa6f 100644 --- a/src/libclang/parse_functions.cpp +++ b/src/libclang/parse_functions.cpp @@ -23,6 +23,15 @@ cpp_entity_id detail::get_entity_id(const CXCursor& cur) cxstring type_spelling(clang_getTypeSpelling(clang_getCursorResultType(cur))); return cpp_entity_id(std::string(usr.c_str()) + type_spelling.c_str()); } + else if (clang_getCursorKind(cur) == CXCursor_ClassTemplatePartialSpecialization) + { + // libclang issue: templ vs templ + // but identical USR + // same workaround: combine display name with usr + // (and hope this prevents all collisions...) + cxstring display_name(clang_getCursorDisplayName(cur)); + return cpp_entity_id(std::string(usr.c_str()) + display_name.c_str()); + } else return cpp_entity_id(usr.c_str()); } From 43e50599e730a8c83948d807e58d3b97a51b7faa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonathan=20M=C3=BCller?= Date: Thu, 15 Jun 2017 20:31:31 +0200 Subject: [PATCH 07/20] Make using declaration parser more robust --- src/libclang/namespace_parser.cpp | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/src/libclang/namespace_parser.cpp b/src/libclang/namespace_parser.cpp index 77f4615..f3b8797 100644 --- a/src/libclang/namespace_parser.cpp +++ b/src/libclang/namespace_parser.cpp @@ -5,6 +5,7 @@ #include "parse_functions.hpp" #include +#include #include "libclang_visitor.hpp" @@ -127,13 +128,14 @@ std::unique_ptr detail::parse_cpp_using_directive(const detail::pars namespace { - std::vector parse_entity_target_cursor(const CXCursor& cur) + cpp_entity_ref parse_entity_target_cursor(const CXCursor& cur, std::string name) { - std::vector result; + type_safe::deferred_construction result; detail::visit_children(cur, [&](const CXCursor& child) { - if (!result.empty()) + if (result) return; + switch (clang_getCursorKind(child)) { case CXCursor_TypeRef: @@ -143,17 +145,21 @@ namespace case CXCursor_DeclRefExpr: { auto referenced = clang_getCursorReferenced(child); - result.push_back(detail::get_entity_id(referenced)); + result = cpp_entity_ref(detail::get_entity_id(referenced), + std::move(name)); break; } case CXCursor_OverloadedDeclRef: { auto size = clang_getNumOverloadedDecls(child); - DEBUG_ASSERT(size >= 1u, detail::assert_handler{}); + DEBUG_ASSERT(size >= 1u, detail::parse_error_handler{}, cur, + "no target for using declaration"); + std::vector ids; for (auto i = 0u; i != size; ++i) - result.push_back(detail::get_entity_id( + ids.push_back(detail::get_entity_id( clang_getOverloadedDecl(child, i))); + result = cpp_entity_ref(std::move(ids), std::move(name)); break; } @@ -167,7 +173,7 @@ namespace }, true); - return result; + return result.value(); } } @@ -187,7 +193,7 @@ std::unique_ptr detail::parse_cpp_using_declaration( while (!stream.done() && !detail::skip_if(stream, ";")) target_name += stream.get().c_str(); - auto target = cpp_entity_ref(parse_entity_target_cursor(cur), std::move(target_name)); + auto target = parse_entity_target_cursor(cur, std::move(target_name)); auto result = cpp_using_declaration::build(std::move(target)); context.comments.match(*result, cur); return std::move(result); From e6271f4c0d2cba65e205edc8082546327c576458 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonathan=20M=C3=BCller?= Date: Thu, 15 Jun 2017 20:36:56 +0200 Subject: [PATCH 08/20] Silently ignore _Complex types --- src/libclang/type_parser.cpp | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/libclang/type_parser.cpp b/src/libclang/type_parser.cpp index 91dbcbb..5de1d98 100644 --- a/src/libclang/type_parser.cpp +++ b/src/libclang/type_parser.cpp @@ -468,7 +468,6 @@ namespace case CXType_ObjCId: case CXType_ObjCClass: case CXType_ObjCSel: - case CXType_Complex: case CXType_BlockPointer: case CXType_Vector: case CXType_ObjCInterface: @@ -498,6 +497,8 @@ namespace else if (auto ptype = try_parse_template_parameter_type(context, cur, type)) // template parameter type is unexposed return ptype; + // fallthrough + case CXType_Complex: return cpp_unexposed_type::build(get_type_spelling(type).c_str()); case CXType_Void: From b5a5e42955da16453acbfab6281df54a45466382 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonathan=20M=C3=BCller?= Date: Thu, 15 Jun 2017 21:08:36 +0200 Subject: [PATCH 09/20] Fix out-of-class destructor definition --- src/libclang/function_parser.cpp | 9 ++++++--- test/cpp_member_function.cpp | 13 ++++++++++--- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/src/libclang/function_parser.cpp b/src/libclang/function_parser.cpp index 80c1973..b8571af 100644 --- a/src/libclang/function_parser.cpp +++ b/src/libclang/function_parser.cpp @@ -4,6 +4,7 @@ #include #include +#include #include "libclang_visitor.hpp" #include "parse_functions.hpp" @@ -657,13 +658,15 @@ std::unique_ptr detail::parse_cpp_destructor(const detail::parse_con detail::tokenizer tokenizer(context.tu, context.file, cur); detail::token_stream stream(tokenizer, cur); - auto is_virtual = detail::skip_if(stream, "virtual"); - detail::skip(stream, "~"); + auto prefix_info = parse_prefix_info(stream, "~", false); + DEBUG_ASSERT(!prefix_info.is_constexpr && !prefix_info.is_explicit, detail::assert_handler{}); + auto name = std::string("~") + stream.get().c_str(); cpp_destructor::builder builder(std::move(name)); context.comments.match(builder.get(), cur); detail::skip(stream, "("); detail::skip(stream, ")"); - return handle_suffix(context, cur, builder, stream, is_virtual, type_safe::nullopt); + return handle_suffix(context, cur, builder, stream, prefix_info.is_virtual, + std::move(prefix_info.semantic_parent)); } diff --git a/test/cpp_member_function.cpp b/test/cpp_member_function.cpp index 06c51a8..9ac583b 100644 --- a/test/cpp_member_function.cpp +++ b/test/cpp_member_function.cpp @@ -331,6 +331,10 @@ struct d : c /// virtual ~d() override final; ~d() final; }; + +/// virtual d::~d() override; +d::~d() {} + )"; auto file = parse({}, "cpp_destructor.cpp", code); @@ -364,12 +368,15 @@ struct d : c else if (dtor.name() == "~d") { REQUIRE(dtor.virtual_info()); - REQUIRE(dtor.virtual_info().value() - == (cpp_virtual_flags::override | cpp_virtual_flags::final)); + if (dtor.is_declaration()) + REQUIRE(dtor.virtual_info().value() + == (cpp_virtual_flags::override | cpp_virtual_flags::final)); + else + REQUIRE(dtor.virtual_info().value() == cpp_virtual_flags::override); REQUIRE(!dtor.noexcept_condition()); } else REQUIRE(false); }); - REQUIRE(count == 4u); + REQUIRE(count == 5u); } From f596ef398f6cebf029f3eb4383d063a1bfae0d23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonathan=20M=C3=BCller?= Date: Thu, 15 Jun 2017 21:10:34 +0200 Subject: [PATCH 10/20] Don't generate override and final on definitions --- src/code_generator.cpp | 13 +++++++++---- test/cpp_member_function.cpp | 2 +- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/src/code_generator.cpp b/src/code_generator.cpp index eb50362..91d36ad 100644 --- a/src/code_generator.cpp +++ b/src/code_generator.cpp @@ -577,8 +577,13 @@ namespace output << keyword("virtual") << whitespace; } - void write_suffix_virtual(code_generator::output& output, const cpp_virtual& virt) + void write_suffix_virtual(code_generator::output& output, const cpp_virtual& virt, + bool is_definition) { + if (is_definition) + // don't include it in definition + return; + if (is_overriding(virt)) output << whitespace << keyword("override"); if (is_final(virt)) @@ -667,7 +672,7 @@ namespace detail::write_type(output, func.return_type(), ""); } - write_suffix_virtual(output, func.virtual_info()); + write_suffix_virtual(output, func.virtual_info(), func.is_definition()); write_function_body(output, func, is_pure(func.virtual_info())); } return static_cast(output); @@ -701,7 +706,7 @@ namespace write_noexcept(output, op, need_ws || output.formatting().is_set(formatting_flags::operator_ws)); - write_suffix_virtual(output, op.virtual_info()); + write_suffix_virtual(output, op.virtual_info(), op.is_definition()); write_function_body(output, op, is_pure(op.virtual_info())); } return static_cast(output); @@ -740,7 +745,7 @@ namespace << punctuation("(") << punctuation(")"); write_noexcept(output, dtor, output.formatting().is_set(formatting_flags::operator_ws)); - write_suffix_virtual(output, dtor.virtual_info()); + write_suffix_virtual(output, dtor.virtual_info(), dtor.is_definition()); write_function_body(output, dtor, is_pure(dtor.virtual_info())); } return static_cast(output); diff --git a/test/cpp_member_function.cpp b/test/cpp_member_function.cpp index 9ac583b..24687e3 100644 --- a/test/cpp_member_function.cpp +++ b/test/cpp_member_function.cpp @@ -332,7 +332,7 @@ struct d : c ~d() final; }; -/// virtual d::~d() override; +/// virtual d::~d(); d::~d() {} )"; From 9590b63c2c06cbd8cc61d23f67c7ac981f70b3fe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonathan=20M=C3=BCller?= Date: Wed, 21 Jun 2017 09:33:11 +0200 Subject: [PATCH 11/20] Fix missing paren in destructor tokens --- src/libclang/tokenizer.cpp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/libclang/tokenizer.cpp b/src/libclang/tokenizer.cpp index d972272..ea71289 100644 --- a/src/libclang/tokenizer.cpp +++ b/src/libclang/tokenizer.cpp @@ -135,6 +135,9 @@ namespace else if (kind == CXCursor_CXXMethod) // necessary for some reason begin = get_next_location(tu, file, begin, -1); + else if (kind == CXCursor_Destructor && token_after_is(tu, file, cur, end, ")")) + // necessary for some other reason + end = get_next_location(tu, file, end); } else if (kind == CXCursor_TemplateTypeParameter && token_after_is(tu, file, cur, end, "(")) { From 35c090e7c32c8f52ee9f4e3c463bdf962117a4a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonathan=20M=C3=BCller?= Date: Wed, 21 Jun 2017 10:00:35 +0200 Subject: [PATCH 12/20] Put full path in quotes --- src/libclang/preprocessor.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/src/libclang/preprocessor.cpp b/src/libclang/preprocessor.cpp index 69292bb..9b40e49 100644 --- a/src/libclang/preprocessor.cpp +++ b/src/libclang/preprocessor.cpp @@ -25,6 +25,11 @@ namespace ts = type_safe; namespace { + std::string quote(std::string str) + { + return '"' + std::move(str) + '"'; + } + // build the command that runs the preprocessor std::string get_command(const libclang_compile_config& c, const char* full_path) { @@ -56,7 +61,7 @@ namespace } // add path to file being processed - cmd += full_path; + cmd += quote(full_path); return cmd; } From 7e71a64437a81c4688d8913533b9ab438373ec2a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonathan=20M=C3=BCller?= Date: Wed, 21 Jun 2017 15:34:15 +0200 Subject: [PATCH 13/20] Allow function names enclosed in parenthesis --- src/libclang/function_parser.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/libclang/function_parser.cpp b/src/libclang/function_parser.cpp index b8571af..cc92d1a 100644 --- a/src/libclang/function_parser.cpp +++ b/src/libclang/function_parser.cpp @@ -160,6 +160,10 @@ namespace } DEBUG_ASSERT(!stream.done(), detail::parse_error_handler{}, stream.cursor(), "unable to find end of function prefix"); + while (detail::skip_if(stream, ")")) + { // function name can be enclosed in parentheses + } + if (!scope.empty() && scope.back() == ':') { result.semantic_parent = From c07f80ccf381f234ed7653ee03c414aee45d766e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonathan=20M=C3=BCller?= Date: Wed, 21 Jun 2017 19:18:00 +0200 Subject: [PATCH 14/20] Improve skip brackets heuristic --- src/libclang/tokenizer.cpp | 25 ++++++++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/src/libclang/tokenizer.cpp b/src/libclang/tokenizer.cpp index ea71289..5e5642e 100644 --- a/src/libclang/tokenizer.cpp +++ b/src/libclang/tokenizer.cpp @@ -288,6 +288,20 @@ bool detail::skip_if(detail::token_stream& stream, const char* str, bool multi_t return true; } +namespace +{ + // whether or not the current angle bracket can be a comparison + // note: this is a heuristic I hope works often enough + bool is_comparison(CXTokenKind last_kind, const detail::token& cur, CXTokenKind next_kind) + { + if (cur == "<") + return last_kind == CXToken_Literal; + else if (cur == ">") + return next_kind == CXToken_Literal; + return false; + } +} + detail::token_iterator detail::find_closing_bracket(detail::token_stream stream) { auto template_bracket = false; @@ -310,12 +324,15 @@ detail::token_iterator detail::find_closing_bracket(detail::token_stream stream) auto bracket_count = 1; auto paren_count = 0; // internal nested parenthesis + auto last_token = CXToken_Comment; while (!stream.done() && bracket_count != 0) { - auto& cur = stream.get().value(); - if (paren_count == 0 && cur == open_bracket) + auto& cur = stream.get(); + if (paren_count == 0 && cur == open_bracket + && !is_comparison(last_token, cur, stream.peek().kind())) ++bracket_count; - else if (paren_count == 0 && cur == close_bracket) + else if (paren_count == 0 && cur == close_bracket + && !is_comparison(last_token, cur, stream.peek().kind())) --bracket_count; else if (paren_count == 0 && template_bracket && cur == ">>") // maximal munch @@ -324,6 +341,8 @@ detail::token_iterator detail::find_closing_bracket(detail::token_stream stream) ++paren_count; else if (cur == ")" || cur == "}" || cur == "]") --paren_count; + + last_token = cur.kind(); } stream.bump_back(); // only check first parameter, token might be ">>" From 7a9f6fdfac27a731e4520d6d0308f9e35870b604 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonathan=20M=C3=BCller?= Date: Thu, 22 Jun 2017 22:35:36 +0200 Subject: [PATCH 15/20] Improve function scope parsing --- src/libclang/friend_parser.cpp | 2 +- src/libclang/function_parser.cpp | 118 +++++++++++++++++++++---------- src/libclang/parse_functions.cpp | 37 +++++++--- src/libclang/parse_functions.hpp | 15 ++-- src/libclang/template_parser.cpp | 20 +++--- 5 files changed, 126 insertions(+), 66 deletions(-) diff --git a/src/libclang/friend_parser.cpp b/src/libclang/friend_parser.cpp index 14830e8..9649fa5 100644 --- a/src/libclang/friend_parser.cpp +++ b/src/libclang/friend_parser.cpp @@ -74,7 +74,7 @@ std::unique_ptr detail::parse_cpp_friend(const detail::parse_context } else if (clang_isDeclaration(kind)) { - entity = parse_entity(context, child); + entity = parse_entity(context, child, cur); if (entity) { // steal comment diff --git a/src/libclang/function_parser.cpp b/src/libclang/function_parser.cpp index cc92d1a..3491f9e 100644 --- a/src/libclang/function_parser.cpp +++ b/src/libclang/function_parser.cpp @@ -85,13 +85,72 @@ namespace detail::skip_brackets(stream); } + bool is_class(const CXCursor& parent) + { + auto kind = clang_getCursorKind(parent); + return kind == CXCursor_ClassDecl || kind == CXCursor_StructDecl + || kind == CXCursor_UnionDecl || kind == CXCursor_ClassTemplate + || kind == CXCursor_ClassTemplatePartialSpecialization; + } + + CXCursor get_definition_scope(const CXCursor& cur, bool is_friend) + { + auto parent = clang_getCursorLexicalParent(cur); + if (is_friend) + { + // find the lexical parent that isn't a class + // as the definition scope is a namespace + while (is_class(parent)) + parent = clang_getCursorSemanticParent(parent); + + DEBUG_ASSERT(clang_getCursorKind(parent) == CXCursor_Namespace + || clang_getCursorKind(parent) == CXCursor_TranslationUnit, + detail::parse_error_handler{}, cur, + "unable to find definition scope of friend"); + } + return parent; + } + + bool equivalent_cursor(const CXCursor& a, const CXCursor& b) + { + if (clang_getCursorKind(a) == clang_getCursorKind(b) + && clang_getCursorKind(a) == CXCursor_Namespace) + return detail::cxstring(clang_getCursorUSR(a)) + == detail::cxstring(clang_getCursorUSR(b)); + else + return clang_equalCursors(a, b) == 1; + } + + type_safe::optional parse_scope(const CXCursor& cur, bool is_friend) + { + std::string scope; + // find the semantic parents we need until we're at the same level of the parent where the definition is + // the semantic parents are all the scopes that need to be appended + for (auto definition = get_definition_scope(cur, is_friend), + parent = clang_getCursorSemanticParent(cur); + !equivalent_cursor(definition, parent); parent = clang_getCursorSemanticParent(parent)) + { + DEBUG_ASSERT(!clang_isTranslationUnit(clang_getCursorKind(parent)), + detail::parse_error_handler{}, cur, + "infinite loop while calculating scope"); + auto parent_name = detail::cxstring(clang_getCursorDisplayName(parent)); + scope = parent_name.std_str() + "::" + std::move(scope); + } + + if (scope.empty()) + return type_safe::nullopt; + else + return cpp_entity_ref(detail::get_entity_id(clang_getCursorSemanticParent(cur)), + std::move(scope)); + } + // just the tokens occurring in the prefix struct prefix_info { - type_safe::optional semantic_parent; - bool is_constexpr = false; - bool is_virtual = false; - bool is_explicit = false; + bool is_constexpr = false; + bool is_virtual = false; + bool is_explicit = false; + bool is_friend = false; }; bool prefix_end(detail::token_stream& stream, const char* name, bool is_ctor) @@ -137,25 +196,15 @@ namespace { prefix_info result; - std::string scope; while (!stream.done() && !prefix_end(stream, name, is_ctor)) { if (detail::skip_if(stream, "constexpr")) - { result.is_constexpr = true; - scope.clear(); - } else if (detail::skip_if(stream, "virtual")) - { result.is_virtual = true; - scope.clear(); - } else if (detail::skip_if(stream, "explicit")) - { result.is_explicit = true; - scope.clear(); - } - else if (!detail::append_scope(stream, scope)) + else stream.bump(); } DEBUG_ASSERT(!stream.done(), detail::parse_error_handler{}, stream.cursor(), @@ -164,14 +213,6 @@ namespace { // function name can be enclosed in parentheses } - if (!scope.empty() && scope.back() == ':') - { - result.semantic_parent = - cpp_entity_ref(detail::get_entity_id( - clang_getCursorSemanticParent(stream.cursor())), - std::move(scope)); - } - return result; } @@ -372,7 +413,8 @@ namespace } std::unique_ptr parse_cpp_function_impl(const detail::parse_context& context, - const CXCursor& cur, bool is_static) + const CXCursor& cur, bool is_static, + bool is_friend) { auto name = detail::get_cursor_name(cur); @@ -405,21 +447,21 @@ namespace if (is_templated_cursor(cur)) return builder.finish(detail::get_entity_id(cur), suffix.body_kind, - std::move(prefix.semantic_parent)); + parse_scope(cur, is_friend)); else return builder.finish(*context.idx, detail::get_entity_id(cur), suffix.body_kind, - std::move(prefix.semantic_parent)); + parse_scope(cur, is_friend)); } } std::unique_ptr detail::parse_cpp_function(const detail::parse_context& context, - const CXCursor& cur) + const CXCursor& cur, bool is_friend) { DEBUG_ASSERT(clang_getCursorKind(cur) == CXCursor_FunctionDecl || clang_getTemplateCursorKind(cur) == CXCursor_FunctionDecl, detail::assert_handler{}); type_safe::optional semantic_parent; - return parse_cpp_function_impl(context, cur, false); + return parse_cpp_function_impl(context, cur, false, is_friend); } std::unique_ptr detail::try_parse_static_cpp_function( @@ -429,7 +471,7 @@ std::unique_ptr detail::try_parse_static_cpp_function( || clang_getTemplateCursorKind(cur) == CXCursor_CXXMethod, detail::assert_handler{}); if (clang_CXXMethod_isStatic(cur)) - return parse_cpp_function_impl(context, cur, true); + return parse_cpp_function_impl(context, cur, true, false); return nullptr; } @@ -521,7 +563,7 @@ namespace } std::unique_ptr detail::parse_cpp_member_function(const detail::parse_context& context, - const CXCursor& cur) + const CXCursor& cur, bool is_friend) { DEBUG_ASSERT(clang_getCursorKind(cur) == CXCursor_CXXMethod || clang_getTemplateCursorKind(cur) == CXCursor_CXXMethod, @@ -548,11 +590,11 @@ std::unique_ptr detail::parse_cpp_member_function(const detail::pars skip_parameters(stream); return handle_suffix(context, cur, builder, stream, prefix.is_virtual, - std::move(prefix.semantic_parent)); + parse_scope(cur, is_friend)); } std::unique_ptr detail::parse_cpp_conversion_op(const detail::parse_context& context, - const CXCursor& cur) + const CXCursor& cur, bool is_friend) { DEBUG_ASSERT(clang_getCursorKind(cur) == CXCursor_ConversionFunction || clang_getTemplateCursorKind(cur) == CXCursor_ConversionFunction, @@ -609,11 +651,11 @@ std::unique_ptr detail::parse_cpp_conversion_op(const detail::parse_ builder.is_constexpr(); return handle_suffix(context, cur, builder, stream, prefix.is_virtual, - std::move(prefix.semantic_parent)); + parse_scope(cur, is_friend)); } std::unique_ptr detail::parse_cpp_constructor(const detail::parse_context& context, - const CXCursor& cur) + const CXCursor& cur, bool is_friend) { DEBUG_ASSERT(clang_getCursorKind(cur) == CXCursor_Constructor || clang_getTemplateCursorKind(cur) == CXCursor_Constructor, @@ -648,14 +690,14 @@ std::unique_ptr detail::parse_cpp_constructor(const detail::parse_co if (is_templated_cursor(cur)) return builder.finish(detail::get_entity_id(cur), suffix.body_kind, - std::move(prefix.semantic_parent)); + parse_scope(cur, is_friend)); else return builder.finish(*context.idx, detail::get_entity_id(cur), suffix.body_kind, - std::move(prefix.semantic_parent)); + parse_scope(cur, is_friend)); } std::unique_ptr detail::parse_cpp_destructor(const detail::parse_context& context, - const CXCursor& cur) + const CXCursor& cur, bool is_friend) { DEBUG_ASSERT(clang_getCursorKind(cur) == CXCursor_Destructor, detail::assert_handler{}); @@ -672,5 +714,5 @@ std::unique_ptr detail::parse_cpp_destructor(const detail::parse_con detail::skip(stream, "("); detail::skip(stream, ")"); return handle_suffix(context, cur, builder, stream, prefix_info.is_virtual, - std::move(prefix_info.semantic_parent)); + parse_scope(cur, is_friend)); } diff --git a/src/libclang/parse_functions.cpp b/src/libclang/parse_functions.cpp index d17aa6f..4940cdd 100644 --- a/src/libclang/parse_functions.cpp +++ b/src/libclang/parse_functions.cpp @@ -91,6 +91,19 @@ void detail::comment_context::match(cpp_entity& e, unsigned line) const e.set_comment(std::move(cur_++->comment)); } +namespace +{ + bool is_friend(const CXCursor& parent_cur) + { +#if CPPAST_CINDEX_HAS_FRIEND + return clang_getCursorKind(parent_cur) == CXCursor_FriendDecl; +#else + (void)parent_cur; + return false; +#endif + } +} + std::unique_ptr detail::parse_entity(const detail::parse_context& context, const CXCursor& cur, const CXCursor& parent_cur) try @@ -145,25 +158,29 @@ std::unique_ptr detail::parse_entity(const detail::parse_context& co return parse_cpp_member_variable(context, cur); case CXCursor_FunctionDecl: - if (auto tfunc = try_parse_cpp_function_template_specialization(context, cur)) + if (auto tfunc = + try_parse_cpp_function_template_specialization(context, cur, is_friend(parent_cur))) return tfunc; - return parse_cpp_function(context, cur); + return parse_cpp_function(context, cur, is_friend(parent_cur)); case CXCursor_CXXMethod: - if (auto tfunc = try_parse_cpp_function_template_specialization(context, cur)) + if (auto tfunc = + try_parse_cpp_function_template_specialization(context, cur, is_friend(parent_cur))) return tfunc; else if (auto func = try_parse_static_cpp_function(context, cur)) return func; - return parse_cpp_member_function(context, cur); + return parse_cpp_member_function(context, cur, is_friend(parent_cur)); case CXCursor_ConversionFunction: - if (auto tfunc = try_parse_cpp_function_template_specialization(context, cur)) + if (auto tfunc = + try_parse_cpp_function_template_specialization(context, cur, is_friend(parent_cur))) return tfunc; - return parse_cpp_conversion_op(context, cur); + return parse_cpp_conversion_op(context, cur, is_friend(parent_cur)); case CXCursor_Constructor: - if (auto tfunc = try_parse_cpp_function_template_specialization(context, cur)) + if (auto tfunc = + try_parse_cpp_function_template_specialization(context, cur, is_friend(parent_cur))) return tfunc; - return parse_cpp_constructor(context, cur); + return parse_cpp_constructor(context, cur, is_friend(parent_cur)); case CXCursor_Destructor: - return parse_cpp_destructor(context, cur); + return parse_cpp_destructor(context, cur, is_friend(parent_cur)); #if CPPAST_CINDEX_HAS_FRIEND case CXCursor_FriendDecl: @@ -173,7 +190,7 @@ std::unique_ptr detail::parse_entity(const detail::parse_context& co case CXCursor_TypeAliasTemplateDecl: return parse_cpp_alias_template(context, cur); case CXCursor_FunctionTemplate: - return parse_cpp_function_template(context, cur); + return parse_cpp_function_template(context, cur, is_friend(parent_cur)); case CXCursor_ClassTemplate: return parse_cpp_class_template(context, cur); case CXCursor_ClassTemplatePartialSpecialization: diff --git a/src/libclang/parse_functions.hpp b/src/libclang/parse_functions.hpp index db84937..4572437 100644 --- a/src/libclang/parse_functions.hpp +++ b/src/libclang/parse_functions.hpp @@ -103,7 +103,7 @@ namespace cppast // on all function cursors except on destructor std::unique_ptr try_parse_cpp_function_template_specialization( - const parse_context& context, const CXCursor& cur); + const parse_context& context, const CXCursor& cur, bool is_friend); // on class cursors std::unique_ptr try_parse_full_cpp_class_template_specialization( @@ -134,15 +134,15 @@ namespace cppast const CXCursor& cur); std::unique_ptr parse_cpp_function(const parse_context& context, - const CXCursor& cur); + const CXCursor& cur, bool is_friend); std::unique_ptr parse_cpp_member_function(const parse_context& context, - const CXCursor& cur); + const CXCursor& cur, bool is_friend); std::unique_ptr parse_cpp_conversion_op(const parse_context& context, - const CXCursor& cur); + const CXCursor& cur, bool is_friend); std::unique_ptr parse_cpp_constructor(const parse_context& context, - const CXCursor& cur); + const CXCursor& cur, bool is_friend); std::unique_ptr parse_cpp_destructor(const parse_context& context, - const CXCursor& cur); + const CXCursor& cur, bool is_friend); std::unique_ptr parse_cpp_friend(const parse_context& context, const CXCursor& cur); @@ -150,7 +150,8 @@ namespace cppast std::unique_ptr parse_cpp_alias_template(const parse_context& context, const CXCursor& cur); std::unique_ptr parse_cpp_function_template(const parse_context& context, - const CXCursor& cur); + const CXCursor& cur, + bool is_friend); std::unique_ptr parse_cpp_class_template(const parse_context& context, const CXCursor& cur); std::unique_ptr parse_cpp_class_template_specialization( diff --git a/src/libclang/template_parser.cpp b/src/libclang/template_parser.cpp index ef25975..1b4ca78 100644 --- a/src/libclang/template_parser.cpp +++ b/src/libclang/template_parser.cpp @@ -216,7 +216,7 @@ std::unique_ptr detail::parse_cpp_alias_template(const detail::parse } std::unique_ptr detail::parse_cpp_function_template( - const detail::parse_context& context, const CXCursor& cur) + const detail::parse_context& context, const CXCursor& cur, bool is_friend) { DEBUG_ASSERT(clang_getCursorKind(cur) == CXCursor_FunctionTemplate, detail::assert_handler{}); @@ -224,19 +224,19 @@ std::unique_ptr detail::parse_cpp_function_template( switch (clang_getTemplateCursorKind(cur)) { case CXCursor_FunctionDecl: - func = detail::parse_cpp_function(context, cur); + func = detail::parse_cpp_function(context, cur, is_friend); break; case CXCursor_CXXMethod: if (auto sfunc = detail::try_parse_static_cpp_function(context, cur)) func = std::move(sfunc); else - func = detail::parse_cpp_member_function(context, cur); + func = detail::parse_cpp_member_function(context, cur, is_friend); break; case CXCursor_ConversionFunction: - func = detail::parse_cpp_conversion_op(context, cur); + func = detail::parse_cpp_conversion_op(context, cur, is_friend); break; case CXCursor_Constructor: - func = detail::parse_cpp_constructor(context, cur); + func = detail::parse_cpp_constructor(context, cur, is_friend); break; default: @@ -284,7 +284,7 @@ namespace } std::unique_ptr detail::try_parse_cpp_function_template_specialization( - const detail::parse_context& context, const CXCursor& cur) + const detail::parse_context& context, const CXCursor& cur, bool is_friend) { auto templ = clang_getSpecializedCursorTemplate(cur); if (clang_Cursor_isNull(templ)) @@ -294,19 +294,19 @@ std::unique_ptr detail::try_parse_cpp_function_template_specializati switch (clang_getCursorKind(cur)) { case CXCursor_FunctionDecl: - func = detail::parse_cpp_function(context, cur); + func = detail::parse_cpp_function(context, cur, is_friend); break; case CXCursor_CXXMethod: if (auto sfunc = detail::try_parse_static_cpp_function(context, cur)) func = std::move(sfunc); else - func = detail::parse_cpp_member_function(context, cur); + func = detail::parse_cpp_member_function(context, cur, is_friend); break; case CXCursor_ConversionFunction: - func = detail::parse_cpp_conversion_op(context, cur); + func = detail::parse_cpp_conversion_op(context, cur, is_friend); break; case CXCursor_Constructor: - func = detail::parse_cpp_constructor(context, cur); + func = detail::parse_cpp_constructor(context, cur, is_friend); break; default: From 232842f2f67ef52846e4647612197c6e1308618f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonathan=20M=C3=BCller?= Date: Fri, 23 Jun 2017 07:22:44 +0200 Subject: [PATCH 16/20] Fix constructor prefix skipping --- src/libclang/function_parser.cpp | 50 +++++++++++++++++--------------- 1 file changed, 27 insertions(+), 23 deletions(-) diff --git a/src/libclang/function_parser.cpp b/src/libclang/function_parser.cpp index 3491f9e..7217617 100644 --- a/src/libclang/function_parser.cpp +++ b/src/libclang/function_parser.cpp @@ -4,7 +4,6 @@ #include #include -#include #include "libclang_visitor.hpp" #include "parse_functions.hpp" @@ -159,34 +158,39 @@ namespace // name can have multiple tokens if it is an operator if (!detail::skip_if(stream, name, true)) return false; - else if (!is_ctor) - return true; - // if we reach this point, we've encountered the name of a constructor - // need to make sure it is not actually a class name - else if (stream.peek() == "::") - { - // after name came "::", it is a class name - stream.set_cur(cur); + else if (stream.peek() == "," || stream.peek() == ">" || stream.peek() == ">>") + // argument to template parameters return false; - } - else if (stream.peek() == "<") + else if (is_ctor) { - // after name came "<", it might be arguments for a class template, - // or just a specialization - // check if ( comes after the arguments - detail::skip_brackets(stream); - if (stream.peek() == "(") + // need to make sure it is not actually a class name + if (stream.peek() == "::") { - // it was just a specialization, we're at the end - stream.set_cur(cur); - return true; - } - else - { - // class arguments + // after name came "::", it is a class name stream.set_cur(cur); return false; } + else if (stream.peek() == "<") + { + // after name came "<", it might be arguments for a class template, + // or just a specialization + // check if ( comes after the arguments + detail::skip_brackets(stream); + if (stream.peek() == "(") + { + // it was just a specialization, we're at the end + stream.set_cur(cur); + return true; + } + else + { + // class arguments + stream.set_cur(cur); + return false; + } + } + else + return true; } else return true; From e355e7a6538e4ba9f643805819840397e66de9b7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonathan=20M=C3=BCller?= Date: Fri, 23 Jun 2017 07:34:22 +0200 Subject: [PATCH 17/20] Fix empty arguments to template instantiation --- include/cppast/code_generator.hpp | 5 ++-- include/cppast/cpp_template.hpp | 7 +++-- src/code_generator.cpp | 49 +++++++++++++++++-------------- test/cpp_type_alias.cpp | 16 ++++++---- 4 files changed, 45 insertions(+), 32 deletions(-) diff --git a/include/cppast/code_generator.hpp b/include/cppast/code_generator.hpp index ccf29d4..85abd35 100644 --- a/include/cppast/code_generator.hpp +++ b/include/cppast/code_generator.hpp @@ -482,8 +482,9 @@ namespace cppast /// \exclude namespace detail { - void write_template_arguments(code_generator::output& output, - type_safe::array_ref arguments); + void write_template_arguments( + code_generator::output& output, + type_safe::optional> arguments); } // namespace detail } // namespace cppast diff --git a/include/cppast/cpp_template.hpp b/include/cppast/cpp_template.hpp index c5155d6..31d5943 100644 --- a/include/cppast/cpp_template.hpp +++ b/include/cppast/cpp_template.hpp @@ -140,12 +140,15 @@ namespace cppast type_safe::variant_type>{}); } - /// \returns An iteratable object iterating over the [cppast::cpp_template_argument]()s. + /// \returns An array ref to the [cppast::cpp_template_argument](), if there are any. /// \requires The arguments are exposed, i.e. `arguments_exposed()` returns `true`. - type_safe::array_ref arguments() const noexcept + type_safe::optional> arguments() const + noexcept { auto& vec = arguments_.value(type_safe::variant_type>{}); + if (vec.empty()) + return type_safe::nullopt; return type_safe::ref(vec.data(), vec.size()); } diff --git a/src/code_generator.cpp b/src/code_generator.cpp index 91d36ad..6079194 100644 --- a/src/code_generator.cpp +++ b/src/code_generator.cpp @@ -1035,29 +1035,34 @@ bool cppast::generate_code(code_generator& generator, const cpp_entity& e) return false; } -void detail::write_template_arguments(code_generator::output& output, - type_safe::array_ref arguments) +void detail::write_template_arguments( + code_generator::output& output, + type_safe::optional> arguments) { - if (arguments.size() == 0u) - return; - - output << punctuation("<") << bracket_ws; - auto need_sep = false; - for (auto& arg : arguments) + if (!arguments) { - if (need_sep) - output << comma; - else - need_sep = true; - - if (auto type = arg.type()) - detail::write_type(output, type.value(), ""); - else if (auto expr = arg.expression()) - detail::write_expression(output, expr.value()); - else if (auto templ = arg.template_ref()) - output << templ.value(); - else - DEBUG_UNREACHABLE(detail::assert_handler{}); + output << punctuation("<") << punctuation(">"); + } + else + { + output << punctuation("<") << bracket_ws; + auto need_sep = false; + for (auto& arg : arguments.value()) + { + if (need_sep) + output << comma; + else + need_sep = true; + + if (auto type = arg.type()) + detail::write_type(output, type.value(), ""); + else if (auto expr = arg.expression()) + detail::write_expression(output, expr.value()); + else if (auto templ = arg.template_ref()) + output << templ.value(); + else + DEBUG_UNREACHABLE(detail::assert_handler{}); + } + output << bracket_ws << punctuation(">"); } - output << bracket_ws << punctuation(">"); } diff --git a/test/cpp_type_alias.cpp b/test/cpp_type_alias.cpp index 754a2cc..13d9b1b 100644 --- a/test/cpp_type_alias.cpp +++ b/test/cpp_type_alias.cpp @@ -149,11 +149,15 @@ bool equal_types(const cpp_entity_index& idx, const cpp_type& parsed, const cpp_ return false; else if (!inst_parsed.arguments_exposed()) return inst_parsed.unexposed_arguments() == inst_synthesized.unexposed_arguments(); + else if (inst_parsed.arguments().has_value() != inst_synthesized.arguments().has_value()) + return false; + else if (!inst_parsed.arguments().has_value()) + return true; - auto iter_a = inst_parsed.arguments().begin(); - auto iter_b = inst_synthesized.arguments().begin(); - while (iter_a != inst_parsed.arguments().end() - && iter_b != inst_synthesized.arguments().end()) + auto iter_a = inst_parsed.arguments().value().begin(); + auto iter_b = inst_synthesized.arguments().value().begin(); + while (iter_a != inst_parsed.arguments().value().end() + && iter_b != inst_synthesized.arguments().value().end()) { if (iter_a->type().has_value() && iter_b->type().has_value()) { @@ -175,8 +179,8 @@ bool equal_types(const cpp_entity_index& idx, const cpp_type& parsed, const cpp_ ++iter_a; ++iter_b; } - return iter_a == inst_parsed.arguments().end() - && iter_b == inst_synthesized.arguments().end(); + return iter_a == inst_parsed.arguments().value().end() + && iter_b == inst_synthesized.arguments().value().end(); } case cpp_type_kind::dependent_t: { From bcf65703d755e9d611acb24310ceb9c5c7e758a9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonathan=20M=C3=BCller?= Date: Thu, 15 Jun 2017 21:20:01 +0200 Subject: [PATCH 18/20] Add integration test parsing the standard library --- test/CMakeLists.txt | 37 +++++++------- test/integration.cpp | 118 +++++++++++++++++++++++++++++++++++++++++++ test/test_parser.hpp | 13 +++-- 3 files changed, 146 insertions(+), 22 deletions(-) create mode 100644 test/integration.cpp diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 34c651b..a613ec6 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -9,24 +9,25 @@ if(NOT EXISTS ${CMAKE_CURRENT_BINARY_DIR}/catch.hpp) endif() set(tests - code_generator.cpp - cpp_alias_template.cpp - cpp_class.cpp - cpp_class_template.cpp - cpp_enum.cpp - cpp_friend.cpp - cpp_function.cpp - cpp_function_template.cpp - cpp_language_linkage.cpp - cpp_member_function.cpp - cpp_member_variable.cpp - cpp_namespace.cpp - cpp_preprocessor.cpp - cpp_static_assert.cpp - cpp_template_parameter.cpp - cpp_type_alias.cpp - cpp_variable.cpp - visitor.cpp) + code_generator.cpp + cpp_alias_template.cpp + cpp_class.cpp + cpp_class_template.cpp + cpp_enum.cpp + cpp_friend.cpp + cpp_function.cpp + cpp_function_template.cpp + cpp_language_linkage.cpp + cpp_member_function.cpp + cpp_member_variable.cpp + cpp_namespace.cpp + cpp_preprocessor.cpp + cpp_static_assert.cpp + cpp_template_parameter.cpp + cpp_type_alias.cpp + cpp_variable.cpp + visitor.cpp + integration.cpp) add_executable(cppast_test test.cpp test_parser.hpp ${tests}) target_include_directories(cppast_test PUBLIC ${CMAKE_CURRENT_BINARY_DIR}) diff --git a/test/integration.cpp b/test/integration.cpp new file mode 100644 index 0000000..847b814 --- /dev/null +++ b/test/integration.cpp @@ -0,0 +1,118 @@ +// Copyright (C) 2017 Jonathan Müller +// This file is subject to the license terms in the LICENSE file +// found in the top-level directory of this distribution. + +#include "test_parser.hpp" + +#include + +using namespace cppast; + +void parse_included_files(const cpp_entity_index& idx, const cpp_file& file) +{ + for (auto& e : file) + { + if (e.kind() == cpp_entity_kind::include_directive_t) + { + auto path = static_cast(e).full_path(); + parse_file(idx, path.c_str()); + } + } +} + +TEST_CASE("stdlib", "[!hide][integration]") +{ + auto code = R"( +// list of headers from: http://en.cppreference.com/w/cpp/header + +//#include -- problem with compiler built-in stuff on OSX +#include +//#include -- same as above +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include +#include +#include +//#include -- missing types from C header (for some reason) +#include + +//#include -- weird issue with compiler built-in stuff +#include +#include +#include +#include + +#include +#include +#include +#include +//#include -- not supported on CI +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include + +//#include -- non-conforming GCC extension with regards to constexpr +//#include -- weird double include issue under MSVC +#include +#include +#include +#include +//#include -- same issue with cinttypes + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +//#include -- issue on OSX + +#include + +#include + +#include +#include +#include +#include +)"; + + cpp_entity_index idx; + auto file = parse(idx, "stdlib.cpp", code); + parse_included_files(idx, *file); +} diff --git a/test/test_parser.hpp b/test/test_parser.hpp index ebb8048..ee38f2e 100644 --- a/test/test_parser.hpp +++ b/test/test_parser.hpp @@ -23,13 +23,11 @@ inline void write_file(const char* name, const char* code) file << code; } -inline std::unique_ptr parse(const cppast::cpp_entity_index& idx, - const char* name, const char* code) +inline std::unique_ptr parse_file(const cppast::cpp_entity_index& idx, + const char* name) { using namespace cppast; - write_file(name, code); - libclang_compile_config config; config.set_flags(cpp_standard::cpp_latest); @@ -42,6 +40,13 @@ inline std::unique_ptr parse(const cppast::cpp_entity_index& i return result; } +inline std::unique_ptr parse(const cppast::cpp_entity_index& idx, + const char* name, const char* code) +{ + write_file(name, code); + return parse_file(idx, name); +} + class test_generator : public cppast::code_generator { public: From 4583a2dd03a737a6a51eeb2dfe092f7146c5d857 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonathan=20M=C3=BCller?= Date: Thu, 15 Jun 2017 21:46:19 +0200 Subject: [PATCH 19/20] Run all tests on Travis --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 44abfd5..e4bf9b7 100644 --- a/.travis.yml +++ b/.travis.yml @@ -59,4 +59,4 @@ script: - mkdir build/ && cd build/ - $CMAKE -DCMAKE_CXX_FLAGS="-Werror -pedantic -Wall -Wextra -Wconversion -Wsign-conversion -Wno-parentheses" -DLLVM_DOWNLOAD_OS_NAME=$LLVM_DOWNLOAD_OS_NAME -DLLVM_PREFERRED_VERSION=$LLVM_VERSION ../ - $CMAKE --build . - - ./test/cppast_test + - if [[ "$LLVM_VERSION" == "4.0.0" ]]; then ./test/cppast_test \*; else ./test/cppast_test; fi From 28bb72850a09b66fd45efdb6c39202ec053dac57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonathan=20M=C3=BCller?= Date: Wed, 21 Jun 2017 09:53:57 +0200 Subject: [PATCH 20/20] Run all tests on appveyor --- appveyor.yml | 4 ++-- test/integration.cpp | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 99c981f..c606c3b 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -5,7 +5,7 @@ build_script: - cmd: clang++ --version - cmd: cmake -G"Visual Studio 14 2015 Win64" -DLLVM_VERSION_EXPLICIT=4.0.0 -DLIBCLANG_LIBRARY="C:/Program Files/LLVM/lib/libclang.lib" -DLIBCLANG_INCLUDE_DIR="C:/Program Files/LLVM/include" -DLIBCLANG_SYSTEM_INCLUDE_DIR="C:/"Program Files"/LLVM/lib/clang/4.0.0/include" -DCLANG_BINARY="C:/Program Files/LLVM/bin/clang++.exe" ../ - - cmd: cmake --build . -- /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" + - cmd: cmake --build . -- /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" /verbosity:minimal test_script: - - cmd: test\Debug\cppast_test.exe + - cmd: test\Debug\cppast_test.exe * diff --git a/test/integration.cpp b/test/integration.cpp index 847b814..e114931 100644 --- a/test/integration.cpp +++ b/test/integration.cpp @@ -104,7 +104,7 @@ TEST_CASE("stdlib", "[!hide][integration]") #include -#include +//#include -- issue on MSVC #include #include