diff --git a/.travis.yml b/.travis.yml index f5357f0..631cd65 100644 --- a/.travis.yml +++ b/.travis.yml @@ -69,7 +69,7 @@ install: script: - mkdir build/ && cd build/ - - $CMAKE -DCMAKE_BUILD_TYPE=Debug -DCMAKE_CXX_FLAGS="-Werror -pedantic -Wall -Wextra -Wconversion -Wsign-conversion -Wno-parentheses -Wno-assume" ../ -DCPPAST_TEST_GCOV=$CPPAST_TEST_GCOV -DLLVM_CONFIG_BINARY=$LLVM_CONFIG_BINARY + - $CMAKE -DCMAKE_EXPORT_COMPILE_COMMANDS=ON -DCMAKE_BUILD_TYPE=Debug -DCMAKE_CXX_FLAGS="-Werror -pedantic -Wall -Wextra -Wconversion -Wsign-conversion -Wno-parentheses -Wno-assume" ../ -DCPPAST_TEST_GCOV=$CPPAST_TEST_GCOV -DLLVM_CONFIG_BINARY=$LLVM_CONFIG_BINARY - $CMAKE --build . - if [[ "$LLVM_VERSION" == "4.0" ]]; then ./test/cppast_test \*; else ./test/cppast_test; fi diff --git a/CMakeLists.txt b/CMakeLists.txt index 166e5a2..67a0382 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -21,6 +21,7 @@ option(BUILD_TESTING "build test" OFF) # The ctest variable for building tests if(${CPPAST_BUILD_TEST} OR ${BUILD_TESTING} OR (CMAKE_CURRENT_SOURCE_DIR STREQUAL CMAKE_SOURCE_DIR)) set(build_test ON) + set(CMAKE_EXPORT_COMPILE_COMMANDS ON) # for the self integration test else() set(build_test OFF) endif() diff --git a/appveyor.yml b/appveyor.yml index c606c3b..b17aeb8 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -8,4 +8,4 @@ build_script: - cmd: cmake --build . -- /logger:"C:\Program Files\AppVeyor\BuildAgent\Appveyor.MSBuildLogger.dll" /verbosity:minimal test_script: - - cmd: test\Debug\cppast_test.exe * + - cmd: test\Debug\cppast_test.exe * ~cppast diff --git a/include/cppast/diagnostic.hpp b/include/cppast/diagnostic.hpp index 46b66e2..2d4bc3a 100644 --- a/include/cppast/diagnostic.hpp +++ b/include/cppast/diagnostic.hpp @@ -47,6 +47,12 @@ namespace cppast return {type_safe::nullopt, type_safe::nullopt, type_safe::nullopt, type_safe::nullopt}; } + /// \returns A source location where entity and file name is available. + static source_location make_entity(std::string entity, std::string file) + { + return {std::move(entity), std::move(file), type_safe::nullopt, type_safe::nullopt}; + } + /// \returns A possible string representation of the source location. /// \notes It will include a separator, but no trailing whitespace. std::string to_string() const @@ -80,6 +86,7 @@ namespace cppast enum class severity { debug, //< A debug diagnostic that is just for debugging purposes. + info, //< An informational message. warning, //< A warning that doesn't impact AST generation. error, //< A non-critical error that does impact AST generation but not critically. critical, //< A critical error where AST generation isn't possible. @@ -93,6 +100,8 @@ namespace cppast { case severity::debug: return "debug"; + case severity::info: + return "info"; case severity::warning: return "warning"; case severity::error: diff --git a/include/cppast/libclang_parser.hpp b/include/cppast/libclang_parser.hpp index bd3fbe5..4e4df82 100644 --- a/include/cppast/libclang_parser.hpp +++ b/include/cppast/libclang_parser.hpp @@ -25,6 +25,8 @@ namespace cppast static const std::vector& flags(const libclang_compile_config& config); static bool write_preprocessed(const libclang_compile_config& config); + + static bool fast_preprocessing(const libclang_compile_config& config); }; void for_each_file(const libclang_compilation_database& database, void* user_data, @@ -128,6 +130,18 @@ namespace cppast write_preprocessed_ = b; } + /// \effects Sets whether or not the fast preprocessor is enabled. + /// Default value is `false`. + /// \notes The fast preprocessor gets a list of all macros that are defined in the translation unit, + /// then preprocesses it without resolving includes but manually defining the list of macros to ensure correctness. + /// Later stages will use the includes again. + /// This hack breaks if you define the same macro multiple times in the file being parsed (headers don't matter) + /// or you rely on the order of macro directives. + void fast_preprocessing(bool b) noexcept + { + fast_preprocessing_ = b; + } + private: void do_set_flags(cpp_standard standard, compile_flags flags) override; @@ -144,7 +158,8 @@ namespace cppast std::string clang_binary_; int clang_version_; - bool write_preprocessed_; + bool write_preprocessed_ : 1; + bool fast_preprocessing_ : 1; friend detail::libclang_compile_config_access; }; diff --git a/include/cppast/parser.hpp b/include/cppast/parser.hpp index 8e7ea0a..860842e 100644 --- a/include/cppast/parser.hpp +++ b/include/cppast/parser.hpp @@ -11,6 +11,7 @@ #include #include #include +#include namespace cppast { @@ -54,6 +55,12 @@ namespace cppast error_ = false; } + /// \returns A reference to the logger used. + const diagnostic_logger& logger() const noexcept + { + return *logger_; + } + protected: /// \effects Creates it giving it a reference to the logger it uses. explicit parser(type_safe::object_ref logger) @@ -61,12 +68,6 @@ namespace cppast { } - /// \returns A reference to the logger used. - const diagnostic_logger& logger() const noexcept - { - return *logger_; - } - /// \effects Sets the error state. /// This must be called when an error or critical diagnostic is logged and the AST is incomplete. void set_error() const noexcept @@ -111,6 +112,9 @@ namespace cppast /// \returns The parsed file or an empty optional, if a fatal error occurred. type_safe::optional_ref parse(std::string path, const config& c) { + parser_.logger().log("simple file parser", + diagnostic{"parsing file '" + path + "'", source_location(), + severity::info}); auto file = parser_.parse(*idx_, std::move(path), c); auto ptr = file.get(); if (file) @@ -149,6 +153,33 @@ namespace cppast type_safe::object_ref idx_; }; + namespace detail + { + struct std_begin + { + }; + struct adl_begin : std_begin + { + }; + struct member_begin : adl_begin + { + }; + + template + auto get_value_type_impl(member_begin, Range&& r) + -> decltype(std::forward(r).begin()); + + template + auto get_value_type_impl(adl_begin, Range&& r) -> decltype(begin(std::forward(r))); + + template + auto get_value_type_impl(std_begin, Range&& r) + -> decltype(std::begin(std::forward(r))); + + template + using value_type = decltype(*get_value_type_impl(member_begin{}, std::declval())); + } // namespace detail + /// Parses multiple files using a given `FileParser`. /// /// \effects Will call the `parse()` function for each path specified in the `file_names`, @@ -166,7 +197,7 @@ namespace cppast template auto parse_files(FileParser& parser, Range&& file_names, const Configuration& get_config) -> typename std::enable_if(file_names).begin()))>::type, + std::declval>()))>::type, typename FileParser::config>::value>::type { for (auto&& file : std::forward(file_names)) @@ -190,17 +221,20 @@ namespace cppast /// Parses all files included by `file`. /// \effects For each [cppast::cpp_include_directive]() in file it will parse the included file. template - void resolve_includes(FileParser& parser, const cpp_file& file, - typename FileParser::config config) + std::size_t resolve_includes(FileParser& parser, const cpp_file& file, + typename FileParser::config config) { + auto count = 0u; for (auto& entity : file) { if (entity.kind() == cpp_include_directive::kind()) { auto& include = static_cast(entity); parser.parse(include.full_path(), config); + ++count; } } + return count; } } // namespace cppast diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index b83bf1e..fe00494 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -15,7 +15,6 @@ set(header ../include/cppast/cpp_class_template.hpp ../include/cppast/cpp_decltype_type.hpp ../include/cppast/cpp_entity.hpp - ../include/cppast/cpp_entity.hpp ../include/cppast/cpp_entity_container.hpp ../include/cppast/cpp_entity_index.hpp ../include/cppast/cpp_entity_kind.hpp diff --git a/src/libclang/enum_parser.cpp b/src/libclang/enum_parser.cpp index a49096d..e1a6201 100644 --- a/src/libclang/enum_parser.cpp +++ b/src/libclang/enum_parser.cpp @@ -99,13 +99,14 @@ std::unique_ptr detail::parse_cpp_enum(const detail::parse_context& catch (parse_error& ex) { context.error = true; - context.logger->log("libclang parser", ex.get_diagnostic()); + context.logger->log("libclang parser", ex.get_diagnostic(context.file)); } catch (std::logic_error& ex) { context.error = true; + auto location = make_location(child); context.logger->log("libclang parser", - diagnostic{ex.what(), make_location(child), severity::error}); + diagnostic{ex.what(), location, severity::error}); } }); if (clang_isCursorDefinition(cur)) diff --git a/src/libclang/function_parser.cpp b/src/libclang/function_parser.cpp index 568a6fd..4e4b2b1 100644 --- a/src/libclang/function_parser.cpp +++ b/src/libclang/function_parser.cpp @@ -51,13 +51,14 @@ namespace catch (detail::parse_error& ex) { context.error = true; - context.logger->log("libclang parser", ex.get_diagnostic()); + context.logger->log("libclang parser", ex.get_diagnostic(context.file)); } catch (std::logic_error& ex) { context.error = true; context.logger->log("libclang parser", - diagnostic{ex.what(), detail::make_location(child), + diagnostic{ex.what(), + detail::make_location(context.file, child), severity::error}); } }); @@ -76,15 +77,20 @@ namespace } catch (detail::parse_error& ex) { - context.logger->log("libclang parser", ex.get_diagnostic()); + context.error = true; + context.logger->log("libclang parser", ex.get_diagnostic(context.file)); } catch (std::logic_error& ex) { - context.logger->log("libclang parser", - diagnostic{ex.what(), - detail::make_location( - clang_Cursor_getArgument(cur, unsigned(i))), - severity::error}); + context.error = true; + context.logger + ->log("libclang parser", + diagnostic{ex.what(), + detail::make_location(context.file, + clang_Cursor_getArgument(cur, + unsigned( + i))), + severity::error}); } } } @@ -223,16 +229,19 @@ namespace bool is_friend = false; }; - bool prefix_end(detail::cxtoken_stream& stream, const char* name, bool is_ctor) + bool prefix_end(detail::cxtoken_stream& stream, const char* name, bool is_ctor_dtor) { auto cur = stream.cur(); // name can have multiple tokens if it is an operator if (!detail::skip_if(stream, name, true)) return false; else if (stream.peek() == "," || stream.peek() == ">" || stream.peek() == ">>") + { // argument to template parameters + stream.set_cur(cur); return false; - else if (is_ctor) + } + else if (is_ctor_dtor) { // need to make sure it is not actually a class name if (stream.peek() == "::") @@ -263,15 +272,22 @@ namespace else return true; } + else if (std::strcmp(name, "operator") != 0 && stream.peek().kind() == CXToken_Identifier) + { + // can't be function name + stream.set_cur(cur); + return false; + } else return true; } - prefix_info parse_prefix_info(detail::cxtoken_stream& stream, const char* name, bool is_ctor) + prefix_info parse_prefix_info(detail::cxtoken_stream& stream, const char* name, + bool is_ctor_dtor) { prefix_info result; - while (!stream.done() && !prefix_end(stream, name, is_ctor)) + while (!stream.done() && !prefix_end(stream, name, is_ctor_dtor)) { if (detail::skip_if(stream, "constexpr")) result.is_constexpr = true; @@ -805,7 +821,7 @@ std::unique_ptr detail::parse_cpp_destructor(const detail::parse_con detail::cxtokenizer tokenizer(context.tu, context.file, cur); detail::cxtoken_stream stream(tokenizer, cur); - auto prefix_info = parse_prefix_info(stream, "~", false); + auto prefix_info = parse_prefix_info(stream, "~", true); DEBUG_ASSERT(!prefix_info.is_constexpr && !prefix_info.is_explicit, detail::assert_handler{}); auto name = std::string("~") + stream.get().c_str(); diff --git a/src/libclang/libclang_parser.cpp b/src/libclang/libclang_parser.cpp index e85d7e2..5c2c78c 100644 --- a/src/libclang/libclang_parser.cpp +++ b/src/libclang/libclang_parser.cpp @@ -42,6 +42,12 @@ bool detail::libclang_compile_config_access::write_preprocessed( return config.write_preprocessed_; } +bool detail::libclang_compile_config_access::fast_preprocessing( + const libclang_compile_config& config) +{ + return config.fast_preprocessing_; +} + libclang_compilation_database::libclang_compilation_database(const std::string& build_directory) { static_assert(std::is_same::value, "forgot to update type"); @@ -81,7 +87,8 @@ namespace } } -libclang_compile_config::libclang_compile_config() : compile_config({}), write_preprocessed_(false) +libclang_compile_config::libclang_compile_config() +: compile_config({}), write_preprocessed_(false), fast_preprocessing_(false) { // set given clang binary auto ptr = CPPAST_CLANG_VERSION_STRING; @@ -231,8 +238,15 @@ libclang_compile_config::libclang_compile_config(const libclang_compilation_data if (flag == "-I") add_flag(std::move(flag) + get_full_path(dir, args)); else if (flag == "-D" || flag == "-U") + { // preprocessor options - add_flag(std::move(flag) + std::move(args)); + for (auto c : args) + if (c == '"') + flag += "\\\""; + else + flag += c; + add_flag(std::move(flag)); + } else if (flag == "-std") // standard add_flag(std::move(flag) + "=" + std::move(args)); @@ -294,7 +308,17 @@ void libclang_compile_config::do_add_macro_definition(std::string name, std::str { auto str = "-D" + std::move(name); if (!definition.empty()) - str += "=" + std::move(definition); + { + str += "=\""; + for (auto c : definition) + { + if (c == '"') + str += "\\\""; + else + str += c; + } + str += "\""; + } add_flag(std::move(str)); } @@ -516,7 +540,8 @@ std::unique_ptr libclang_parser::do_parse(const cpp_entity_index& idx, ++include_iter; } } - else if (clang_getCursorKind(cur) != CXCursor_MacroDefinition) + else if (clang_getCursorKind(cur) != CXCursor_MacroDefinition + && clang_getCursorKind(cur) != CXCursor_MacroExpansion) { // add macro if needed for (auto line = get_line_no(cur); @@ -545,7 +570,7 @@ std::unique_ptr libclang_parser::do_parse(const cpp_entity_index& idx, } catch (detail::parse_error& ex) { - logger().log("libclang parser", ex.get_diagnostic()); + logger().log("libclang parser", ex.get_diagnostic(path)); set_error(); return nullptr; } diff --git a/src/libclang/parse_error.hpp b/src/libclang/parse_error.hpp index a0e0707..4f63766 100644 --- a/src/libclang/parse_error.hpp +++ b/src/libclang/parse_error.hpp @@ -27,6 +27,12 @@ namespace cppast return source_location::make_file(cxstring(file).c_str(), line); } + inline source_location make_location(const CXFile& file, const CXCursor& cur) + { + return source_location::make_entity(get_display_name(cur).c_str(), + cxstring(clang_getFileName(file)).c_str()); + } + inline source_location make_location(const CXType& type) { return source_location::make_entity(cxstring(clang_getTypeSpelling(type)).c_str()); @@ -52,8 +58,14 @@ namespace cppast { } - diagnostic get_diagnostic() const + diagnostic get_diagnostic(const CXFile& file) { + return get_diagnostic(cxstring(clang_getFileName(file)).c_str()); + } + + diagnostic get_diagnostic(std::string file) + { + location_.file = std::move(file); return diagnostic{what(), location_, severity::error}; } diff --git a/src/libclang/parse_functions.cpp b/src/libclang/parse_functions.cpp index 22d3cf0..1a93792 100644 --- a/src/libclang/parse_functions.cpp +++ b/src/libclang/parse_functions.cpp @@ -241,14 +241,15 @@ std::unique_ptr detail::parse_entity(const detail::parse_context& co catch (parse_error& ex) { context.error = true; - context.logger->log("libclang parser", ex.get_diagnostic()); + context.logger->log("libclang parser", ex.get_diagnostic(context.file)); return nullptr; } catch (std::logic_error& ex) { context.error = true; context.logger->log("libclang parser", - diagnostic{ex.what(), detail::make_location(cur), severity::error}); + diagnostic{ex.what(), detail::make_location(context.file, cur), + severity::error}); return nullptr; } diff --git a/src/libclang/preprocessor.cpp b/src/libclang/preprocessor.cpp index cc97ff6..adb1eea 100644 --- a/src/libclang/preprocessor.cpp +++ b/src/libclang/preprocessor.cpp @@ -5,72 +5,32 @@ #include "preprocessor.hpp" #include -#include -#include +#include #include - +#include +#include #include +#include -#include -#include -#include - -#include #include #include "parse_error.hpp" using namespace cppast; -namespace ts = type_safe; +namespace tpl = TinyProcessLib; +namespace ts = type_safe; + +bool detail::pp_doc_comment::matches(const cpp_entity&, unsigned e_line) +{ + if (kind == detail::pp_doc_comment::end_of_line) + return line == e_line; + else + return line + 1u == e_line; +} namespace { - std::string quote(std::string str) - { - return '"' + std::move(str) + '"'; - } - - bool support_include(const libclang_compile_config& c) - { - return detail::libclang_compile_config_access::clang_version(c) >= 40000; - } - - // build the command that runs the preprocessor - std::string get_command(const libclang_compile_config& c, const char* full_path) - { - // -x c++: force C++ as input language - // -I.: add current working directory to include search path - // -E: print preprocessor output - // -C: keep comments - // -dD: print macro definitions as well - auto flags = std::string("-x c++ -I. -E -C -dD"); - if (support_include(c)) - // -Xclang -dI: print include directives as well (clang >= 4.0.0) - flags += " -Xclang -dI"; - // -fno-caret-diagnostics: don't show the source extract in diagnostics - // -fno-show-column: don't show the column number - // -fdiagnostics-format msvc: use easier to parse MSVC format - flags += " -fno-caret-diagnostics -fno-show-column -fdiagnostics-format=msvc"; - // -Wno-*: hide wrong warnings if header file is directly parsed - flags += " -Wno-pragma-once-outside-header -Wno-pragma-system-header-outside-header " - "-Wno-include-next-outside-header"; - - std::string cmd(detail::libclang_compile_config_access::clang_binary(c) + " " - + std::move(flags) + " "); - - // add other flags - for (auto& flag : detail::libclang_compile_config_access::flags(c)) - { - cmd += flag; - cmd += ' '; - } - - // add path to file being processed - cmd += quote(full_path); - - return cmd; - } - + //=== diagnostic parsing ===// source_location parse_source_location(const char*& ptr) { // format: (): @@ -145,49 +105,250 @@ namespace logger.log("preprocessor", diagnostic{std::move(message), std::move(loc), sev}); } - // gets the full preprocessor output - std::string get_full_preprocess_output(const libclang_compile_config& c, const char* full_path, - const diagnostic_logger& logger) + // parses missing header file diagnostic and returns the file name, + // if it is a missing header file diagnostic + ts::optional parse_missing_file(const std::string& cur_file, + const std::string& msg) { - namespace tpl = TinyProcessLib; + auto ptr = msg.c_str(); - std::string preprocessed, diagnostic; + auto loc = parse_source_location(ptr); + if (loc.file != cur_file) + return type_safe::nullopt; - auto cmd = get_command(c, full_path); + while (*ptr == ' ') + ++ptr; + + parse_severity(ptr); + while (*ptr == ' ') + ++ptr; + + // format 'file-name' file not found + if (*ptr != '\'') + return ts::nullopt; + ++ptr; + + std::string filename; + while (*ptr != '\'') + filename += *ptr++; + ++ptr; + + if (std::strcmp(ptr, " file not found") == 0) + return std::move(filename); + else + throw libclang_error("preprocessor: unexpected diagnostic '" + msg + "'"); + } + + //=== external preprocessor invocation ==// + // quote a string + std::string quote(std::string str) + { + return '"' + std::move(str) + '"'; + } + + std::string diagnostics_flags() + { + std::string flags; + + // -fno-caret-diagnostics: don't show the source extract in diagnostics + // -fno-show-column: don't show the column number + // -fdiagnostics-format msvc: use easier to parse MSVC format + flags += " -fno-caret-diagnostics -fno-show-column -fdiagnostics-format=msvc"; + // -Wno-*: hide wrong warnings if header file is directly parsed/duplicate macro handling + flags += " -Wno-macro-redefined -Wno-pragma-once-outside-header " + "-Wno-pragma-system-header-outside-header " + "-Wno-include-next-outside-header"; + + return flags; + } + + // get the command that returns all macros defined in the TU + std::string get_macro_command(const libclang_compile_config& c, const char* full_path) + { + // -x c++: force C++ as input language + // -I.: add current working directory to include search path + // -E: print preprocessor output + // -dM: print macro definitions instead of preprocessed file + auto flags = std::string("-x c++ -I. -E -dM"); + flags += diagnostics_flags(); + + std::string cmd(detail::libclang_compile_config_access::clang_binary(c) + " " + + std::move(flags) + " "); + // other flags + for (auto& flag : detail::libclang_compile_config_access::flags(c)) + { + cmd += flag; + cmd += ' '; + } + + return cmd + quote(full_path); + } + + // get the command that preprocess a translation unit given the macros + std::string get_preprocess_command(const libclang_compile_config& c, const char* full_path, + const char* macro_file_path) + { + // -x c++: force C++ as input language + // -E: print preprocessor output + // -CC: keep comments, even in macro + // -dD: keep macros + // -no*: disable default include search paths + auto flags = std::string("-x c++ -E -CC -dD -nostdinc -nostdinc++"); + if (detail::libclang_compile_config_access::clang_version(c) >= 40000) + // -Xclang -dI: print include directives as well (clang >= 4.0.0) + flags += " -Xclang -dI"; + flags += diagnostics_flags(); + + if (macro_file_path) + { + // include file that defines all macros + flags += " -include "; + flags += macro_file_path; + } + + std::string cmd(detail::libclang_compile_config_access::clang_binary(c) + " " + + std::move(flags) + " "); + + // other flags, as long as they don't add include directories (if we're doing the single file optimization) + for (const auto& flag : detail::libclang_compile_config_access::flags(c)) + { + DEBUG_ASSERT(flag.size() > 2u && flag[0] == '-', detail::assert_handler{}, + "that's an odd flag"); + if (macro_file_path && (flag[0] != '-' || flag[1] != 'I')) + { + cmd += flag; + cmd += ' '; + } + } + + return cmd + quote(full_path); + } + + std::string get_macro_file_name() + { + static std::atomic counter(0u); + return "standardese-macro-file-" + std::to_string(++counter) + ".delete-me"; + } + + std::string write_macro_file(const libclang_compile_config& c, const char* full_path, + const diagnostic_logger& logger) + { + std::string diagnostic; + auto diagnostic_logger = [&](const char* str, std::size_t n) { + diagnostic.reserve(diagnostic.size() + n); + for (auto end = str + n; str != end; ++str) + if (*str == '\r') + continue; + else if (*str == '\n') + { + // consume current diagnostic + log_diagnostic(logger, diagnostic); + diagnostic.clear(); + } + else + diagnostic.push_back(*str); + }; + + auto file = get_macro_file_name(); + std::ofstream stream(file); + + auto cmd = get_macro_command(c, full_path); tpl::Process process(cmd, "", [&](const char* str, std::size_t n) { - preprocessed.reserve(preprocessed.size() + n); - for (auto end = str + n; str != end; ++str) - if (*str == '\t') - preprocessed += " "; // just two spaces because why not - else if (*str != '\r') - preprocessed.push_back(*str); + stream.write(str, std::streamsize(n)); }, - [&](const char* str, std::size_t n) { - diagnostic.reserve(diagnostic.size() + n); - for (auto end = str + n; str != end; ++str) - if (*str == '\r') - continue; - else if (*str == '\n') - { - // consume current diagnostic - log_diagnostic(logger, diagnostic); - diagnostic.clear(); - } - else - diagnostic.push_back(*str); - }); + diagnostic_logger); auto exit_code = process.get_exit_status(); DEBUG_ASSERT(diagnostic.empty(), detail::assert_handler{}); if (exit_code != 0) - throw libclang_error("preprocessor: command '" + cmd + throw libclang_error("preprocessor (macro): command '" + cmd + "' exited with non-zero exit code (" + std::to_string(exit_code) + ")"); - - return preprocessed; + return file; } + struct clang_preprocess_result + { + std::string file; + std::vector included_files; // needed for pre-clang 4.0.0 + }; + + clang_preprocess_result clang_preprocess_impl(const libclang_compile_config& c, + const char* full_path, const char* macro_path) + { + clang_preprocess_result result; + + std::string diagnostic; + auto diagnostic_handler = [&](const char* str, std::size_t n) { + diagnostic.reserve(diagnostic.size() + n); + for (auto end = str + n; str != end; ++str) + if (*str == '\r') + continue; + else if (*str == '\n') + { + // handle current diagnostic + auto file = parse_missing_file(full_path, diagnostic); + if (file) + // save for clang without -dI flag + result.included_files.push_back(file.value()); + + diagnostic.clear(); + } + else + diagnostic.push_back(*str); + }; + + auto cmd = get_preprocess_command(c, full_path, macro_path); + tpl::Process process(cmd, "", + [&](const char* str, std::size_t n) { + result.file.reserve(result.file.size() + n); + for (auto ptr = str; ptr != str + n; ++ptr) + if (*ptr == '\t') + result.file += " "; // convert to two spaces + else if (*ptr != '\r') + result.file += *ptr; + }, + diagnostic_handler); + // wait for process end + process.get_exit_status(); + + return result; + } + + clang_preprocess_result clang_preprocess(const libclang_compile_config& c, + const char* full_path, const diagnostic_logger& logger) + { + auto fast_preprocessing = detail::libclang_compile_config_access::fast_preprocessing(c); + + auto macro_file = fast_preprocessing ? write_macro_file(c, full_path, logger) : ""; + + clang_preprocess_result result; + try + { + result = clang_preprocess_impl(c, full_path, + fast_preprocessing ? macro_file.c_str() : nullptr); + } + catch (...) + { + if (fast_preprocessing) + { + auto err = std::remove(macro_file.c_str()); + DEBUG_ASSERT(err == 0, detail::assert_handler{}); + } + throw; + } + + if (fast_preprocessing) + { + auto err = std::remove(macro_file.c_str()); + DEBUG_ASSERT(err == 0, detail::assert_handler{}); + } + + return result; + } + + //==== parsing ===// class position { public: @@ -282,6 +443,11 @@ namespace write_.try_reset(); } + bool write_enabled() const noexcept + { + return write_ == true; + } + explicit operator bool() const noexcept { return *ptr_ != '\0'; @@ -331,7 +497,7 @@ namespace p.skip(std::strlen(str)); } - void skip_spaces(position& p, bool bump = false) + void bump_spaces(position& p, bool bump = false) { while (starts_with(p, " ")) if (bump) @@ -340,80 +506,6 @@ namespace p.skip(); } - struct linemarker - { - std::string file; - unsigned line; - enum - { - line_directive, // no change in file - enter_new, // open a new file - enter_old, // return to an old file - } flag = line_directive; - bool is_system = false; - }; - - ts::optional parse_linemarker(position& p) - { - // format (at new line): # "" - // flag 1: enter_new - // flag 2: enter_old - // flag 3: system file - // flag 4: ignored - if (!p.was_newl() || !starts_with(p, "#")) - return ts::nullopt; - p.skip(); - DEBUG_ASSERT(!starts_with(p, "define") && !starts_with(p, "undef") - && !starts_with(p, "pragma"), - detail::assert_handler{}, "handle macros first"); - - linemarker result; - - std::string line; - for (skip_spaces(p); std::isdigit(*p.ptr()); p.skip()) - line += *p.ptr(); - result.line = unsigned(std::stoi(line)); - - skip_spaces(p); - DEBUG_ASSERT(*p.ptr() == '"', detail::assert_handler{}); - p.skip(); - - std::string file_name; - for (; !starts_with(p, "\""); p.skip()) - file_name += *p.ptr(); - p.skip(); - result.file = std::move(file_name); - - for (; !starts_with(p, "\n"); p.skip()) - { - skip_spaces(p); - - switch (*p.ptr()) - { - case '1': - DEBUG_ASSERT(result.flag == linemarker::line_directive, detail::assert_handler{}); - result.flag = linemarker::enter_new; - break; - case '2': - DEBUG_ASSERT(result.flag == linemarker::line_directive, detail::assert_handler{}); - result.flag = linemarker::enter_old; - break; - case '3': - result.is_system = true; - break; - case '4': - break; // ignored - - default: - DEBUG_UNREACHABLE(detail::assert_handler{}, "invalid line marker"); - break; - } - } - skip(p, "\n"); - - return result; - } - detail::pp_doc_comment parse_c_doc_comment(position& p) { detail::pp_doc_comment result; @@ -493,7 +585,7 @@ namespace return result; } - bool skip_c_comment(position& p, detail::preprocessor_output& output, bool in_main_file) + bool skip_c_comment(position& p, detail::preprocessor_output& output) { if (!starts_with(p, "/*")) return false; @@ -502,7 +594,7 @@ namespace if (starts_with(p, "*/")) // empty comment p.skip(2u); - else if (in_main_file && (starts_with(p, "*") || starts_with(p, "!"))) + else if (starts_with(p, "*") || starts_with(p, "!")) { // doc comment p.skip(); @@ -561,20 +653,20 @@ namespace } } - bool skip_cpp_comment(position& p, detail::preprocessor_output& output, bool in_main_file) + bool skip_cpp_comment(position& p, detail::preprocessor_output& output) { if (!starts_with(p, "//")) return false; p.skip(2u); - if (in_main_file && (starts_with(p, "/") || starts_with(p, "!"))) + if (starts_with(p, "/") || starts_with(p, "!")) { // C++ style doc comment p.skip(); auto comment = parse_cpp_doc_comment(p, false); merge_or_add(output, std::move(comment)); } - else if (in_main_file && starts_with(p, "<")) + else if (starts_with(p, "<")) { // end of line doc comment p.skip(); @@ -591,8 +683,7 @@ namespace } std::unique_ptr parse_macro(position& p, - detail::preprocessor_output& output, - bool in_main_file) + detail::preprocessor_output& output) { // format (at new line): #define [replacement] // or: #define () [replacement] @@ -602,7 +693,7 @@ namespace // read line here for comment matching auto cur_line = p.cur_line(); p.bump(std::strlen("#define")); - skip_spaces(p, true); + bump_spaces(p, true); std::string name; while (!starts_with(p, "(") && !starts_with(p, " ") && !starts_with(p, "\n")) @@ -623,7 +714,7 @@ namespace std::string rep; auto in_c_comment = false; - for (skip_spaces(p, true); in_c_comment || !starts_with(p, "\n"); p.bump()) + for (bump_spaces(p, true); in_c_comment || !starts_with(p, "\n"); p.bump()) { if (starts_with(p, "/*")) in_c_comment = true; @@ -633,7 +724,7 @@ namespace } // don't skip newline - if (!in_main_file) + if (!p.write_enabled()) return nullptr; auto result = cpp_macro_definition::build(std::move(name), std::move(args), std::move(rep)); @@ -655,24 +746,24 @@ namespace p.bump(std::strlen("#undef")); std::string result; - for (skip_spaces(p, true); !starts_with(p, "\n"); p.bump()) + for (bump_spaces(p, true); !starts_with(p, "\n"); p.bump()) result += *p.ptr(); // don't skip newline return result; } - type_safe::optional parse_include(position& p, bool in_main_file) + type_safe::optional parse_include(position& p) { // format (at new line, literal <>): #include // or: #include "filename" - // note: don't write include back + // note: write include back if (!p.was_newl() || !starts_with(p, "#include")) return type_safe::nullopt; - p.skip(std::strlen("#include")); + p.bump(std::strlen("#include")); if (starts_with(p, "_next")) - p.skip(std::strlen("_next")); - skip_spaces(p); + p.bump(std::strlen("_next")); + bump_spaces(p); auto include_kind = cpp_include_kind::system; auto end_str = ""; @@ -688,21 +779,18 @@ namespace } else DEBUG_UNREACHABLE(detail::assert_handler{}); - p.skip(); + p.bump(); std::string filename; - for (; !starts_with(p, "\"") && !starts_with(p, ">"); p.skip()) + for (; !starts_with(p, "\"") && !starts_with(p, ">"); p.bump()) filename += *p.ptr(); DEBUG_ASSERT(starts_with(p, end_str, std::strlen(end_str)), detail::assert_handler{}, "bad termination"); - p.skip(); + p.bump(); skip(p, " /* clang -E -dI */"); DEBUG_ASSERT(starts_with(p, "\n"), detail::assert_handler{}); // don't skip newline - if (!in_main_file) - return type_safe::nullopt; - if (filename.size() > 2u && filename[0] == '.' && (filename[1] == '/' || filename[1] == '\\')) filename = filename.substr(2); @@ -722,6 +810,80 @@ namespace return true; } + + struct linemarker + { + std::string file; + unsigned line; + enum + { + line_directive, // no change in file + enter_new, // open a new file + enter_old, // return to an old file + } flag = line_directive; + bool is_system = false; + }; + + ts::optional parse_linemarker(position& p) + { + // format (at new line): # "" + // flag 1: enter_new + // flag 2: enter_old + // flag 3: system file + // flag 4: ignored + if (!p.was_newl() || !starts_with(p, "#")) + return ts::nullopt; + p.skip(); + DEBUG_ASSERT(!starts_with(p, "define") && !starts_with(p, "undef") + && !starts_with(p, "pragma"), + detail::assert_handler{}, "handle macros first"); + + linemarker result; + + std::string line; + for (bump_spaces(p); std::isdigit(*p.ptr()); p.skip()) + line += *p.ptr(); + result.line = unsigned(std::stoi(line)); + + bump_spaces(p); + DEBUG_ASSERT(*p.ptr() == '"', detail::assert_handler{}); + p.skip(); + + std::string file_name; + for (; !starts_with(p, "\""); p.skip()) + file_name += *p.ptr(); + p.skip(); + result.file = std::move(file_name); + + for (; !starts_with(p, "\n"); p.skip()) + { + bump_spaces(p); + + switch (*p.ptr()) + { + case '1': + DEBUG_ASSERT(result.flag == linemarker::line_directive, detail::assert_handler{}); + result.flag = linemarker::enter_new; + break; + case '2': + DEBUG_ASSERT(result.flag == linemarker::line_directive, detail::assert_handler{}); + result.flag = linemarker::enter_old; + break; + case '3': + result.is_system = true; + break; + case '4': + break; // ignored + + default: + DEBUG_UNREACHABLE(detail::assert_handler{}, "invalid line marker"); + break; + } + } + p.skip(); + + return result; + } } detail::preprocessor_output detail::preprocess(const libclang_compile_config& config, @@ -729,11 +891,10 @@ detail::preprocessor_output detail::preprocess(const libclang_compile_config& co { detail::preprocessor_output result; - auto output = get_full_preprocess_output(config, path, logger); + auto preprocessed = clang_preprocess(config, path, logger); - position p(ts::ref(result.source), output.c_str()); - std::size_t file_depth = 0u; - ts::flag in_string(false), in_char(false); + position p(ts::ref(result.source), preprocessed.file.c_str()); + ts::flag in_string(false), in_char(false), first_line(true); while (p) { auto next = std::strpbrk(p.ptr(), R"(\"'#/)"); // look for \, ", ', # or / @@ -742,6 +903,8 @@ detail::preprocessor_output detail::preprocess(const libclang_compile_config& co if (starts_with(p, R"(\\)")) // starts with two backslashes p.bump(2u); + else if (starts_with(p, "\\\"")) // starts with \" + p.bump(2u); else if (starts_with(p, R"(\")")) // starts with \" p.bump(2u); else if (starts_with(p, R"(\')")) // starts with \' @@ -758,7 +921,7 @@ detail::preprocessor_output detail::preprocess(const libclang_compile_config& co } else if (in_string == true || in_char == true) p.bump(); - else if (auto macro = parse_macro(p, result, file_depth == 0u)) + else if (auto macro = parse_macro(p, result)) { if (logger.is_verbose()) logger.log("preprocessor", @@ -770,7 +933,7 @@ detail::preprocessor_output detail::preprocess(const libclang_compile_config& co } else if (auto undef = parse_undef(p)) { - if (file_depth == 0u) + if (p.write_enabled()) { if (logger.is_verbose()) logger.log("preprocessor", @@ -785,68 +948,71 @@ detail::preprocessor_output detail::preprocess(const libclang_compile_config& co result.macros.end()); } } - else if (auto include = parse_include(p, file_depth == 0u)) + else if (auto include = parse_include(p)) { - if (logger.is_verbose()) - logger.log("preprocessor", - format_diagnostic(severity::debug, - source_location::make_file(path, p.cur_line()), - "parsing include '", include.value().file_name, "'")); + if (p.write_enabled()) + { + if (logger.is_verbose()) + logger.log("preprocessor", + format_diagnostic(severity::debug, + source_location::make_file(path, p.cur_line()), + "parsing include '", include.value().file_name, + "'")); - result.includes.push_back(std::move(include.value())); + result.includes.push_back(std::move(include.value())); + } } else if (bump_pragma(p)) continue; else if (auto lm = parse_linemarker(p)) { - switch (lm.value().flag) - { - case linemarker::line_directive: - if (file_depth == 0u) - p.set_line(lm.value().line); - break; - - case linemarker::enter_new: - if (file_depth == 0u && lm.value().file.front() != '<') - { - // this file is directly included by the given file - // and it is not a fake file like builtin or command line - - // write include with full path - p.write_str("#include \"" + lm.value().file + "\"\n"); - // note: don't build include here, do it when an #include is encountered - } - - ++file_depth; + if (lm.value().flag == linemarker::enter_new) p.disable_write(); - break; - - case linemarker::enter_old: - --file_depth; - if (file_depth == 0u) - { - DEBUG_ASSERT(lm.value().file == path, detail::assert_handler{}); - p.set_line(lm.value().line); + else if (lm.value().flag == linemarker::enter_old) + { + if (lm.value().file == path) p.enable_write(); + } + else if (lm.value().flag == linemarker::line_directive && p.write_enabled()) + { + if (first_line.try_reset() && lm.value().file == path && lm.value().line == 1u) + { + // this is the first line marker + // just skip all builtin macro stuff until we reach the file again + auto closing_line_marker = std::string("# 1 \"") + path + "\" 2\n"; + + auto ptr = std::strstr(p.ptr(), closing_line_marker.c_str()); + DEBUG_ASSERT(ptr, detail::assert_handler{}); + p.skip(std::size_t(ptr - p.ptr())); + p.skip(closing_line_marker.size()); + } + else if (lm.value().line + 1 == p.cur_line()) + { + // this is a linemarker after a -dI injected include directive + // it simply adds the newline we already did + } + else + { + DEBUG_ASSERT(lm.value().line >= p.cur_line(), detail::assert_handler{}); + while (p.cur_line() < lm.value().line) + p.write_str("\n"); } - break; } } - else if (skip_c_comment(p, result, file_depth == 0u)) + else if (skip_c_comment(p, result)) continue; - else if (skip_cpp_comment(p, result, file_depth == 0u)) + else if (skip_cpp_comment(p, result)) continue; else p.bump(); } + if (result.includes.empty()) + { + // add headers from diagnostics w/o line information + for (auto name : preprocessed.included_files) + result.includes.push_back(pp_include{name, cpp_include_kind::local, 1u}); + } + return result; } - -bool detail::pp_doc_comment::matches(const cpp_entity&, unsigned e_line) -{ - if (kind == detail::pp_doc_comment::end_of_line) - return line == e_line; - else - return line + 1u == e_line; -} diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 0475d8d..eed8c47 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -34,10 +34,19 @@ set(tests preprocessor.cpp visitor.cpp) +# generate list of source files for the self parsing test +get_target_property(files cppast SOURCES) +file(WRITE ${CMAKE_CURRENT_BINARY_DIR}/cppast_files.hpp "// list of cppast source file includes\n") +foreach(file ${files}) + file(APPEND ${CMAKE_CURRENT_BINARY_DIR}/cppast_files.hpp "\"${CMAKE_CURRENT_SOURCE_DIR}/../src/${file}\",\n") +endforeach() + add_executable(cppast_test test.cpp test_parser.hpp ${tests}) target_include_directories(cppast_test PUBLIC ${CMAKE_CURRENT_BINARY_DIR}) target_include_directories(cppast_test PRIVATE ${CMAKE_CURRENT_LIST_DIR}/../src) target_link_libraries(cppast_test PUBLIC cppast) +target_compile_definitions(cppast_test PUBLIC CPPAST_INTEGRATION_FILE="${CMAKE_CURRENT_SOURCE_DIR}/integration.cpp" + CPPAST_COMPILE_COMMANDS="${CMAKE_BINARY_DIR}") set_target_properties(cppast_test PROPERTIES CXX_STANDARD 11) enable_testing() diff --git a/test/cpp_function.cpp b/test/cpp_function.cpp index aa73a33..43c8ebd 100644 --- a/test/cpp_function.cpp +++ b/test/cpp_function.cpp @@ -48,6 +48,8 @@ namespace ns /// void l(); void l(); + + using m = int; } /// void ns::l(); @@ -56,6 +58,9 @@ void ns::l() // might confuse parser auto b = noexcept(g()); } + +/// ns::m m(); +ns::m m(); )"; auto check_body = [](const cpp_function& func, cpp_function_body_kind kind) { @@ -223,10 +228,22 @@ void ns::l() check_body(func, cpp_function_declaration); } } + else if (func.name() == "m") + { + REQUIRE(equal_types(idx, func.return_type(), + *cpp_user_defined_type::build( + cpp_type_ref(cpp_entity_id(""), "ns::m")))); + REQUIRE(count_children(func.parameters()) == 0u); + REQUIRE(!func.is_variadic()); + REQUIRE(!func.noexcept_condition()); + REQUIRE(!func.is_constexpr()); + REQUIRE(func.storage_class() == cpp_storage_class_none); + check_body(func, cpp_function_declaration); + } else REQUIRE(false); }); - REQUIRE(count == 13u); + REQUIRE(count == 14u); } TEST_CASE("static cpp_function") diff --git a/test/cpp_preprocessor.cpp b/test/cpp_preprocessor.cpp index c3f4991..9805099 100644 --- a/test/cpp_preprocessor.cpp +++ b/test/cpp_preprocessor.cpp @@ -81,7 +81,8 @@ namespace ns2 TEST_CASE("cpp_include_directive", "[!hide][clang4]") { write_file("cpp_include_directive-header.hpp", R"( -#define FOO +#define FOO a\ +b )"); auto header_a = R"( @@ -114,7 +115,7 @@ TEST_CASE("cpp_include_directive", "[!hide][clang4]") REQUIRE(include.target().name() == include.name()); REQUIRE(include.include_kind() == cppast::cpp_include_kind::local); REQUIRE(include.target().get(idx).empty()); - REQUIRE(include.full_path() == "./cpp_include_directive-header.hpp"); + REQUIRE(include.full_path() == "cpp_include_directive-header.hpp"); } else REQUIRE(false); @@ -128,7 +129,7 @@ TEST_CASE("cpp_include_directive", "[!hide][clang4]") REQUIRE(include.include_kind() == cppast::cpp_include_kind::local); REQUIRE( equal_ref(idx, include.target(), cpp_file_ref(cpp_entity_id(""), "header_a.hpp"))); - REQUIRE(include.full_path() == "./header_a.hpp"); + REQUIRE(include.full_path() == "header_a.hpp"); } else REQUIRE(false); @@ -146,7 +147,7 @@ TEST_CASE("preprocessor line numbers") #include -int foo; +int var; /// 11 diff --git a/test/integration.cpp b/test/integration.cpp index f20f091..2f1efa1 100644 --- a/test/integration.cpp +++ b/test/integration.cpp @@ -110,6 +110,23 @@ TEST_CASE("stdlib", "[!hide][integration]") REQUIRE(!parser.error()); REQUIRE(file); - resolve_includes(parser, file.value(), config); + REQUIRE(resolve_includes(parser, file.value(), config) == 62); + REQUIRE(!parser.error()); +} + +TEST_CASE("cppast", "[!hide][integration]") +{ + const char* files[] = { +#include + }; + + cpp_entity_index idx; + simple_file_parser parser(type_safe::ref(idx), default_logger()); + + libclang_compilation_database database(CPPAST_COMPILE_COMMANDS); + libclang_compile_config config(database, CPPAST_INTEGRATION_FILE); + config.fast_preprocessing(true); + parse_files(parser, files, config); + REQUIRE(!parser.error()); } diff --git a/tool/main.cpp b/tool/main.cpp index d43bbcc..a39def3 100644 --- a/tool/main.cpp +++ b/tool/main.cpp @@ -256,8 +256,13 @@ int main(int argc, char* argv[]) try { auto equal = macro.find('='); auto name = macro.substr(0, equal); - auto def = equal == std::string::npos ? macro.substr(equal + 1u) : ""; - config.define_macro(std::move(name), std::move(def)); + if (equal == std::string::npos) + config.define_macro(std::move(name), ""); + else + { + auto def = macro.substr(equal + 1u); + config.define_macro(std::move(name), std::move(def)); + } } if (options.count("macro_undefinition")) for (auto& name : options["macro_undefinition"].as>())