Log diagnostic when a file is not found

include/cppast/libclang_parser.hpp

@@ -137,6 +137,8 @@ namespace cppast
         /// Later stages will use the includes again.
         /// This hack breaks if you define the same macro multiple times in the file being parsed (headers don't matter)
         /// or you rely on the order of macro directives.
+        /// \notes If this option is `true`, the full file name of include directives is not available,
+        /// just the name as written in the source code.
         void fast_preprocessing(bool b) noexcept
         {
             fast_preprocessing_ = b;

src/libclang/cxtokenizer.cpp

@@ -243,6 +243,11 @@ namespace
         {
             // variadic tokens in unnamed parameter not included
             end = get_next_location(tu, file, end, 3);
+            if (token_after_is(tu, file, cur, end, "."))
+                // extra whitespace, so bump again
+                // this should all go away once I redid the whole token thing...
+                end = get_next_location(tu, file, end, 1);
+
             DEBUG_ASSERT(token_after_is(tu, file, cur, end, ">")
                              || token_after_is(tu, file, cur, end, ","),
                          detail::parse_error_handler{}, cur,

src/libclang/libclang_parser.cpp

@@ -516,8 +516,9 @@ std::unique_ptr<cpp_file> libclang_parser::do_parse(const cpp_entity_index& idx,
                                  && get_line_no(cur) >= include_iter->line,
                              detail::assert_handler{});
 
-                // create an include directive
+                auto full_path = include_iter->full_path.empty() ? include_iter->file_name :
-                auto full_path = detail::get_cursor_name(cur);
+                                                                   include_iter->full_path;
+
                 // if we got an absolute file path for the current file,
                 // also use an absolute file path for the id
                 // otherwise just use the file name as written in the source file
@@ -531,7 +532,7 @@ std::unique_ptr<cpp_file> libclang_parser::do_parse(const cpp_entity_index& idx,
                 auto include =
                     cpp_include_directive::build(cpp_file_ref(id,
                                                               std::move(include_iter->file_name)),
-                                                 include_iter->kind, full_path.std_str());
+                                                 include_iter->kind, std::move(full_path));
                 context.comments.match(*include, include_iter->line,
                                        false); // must not skip comments,
                                                // includes are not reported in order

src/libclang/preprocessor.cpp

@@ -9,8 +9,10 @@
 #include <cctype>
 #include <cstdio>
 #include <cstring>
-#include <process.hpp>
 #include <fstream>
+#include <unordered_map>
+
+#include <process.hpp>
 
 #include <cppast/diagnostic.hpp>
 
@@ -393,13 +395,15 @@ namespace
     };
 
     clang_preprocess_result clang_preprocess_impl(const libclang_compile_config& c,
+                                                  const diagnostic_logger&       logger,
                                                   const std::string&             full_path,
                                                   const char*                    macro_path)
     {
         clang_preprocess_result result;
 
         std::string diagnostic;
-        auto        diagnostic_handler = [&](const char* str, std::size_t n) {
+        auto        expect_bad_exit_code = false;
+        auto        diagnostic_handler   = [&](const char* str, std::size_t n) {
             diagnostic.reserve(diagnostic.size() + n);
             for (auto end = str + n; str != end; ++str)
                 if (*str == '\r')
@@ -407,10 +411,19 @@ namespace
                 else if (*str == '\n')
                 {
                     // handle current diagnostic
-                    auto file = parse_missing_file(full_path, diagnostic);
+                    if (macro_path)
-                    if (file)
+                    {
-                        // save for clang without -dI flag
+                        // hide diagnostics
-                        result.included_files.push_back(file.value());
+
+                        auto file = parse_missing_file(full_path, diagnostic);
+                        if (file)
+                            // save for clang without -dI flag
+                            result.included_files.push_back(file.value());
+
+                        expect_bad_exit_code = true;
+                    }
+                    else
+                        log_diagnostic(logger, diagnostic);
 
                     diagnostic.clear();
                 }
@@ -430,7 +443,12 @@ namespace
                              },
                              diagnostic_handler);
         // wait for process end
-        process.get_exit_status();
+        auto exit_code = process.get_exit_status();
+        DEBUG_ASSERT(diagnostic.empty(), detail::assert_handler{});
+        if (exit_code != 0 && !expect_bad_exit_code)
+            throw libclang_error("preprocessor: command '" + cmd
+                                 + "' exited with non-zero exit code (" + std::to_string(exit_code)
+                                 + ")");
 
         return result;
     }
@@ -438,6 +456,10 @@ namespace
     clang_preprocess_result clang_preprocess(const libclang_compile_config& c,
                                              const char* full_path, const diagnostic_logger& logger)
     {
+        if (!std::ifstream(full_path))
+            throw libclang_error("preprocessor: file '" + std::string(full_path)
+                                 + "' doesn't exist");
+
         // if we're fast preprocessing we only preprocess the main file, not includes
         // this is done by disabling all include search paths when doing the preprocessing
         // to allow macros a separate preprocessing with the -dM flag is done that extracts all macros
@@ -449,7 +471,7 @@ namespace
         clang_preprocess_result result;
         try
         {
-            result = clang_preprocess_impl(c, full_path,
+            result = clang_preprocess_impl(c, logger, full_path,
                                            fast_preprocessing ? macro_file.c_str() : nullptr);
         }
         catch (...)
@@ -921,7 +943,7 @@ namespace
             && (filename[1] == '/' || filename[1] == '\\'))
             filename = filename.substr(2);
 
-        return detail::pp_include{std::move(filename), include_kind, p.cur_line()};
+        return detail::pp_include{std::move(filename), "", include_kind, p.cur_line()};
     }
 
     bool bump_pragma(position& p)
@@ -1015,10 +1037,18 @@ namespace
 detail::preprocessor_output detail::preprocess(const libclang_compile_config& config,
                                                const char* path, const diagnostic_logger& logger)
 {
-    detail::preprocessor_output result;
+    detail::preprocessor_output                  result;
+    std::unordered_map<std::string, std::string> indirect_includes;
 
     auto preprocessed = clang_preprocess(config, path, logger);
 
+    if (detail::libclang_compile_config_access::clang_version(config) < 40000)
+    {
+        // add headers from diagnostics w/o line information
+        for (auto name : preprocessed.included_files)
+            result.includes.push_back(pp_include{name, "", cpp_include_kind::local, 1u});
+    }
+
     position p(ts::ref(result.source), preprocessed.file.c_str());
     ts::flag in_string(false), in_char(false), first_line(true);
     while (p)
@@ -1093,7 +1123,35 @@ detail::preprocessor_output detail::preprocess(const libclang_compile_config& co
         else if (auto lm = parse_linemarker(p))
         {
             if (lm.value().flag == linemarker::enter_new)
+            {
+                if (p.write_enabled())
+                {
+                    // this is a direct include, update the full path of the last include
+                    // note: path can be empty if pre clang 4 and not fast preprocessing
+                    // in this case we can't get the full path at all
+                    if (!result.includes.empty())
+                    {
+                        DEBUG_ASSERT(result.includes.back().full_path.empty()
+                                         && lm.value().file.find(result.includes.back().file_name)
+                                                != std::string::npos,
+                                     detail::assert_handler{});
+                        result.includes.back().full_path = lm.value().file;
+                    }
+                }
+                else
+                {
+                    // this is an indirect include, remember it to get full path for indirect includes
+                    auto& full_path = lm.value().file;
+
+                    auto last_dir = full_path.find_last_of("/\\");
+                    auto file_name =
+                        last_dir == std::string::npos ? full_path : full_path.substr(last_dir + 1u);
+
+                    indirect_includes.emplace(std::move(file_name), full_path);
+                }
+
                 p.disable_write();
+            }
             else if (lm.value().flag == linemarker::enter_old)
             {
                 if (lm.value().file == path)
@@ -1136,11 +1194,28 @@ detail::preprocessor_output detail::preprocess(const libclang_compile_config& co
             p.bump();
     }
 
-    if (result.includes.empty())
+    // get full path for indirect includes
+    // doesn't work if fast preprocessing
+    if (!detail::libclang_compile_config_access::fast_preprocessing(config))
     {
-        // add headers from diagnostics w/o line information
+        for (auto& include : result.includes)
-        for (auto name : preprocessed.included_files)
+            if (include.full_path.empty())
-            result.includes.push_back(pp_include{name, cpp_include_kind::local, 1u});
+            {
+                auto last_sep = include.file_name.find_last_of("/\\");
+
+                auto iter = indirect_includes.find(last_sep == std::string::npos ?
+                                                       include.file_name :
+                                                       include.file_name.substr(last_sep + 1u));
+                if (iter != indirect_includes.end())
+                    include.full_path = iter->second;
+                else
+                    logger.log("preprocessor",
+                               format_diagnostic(severity::warning,
+                                                 source_location::make_file(path, include.line),
+                                                 "unable to retrieve full path for include '",
+                                                 include.file_name,
+                                                 "' (please file a bug report)"));
+            }
     }
 
     return result;

src/libclang/preprocessor.hpp

@@ -20,7 +20,7 @@ namespace cppast
 
         struct pp_include
         {
-            std::string      file_name;
+            std::string      file_name, full_path;
             cpp_include_kind kind;
             unsigned         line;
         };

src/libclang/type_parser.cpp

@@ -582,6 +582,9 @@ namespace
         case CXType_OCLEvent:
         case CXType_OCLQueue:
         case CXType_OCLReserveID:
+#endif
+#if CINDEX_VERSION_MINOR > 43
+        case CXType_Float16:
 #endif
             context.logger->log("libclang parser",
                                 format_diagnostic(severity::warning, detail::make_location(type),

test/cpp_preprocessor.cpp

@@ -115,7 +115,7 @@ b
                 REQUIRE(include.target().name() == include.name());
                 REQUIRE(include.include_kind() == cppast::cpp_include_kind::local);
                 REQUIRE(include.target().get(idx).empty());
-                REQUIRE(include.full_path() == "cpp_include_directive-header.hpp");
+                REQUIRE(include.full_path() == "./cpp_include_directive-header.hpp");
             }
             else
                 REQUIRE(false);
@@ -129,7 +129,7 @@ b
             REQUIRE(include.include_kind() == cppast::cpp_include_kind::local);
             REQUIRE(
                 equal_ref(idx, include.target(), cpp_file_ref(cpp_entity_id(""), "header_a.hpp")));
-            REQUIRE(include.full_path() == "header_a.hpp");
+            REQUIRE(include.full_path() == "./header_a.hpp");
         }
         else
             REQUIRE(false);

test/preprocessor.cpp

@@ -30,6 +30,15 @@ TEST_CASE("preprocessor escaped character", "[!hide][clang4]")
     libclang_compile_config config;
     config.set_flags(cpp_standard::cpp_latest);
 
+    SECTION("fast")
+    {
+        config.fast_preprocessing(true);
+    }
+    SECTION("normal")
+    {
+        config.fast_preprocessing(false);
+    }
+
     auto preprocessed = detail::preprocess(config, "ppec.cpp", default_logger().get());
     REQUIRE(preprocessed.includes.size() == 1);
     REQUIRE(preprocessed.includes[0].file_name == "ppec.hpp");
@@ -37,6 +46,16 @@ TEST_CASE("preprocessor escaped character", "[!hide][clang4]")
 
 TEST_CASE("preprocessing use external macro")
 {
+    bool fast_preprocessing = false;
+    SECTION("fast_preprocessing")
+    {
+        fast_preprocessing = true;
+    }
+    SECTION("normal")
+    {
+        fast_preprocessing = false;
+    }
+
     auto file = parse({}, "preprocessing_external_macro.cpp", R"(
 #include <cmath>
 #ifdef _GLIBCXX_RELEASE
@@ -46,7 +65,7 @@ TEST_CASE("preprocessing use external macro")
 auto result = NAN;
 
 #endif
-)");
+)", fast_preprocessing);
 
     test_visit<cpp_variable>(*file, [&](const cpp_variable&) {});
 }
@@ -77,13 +96,30 @@ struct foo {};
     config.set_flags(cpp_standard::cpp_latest);
     config.fast_preprocessing(true);
 
-    auto result = detail::preprocess(config, file_name, default_logger().get());
+    try
-    REQUIRE(result.macros.size() == 1u);
+    {
-    REQUIRE(result.macros[0].macro->name() == "INCLUDE_GUARD");
+        auto result = detail::preprocess(config, file_name, default_logger().get());
+        REQUIRE(result.macros.size() == 1u);
+        REQUIRE(result.macros[0].macro->name() == "INCLUDE_GUARD");
+    }
+    catch (libclang_error& ex)
+    {
+        FAIL(ex.what());
+    }
 }
 
 TEST_CASE("preprocessor line numbers")
 {
+    bool fast_preprocessing = false;
+    SECTION("fast_preprocessing")
+    {
+        fast_preprocessing = true;
+    }
+    SECTION("normal")
+    {
+        fast_preprocessing = false;
+    }
+
     auto code = R"(/// 1
 
 #include <iostream>
@@ -123,7 +159,7 @@ lines
 /// 37
 )";
 
-    auto file = parse({}, "preprocessor_line_numbers.cpp", code);
+    auto file = parse({}, "preprocessor_line_numbers.cpp", code, fast_preprocessing);
     for (auto& comment : file->unmatched_comments())
     {
         if (comment.content[0] != '\n')
@@ -181,6 +217,16 @@ with indent */
 
 TEST_CASE("comment matching")
 {
+    bool fast_preprocessing = false;
+    SECTION("fast_preprocessing")
+    {
+        fast_preprocessing = true;
+    }
+    SECTION("normal")
+    {
+        fast_preprocessing = false;
+    }
+
     auto code = R"(
 /// u
 
@@ -236,7 +282,7 @@ template <typename T/**/>
 void j();
 )";
 
-    auto file = parse({}, "comment-matching.cpp", code);
+    auto file = parse({}, "comment-matching.cpp", code, fast_preprocessing);
     visit(*file, [&](const cpp_entity& e, visitor_info) {
         if (e.kind() == cpp_entity_kind::file_t)
             return true;

test/test_parser.hpp

@@ -24,12 +24,14 @@ inline void write_file(const char* name, const char* code)
 }
 
 inline std::unique_ptr<cppast::cpp_file> parse_file(const cppast::cpp_entity_index& idx,
-                                                    const char*                     name)
+                                                    const char*                     name,
+                                                    bool fast_preprocessing = false)
 {
     using namespace cppast;
 
     libclang_compile_config config;
     config.set_flags(cpp_standard::cpp_latest);
+    config.fast_preprocessing(fast_preprocessing);
 
     libclang_parser p(default_logger());
 
@@ -40,10 +42,11 @@ inline std::unique_ptr<cppast::cpp_file> parse_file(const cppast::cpp_entity_ind
 }
 
 inline std::unique_ptr<cppast::cpp_file> parse(const cppast::cpp_entity_index& idx,
-                                               const char* name, const char* code)
+                                               const char* name, const char* code,
+                                               bool fast_preprocessing = false)
 {
     write_file(name, code);
-    return parse_file(idx, name);
+    return parse_file(idx, name, fast_preprocessing);
 }
 
 class test_generator : public cppast::code_generator