Ensure matching line numbers in preprocessed file

include/cppast/cpp_file.hpp

@@ -13,6 +13,18 @@
 
 namespace cppast
 {
+    /// An unmatched documentation comment.
+    struct cpp_doc_comment
+    {
+        std::string content;
+        unsigned    line;
+
+        cpp_doc_comment(std::string content, unsigned line)
+        : content(std::move(content)), line(line)
+        {
+        }
+    };
+
     /// A [cppast::cpp_entity]() modelling a file.
     ///
     /// This is the top-level entity of the AST.
@@ -26,9 +38,7 @@ namespace cppast
         {
         public:
             /// \effects Sets the file name.
-            explicit builder(std::string name) : file_(new cpp_file(std::move(name)))
+            explicit builder(std::string name) : file_(new cpp_file(std::move(name))) {}
-            {
-            }
 
             /// \effects Adds an entity.
             void add_child(std::unique_ptr<cpp_entity> child) noexcept
@@ -37,9 +47,9 @@ namespace cppast
             }
 
             /// \effects Adds an unmatched documentation comment.
-            void add_unmatched_comment(std::string str)
+            void add_unmatched_comment(cpp_doc_comment comment)
             {
-                file_->comments_.push_back(std::move(str));
+                file_->comments_.push_back(std::move(comment));
             }
 
             /// \returns The not yet finished file.
@@ -62,20 +72,18 @@ namespace cppast
         };
 
         /// \returns The unmatched documentation comments.
-        type_safe::array_ref<const std::string> unmatched_comments() const noexcept
+        type_safe::array_ref<const cpp_doc_comment> unmatched_comments() const noexcept
         {
             return type_safe::ref(comments_.data(), comments_.size());
         }
 
     private:
-        cpp_file(std::string name) : cpp_entity(std::move(name))
+        cpp_file(std::string name) : cpp_entity(std::move(name)) {}
-        {
-        }
 
         /// \returns [cpp_entity_type::file_t]().
         cpp_entity_kind do_get_entity_kind() const noexcept override;
 
-        std::vector<std::string> comments_;
+        std::vector<cpp_doc_comment> comments_;
     };
 
     /// \exclude

src/libclang/libclang_parser.cpp

@@ -403,9 +403,8 @@ namespace
         auto args = get_arguments(config);
 
         CXTranslationUnit tu;
-        auto              flags = CXTranslationUnit_Incomplete | CXTranslationUnit_KeepGoing;
+        auto              flags = CXTranslationUnit_Incomplete | CXTranslationUnit_KeepGoing
-        if (detail::libclang_compile_config_access::clang_version(config) >= 40000)
+                     | CXTranslationUnit_DetailedPreprocessingRecord;
-            flags |= CXTranslationUnit_DetailedPreprocessingRecord;
 
         auto error =
             clang_parseTranslationUnit2(idx.get(), path, // index and path
@@ -479,17 +478,20 @@ std::unique_ptr<cpp_file> libclang_parser::do_parse(const cpp_entity_index& idx,
     detail::visit_tu(tu, path.c_str(), [&](const CXCursor& cur) {
         if (clang_getCursorKind(cur) == CXCursor_InclusionDirective)
         {
-            DEBUG_ASSERT(include_iter != preprocessed.includes.end()
+            if (!preprocessed.includes.empty())
-                             && get_line_no(cur) >= include_iter->line,
+            {
-                         detail::assert_handler{});
+                DEBUG_ASSERT(include_iter != preprocessed.includes.end()
+                                 && get_line_no(cur) >= include_iter->line,
+                             detail::assert_handler{});
 
-            auto include =
+                auto include =
-                cpp_include_directive::build(std::move(include_iter->file), include_iter->kind,
+                    cpp_include_directive::build(std::move(include_iter->file), include_iter->kind,
-                                             detail::get_cursor_name(cur).c_str());
+                                                 detail::get_cursor_name(cur).c_str());
-            context.comments.match(*include, include_iter->line);
+                context.comments.match(*include, include_iter->line);
-            builder.add_child(std::move(include));
+                builder.add_child(std::move(include));
 
-            ++include_iter;
+                ++include_iter;
+            }
         }
         else if (clang_getCursorKind(cur) != CXCursor_MacroDefinition)
         {
@@ -510,7 +512,7 @@ std::unique_ptr<cpp_file> libclang_parser::do_parse(const cpp_entity_index& idx,
     for (auto& c : preprocessed.comments)
     {
         if (!c.comment.empty())
-            builder.add_unmatched_comment(std::move(c.comment));
+            builder.add_unmatched_comment(cpp_doc_comment(std::move(c.comment), c.line));
     }
 
     if (context.error)

src/libclang/parse_error.hpp

@@ -19,7 +19,13 @@ namespace cppast
     {
         inline source_location make_location(const CXCursor& cur)
         {
-            return source_location::make_entity(get_display_name(cur).c_str());
+            auto loc = clang_getCursorLocation(cur);
+
+            CXString file;
+            unsigned line;
+            clang_getPresumedLocation(loc, &file, &line, nullptr);
+
+            return source_location::make_file(cxstring(file).c_str(), line);
         }
 
         inline source_location make_location(const CXType& type)

src/libclang/parse_functions.cpp

@@ -80,7 +80,7 @@ void detail::comment_context::match(cpp_entity& e, const CXCursor& cur) const
 {
     auto     pos = clang_getRangeStart(clang_getCursorExtent(cur));
     unsigned line;
-    clang_getSpellingLocation(pos, nullptr, &line, nullptr, nullptr);
+    clang_getPresumedLocation(pos, nullptr, &line, nullptr);
 
     match(e, line);
 }

src/libclang/preprocessor.cpp

@@ -30,6 +30,11 @@ namespace
         return '"' + std::move(str) + '"';
     }
 
+    bool support_include(const libclang_compile_config& c)
+    {
+        return detail::libclang_compile_config_access::clang_version(c) >= 40000;
+    }
+
     // build the command that runs the preprocessor
     std::string get_command(const libclang_compile_config& c, const char* full_path)
     {
@@ -39,7 +44,7 @@ namespace
         // -C: keep comments
         // -dD: print macro definitions as well
         auto flags = std::string("-x c++ -I. -E -C -dD");
-        if (detail::libclang_compile_config_access::clang_version(c) >= 40000)
+        if (support_include(c))
             // -Xclang -dI: print include directives as well (clang >= 4.0.0)
             flags += " -Xclang -dI";
         // -fno-caret-diagnostics: don't show the source extract in diagnostics
@@ -191,6 +196,12 @@ namespace
         {
         }
 
+        void set_line(unsigned line)
+        {
+            *result_ += "#line " + std::to_string(line) + "\n";
+            cur_line_ = line;
+        }
+
         void write_str(std::string str)
         {
             if (write_ == false)
@@ -363,7 +374,7 @@ namespace
                 break;
             }
         }
-        p.skip();
+        skip(p, "\n");
 
         return result;
     }
@@ -691,7 +702,8 @@ detail::preprocessor_output detail::preprocess(const libclang_compile_config& co
             {
                 auto message = detail::format("parsing macro '", macro->name(), "'");
                 logger.log("preprocessor",
-                           diagnostic{std::move(message), source_location::make_file(path),
+                           diagnostic{std::move(message),
+                                      source_location::make_file(path, p.cur_line()),
                                       severity::debug});
             }
 
@@ -705,7 +717,8 @@ detail::preprocessor_output detail::preprocess(const libclang_compile_config& co
                 {
                     auto message = detail::format("undefining macro '", undef.value(), "'");
                     logger.log("preprocessor",
-                               diagnostic{std::move(message), source_location::make_file(path),
+                               diagnostic{std::move(message),
+                                          source_location::make_file(path, p.cur_line()),
                                           severity::debug});
                 }
                 result.macros.erase(std::remove_if(result.macros.begin(), result.macros.end(),
@@ -722,7 +735,8 @@ detail::preprocessor_output detail::preprocess(const libclang_compile_config& co
                 auto message =
                     detail::format("parsing include '", include.value().file.name(), "'");
                 logger.log("preprocessor",
-                           diagnostic{std::move(message), source_location::make_file(path),
+                           diagnostic{std::move(message),
+                                      source_location::make_file(path, p.cur_line()),
                                       severity::debug});
             }
             result.includes.push_back(std::move(include.value()));
@@ -734,16 +748,19 @@ detail::preprocessor_output detail::preprocess(const libclang_compile_config& co
             switch (lm.value().flag)
             {
             case linemarker::line_directive:
-                break; // ignore
+                if (file_depth == 0u)
-                // no need to handle it, preprocessed output doesn't need to match line numbers precisely
+                    p.set_line(lm.value().line);
+                break;
 
             case linemarker::enter_new:
                 if (file_depth == 0u && lm.value().file.front() != '<')
                 {
                     // this file is directly included by the given file
+                    // and it is not a fake file like builtin or command line
+
                     // write include with full path
-                    // note: don't build include here, do it when an #include is encountered
                     p.write_str("#include \"" + lm.value().file + "\"\n");
+                    // note: don't build include here, do it when an #include is encountered
                 }
 
                 ++file_depth;
@@ -755,6 +772,7 @@ detail::preprocessor_output detail::preprocess(const libclang_compile_config& co
                 if (file_depth == 0u)
                 {
                     DEBUG_ASSERT(lm.value().file == path, detail::assert_handler{});
+                    p.set_line(lm.value().line);
                     p.enable_write();
                 }
                 break;

src/libclang/type_parser.cpp

@@ -553,7 +553,7 @@ namespace
         {
             auto msg      = detail::format("unexpected type of kind '",
                                       detail::get_type_kind_spelling(type).c_str(), "'");
-            auto location = source_location::make_entity(get_type_spelling(type).c_str());
+            auto location = detail::make_location(type);
             context.logger->log("libclang parser", diagnostic{msg, location, severity::warning});
         }
         // fallthrough

test/cpp_preprocessor.cpp

@@ -136,6 +136,43 @@ TEST_CASE("cpp_include_directive", "[!hide][clang4]")
     REQUIRE(count == 1u);
 }
 
+TEST_CASE("preprocessor line numbers")
+{
+    auto code = R"(/// 1
+
+#include <iostream>
+
+/// 5
+
+#include <string>
+
+int foo;
+
+/// 11
+
+#define foo \
+    \
+    \
+    int \
+    main()
+
+/// 19
+
+foo {}
+
+/// 23
+
+#include <vector>
+
+/// 27
+)";
+
+    auto file = parse({}, "preprocessor_line_numbers.cpp", code);
+    for (auto& comment : file->unmatched_comments())
+        REQUIRE(comment.line == std::stoi(comment.content));
+    REQUIRE((file->unmatched_comments().size() == 6u));
+}
+
 TEST_CASE("comment matching")
 {
     auto code = R"(
@@ -205,6 +242,6 @@ void j();
     });
 
     for (auto& comment : file->unmatched_comments())
-        REQUIRE(comment == "u");
+        REQUIRE(comment.content == "u");
     REQUIRE((file->unmatched_comments().size() == 3u));
 }