Parse documentation comments

include/cppast/cpp_class.hpp

@@ -165,6 +165,12 @@ namespace cppast
                 class_->add_child(std::move(child));
             }
 
+            /// \returns The not yet finished class.
+            cpp_class& get() noexcept
+            {
+                return *class_;
+            }
+
             /// \effects Registers the class in the [cppast::cpp_entity_index](),
             /// using the given [cppast::cpp_entity_id]().
             /// \returns The finished class.

include/cppast/cpp_entity.hpp

@@ -50,6 +50,44 @@ namespace cppast
             return parent_;
         }
 
+        /// \returns The documentation comment associated with that entity, if any.
+        /// \notes A documentation comment can have three forms:
+        ///
+        /// * A C style doc comment. It is a C style comment starting with an additional `*`, i.e. `/**`.
+        /// One space after the leading sequence will be skipped.
+        /// It ends either with `*/` or `**/`.
+        /// After a newline all whitespace is skipped, as well as an optional `*` followed by another optional space,
+        /// as well as trailing whitespace on each line.
+        /// I.e. `/** a\n      * b */` yields the text `a\nb`.
+        /// * A C++ style doc comment. It is a C++ style comment starting with an additional `/` or '!`,
+        /// i.e. `///` or `//!`.
+        /// One space character after the leading sequence will be skipped,
+        /// as well as any trailing whitespace.
+        /// Two C++ style doc comments on two adjacent lines will be merged.
+        /// * An end of line doc comment. It is a C++ style comment starting with an '<', i.e. `//<`.
+        /// One space character after the leading sequence will be skipped,
+        /// as well as any trailing whitespace.
+        /// If the next line is a C++ style doc comment, it will be merged with that one.
+        ///
+        /// A documentation comment is associated with an entity,
+        /// if for C and C++ style doc comments, the entity declaration begins
+        /// on the line after the last line of the comment,
+        /// and if for an end of line comment, the entity declaration ends
+        /// on the same line as the end of line comment.
+        ///
+        /// This comment system is also used by [standardese](https://standardese.foonathan.net).
+        type_safe::optional_ref<const std::string> comment() const noexcept
+        {
+            return comment_.empty() ? nullptr : type_safe::opt_ref(&comment_);
+        }
+
+        /// \effects Sets the associated comment.
+        /// \requires The comment must not be empty, if there is one.
+        void set_comment(type_safe::optional<std::string> comment) noexcept
+        {
+            comment_ = std::move(comment.value());
+        }
+
     protected:
         /// \effects Creates it giving it the the name.
         cpp_entity(std::string name) : name_(std::move(name))
@@ -72,8 +110,9 @@ namespace cppast
             parent_ = parent;
         }
 
-        type_safe::optional_ref<const cpp_entity> parent_;
         std::string                               name_;
+        std::string                               comment_;
+        type_safe::optional_ref<const cpp_entity> parent_;
 
         template <typename T>
         friend struct detail::intrusive_list_access;

include/cppast/cpp_enum.hpp

@@ -76,6 +76,12 @@ namespace cppast
                 enum_->add_child(std::move(value));
             }
 
+            /// \returns The not yet finished enumeration.
+            cpp_enum& get() noexcept
+            {
+                return *enum_;
+            }
+
             /// \effects Registers the enum in the [cppast::cpp_entity_index](),
             /// using the given [cppast::cpp_entity_id]().
             /// \returns The finished enum.

include/cppast/cpp_file.hpp

@@ -5,6 +5,8 @@
 #ifndef CPPAST_CPP_FILE_HPP_INCLUDED
 #define CPPAST_CPP_FILE_HPP_INCLUDED
 
+#include <vector>
+
 #include <cppast/cpp_entity_index.hpp>
 #include <cppast/cpp_entity_container.hpp>
 #include <cppast/cpp_entity_ref.hpp>
@@ -17,6 +19,8 @@ namespace cppast
     class cpp_file final : public cpp_entity, public cpp_entity_container<cpp_file, cpp_entity>
     {
     public:
+        static cpp_entity_kind kind() noexcept;
+
         /// Builds a [cppast::cpp_file]().
         class builder
         {
@@ -32,6 +36,18 @@ namespace cppast
                 file_->add_child(std::move(child));
             }
 
+            /// \effects Adds an unmatched documentation comment.
+            void add_unmatched_comment(std::string str)
+            {
+                file_->comments_.push_back(std::move(str));
+            }
+
+            /// \returns The not yet finished file.
+            cpp_file& get() noexcept
+            {
+                return *file_;
+            }
+
             /// \effects Registers the file in the [cppast::cpp_entity_index]().
             /// It will use the file name as identifier.
             /// \returns The finished file.
@@ -45,6 +61,12 @@ namespace cppast
             std::unique_ptr<cpp_file> file_;
         };
 
+        /// \returns The unmatched documentation comments.
+        type_safe::array_ref<const std::string> unmatched_comments() const noexcept
+        {
+            return type_safe::ref(comments_.data(), comments_.size());
+        }
+
     private:
         cpp_file(std::string name) : cpp_entity(std::move(name))
         {
@@ -52,6 +74,8 @@ namespace cppast
 
         /// \returns [cpp_entity_type::file_t]().
         cpp_entity_kind do_get_entity_kind() const noexcept override;
+
+        std::vector<std::string> comments_;
     };
 
     /// \exclude

include/cppast/cpp_function.hpp

@@ -122,6 +122,12 @@ namespace cppast
                 static_cast<cpp_function_base&>(*function).noexcept_expr_ = std::move(cond);
             }
 
+            /// \returns The not yet finished function.
+            T& get() noexcept
+            {
+                return *function;
+            }
+
             /// \effects If the body is a definition, registers it.
             /// Else marks it as a declaration.
             /// \returns The finished function.

include/cppast/cpp_language_linkage.hpp

@@ -32,6 +32,12 @@ namespace cppast
                 linkage_->add_child(std::move(child));
             }
 
+            /// \returns The not yet finished language linkage.
+            cpp_language_linkage& get() const noexcept
+            {
+                return *linkage_;
+            }
+
             /// \returns The finalized language linkage.
             /// \notes It is not registered on purpose as nothing can refer to it.
             std::unique_ptr<cpp_language_linkage> finish()

include/cppast/cpp_namespace.hpp

@@ -35,6 +35,12 @@ namespace cppast
                 namespace_->add_child(std::move(child));
             }
 
+            /// \returns The not yet finished namespace.
+            cpp_namespace& get() const noexcept
+            {
+                return *namespace_;
+            }
+
             /// \effects Registers the namespace in the [cppast::cpp_entity_index](),
             /// using the given [cppast::cpp_entity_id]().
             /// \returns The finished namespace.

src/cpp_file.cpp

@@ -8,11 +8,16 @@
 
 using namespace cppast;
 
-cpp_entity_kind cpp_file::do_get_entity_kind() const noexcept
+cpp_entity_kind cpp_file::kind() noexcept
 {
     return cpp_entity_kind::file_t;
 }
 
+cpp_entity_kind cpp_file::do_get_entity_kind() const noexcept
+{
+    return kind();
+}
+
 bool detail::cpp_file_ref_predicate::operator()(const cpp_entity& e)
 {
     return e.kind() == cpp_entity_kind::file_t;

src/libclang/class_parser.cpp

@@ -88,13 +88,11 @@ namespace
     }
 }
 
-#include <iostream>
-#include "debug_helper.hpp"
-
 std::unique_ptr<cpp_entity> detail::parse_cpp_class(const detail::parse_context& context,
                                                     const CXCursor&              cur)
 {
     auto builder = make_class_builder(cur);
+    context.comments.match(builder.get(), cur);
     detail::visit_children(cur, [&](const CXCursor& child) {
         auto kind = clang_getCursorKind(child);
         if (kind == CXCursor_CXXAccessSpecifier)

src/libclang/enum_parser.cpp

@@ -68,10 +68,13 @@ std::unique_ptr<cpp_entity> detail::parse_cpp_enum(const detail::parse_context&
     DEBUG_ASSERT(cur.kind == CXCursor_EnumDecl, detail::assert_handler{});
 
     auto builder = make_enum_builder(context, cur);
+    context.comments.match(builder.get(), cur);
     detail::visit_children(cur, [&](const CXCursor& child) {
         try
         {
             auto entity = parse_enum_value(context, child);
+            if (entity)
+                context.comments.match(*entity, child);
             builder.add_value(std::move(entity));
         }
         catch (parse_error& ex)

src/libclang/function_parser.cpp

@@ -279,6 +279,7 @@ namespace
 
         cpp_function::builder builder(name.c_str(),
                                       detail::parse_type(context, clang_getCursorResultType(cur)));
+        context.comments.match(builder.get(), cur);
         add_parameters(context, builder, cur);
         if (clang_Cursor_isVariadic(cur))
             builder.is_variadic();
@@ -414,6 +415,7 @@ std::unique_ptr<cpp_entity> detail::parse_cpp_member_function(const detail::pars
     cpp_member_function::builder builder(name.c_str(),
                                          detail::parse_type(context,
                                                             clang_getCursorResultType(cur)));
+    context.comments.match(builder.get(), cur);
     add_parameters(context, builder, cur);
     if (clang_Cursor_isVariadic(cur))
         builder.is_variadic();
@@ -434,6 +436,7 @@ std::unique_ptr<cpp_entity> detail::parse_cpp_conversion_op(const detail::parse_
 {
     DEBUG_ASSERT(clang_getCursorKind(cur) == CXCursor_ConversionFunction, detail::assert_handler{});
     cpp_conversion_op::builder builder(detail::parse_type(context, clang_getCursorResultType(cur)));
+    context.comments.match(builder.get(), cur);
 
     detail::tokenizer    tokenizer(context.tu, context.file, cur);
     detail::token_stream stream(tokenizer, cur);
@@ -484,6 +487,7 @@ std::unique_ptr<cpp_entity> detail::parse_cpp_constructor(const detail::parse_co
     auto name = detail::get_cursor_name(cur);
 
     cpp_constructor::builder builder(name.c_str());
+    context.comments.match(builder.get(), cur);
     add_parameters(context, builder, cur);
     if (clang_Cursor_isVariadic(cur))
         builder.is_variadic();
@@ -523,6 +527,7 @@ std::unique_ptr<cpp_entity> detail::parse_cpp_destructor(const detail::parse_con
     detail::skip(stream, "~");
     auto                    name = std::string("~") + stream.get().c_str();
     cpp_destructor::builder builder(std::move(name));
+    context.comments.match(builder.get(), cur);
 
     detail::skip(stream, "(");
     detail::skip(stream, ")");

src/libclang/language_linkage_parser.cpp

@@ -28,6 +28,7 @@ std::unique_ptr<cpp_entity> detail::try_parse_cpp_language_linkage(const parse_c
     auto& name = stream.get().value();
 
     auto builder = cpp_language_linkage::builder(name.c_str());
+    context.comments.match(builder.get(), cur);
     detail::visit_children(cur, [&](const CXCursor& child) {
         auto entity = parse_entity(context, child);
         if (entity)

src/libclang/libclang_parser.cpp

@@ -185,7 +185,8 @@ std::unique_ptr<cpp_file> libclang_parser::do_parse(const cpp_entity_index& idx,
     auto              preprocessed_iter = preprocessed.entities.begin();
 
     // convert entity hierachies
-    detail::parse_context context{tu.get(), file, type_safe::ref(logger()), type_safe::ref(idx)};
+    detail::parse_context context{tu.get(), file, type_safe::ref(logger()), type_safe::ref(idx),
+                                  detail::comment_context(preprocessed.comments)};
     detail::visit_tu(tu, path.c_str(), [&](const CXCursor& cur) {
         // add macro if needed
         for (auto line = get_line_no(cur);
@@ -201,6 +202,12 @@ std::unique_ptr<cpp_file> libclang_parser::do_parse(const cpp_entity_index& idx,
     for (; preprocessed_iter != preprocessed.entities.end(); ++preprocessed_iter)
         builder.add_child(std::move(preprocessed_iter->entity));
 
+    for (auto& c : preprocessed.comments)
+    {
+        if (!c.comment.empty())
+            builder.add_unmatched_comment(std::move(c.comment));
+    }
+
     return builder.finish(idx);
 }
 catch (detail::parse_error& ex)

src/libclang/namespace_parser.cpp

@@ -40,6 +40,7 @@ std::unique_ptr<cpp_entity> detail::parse_cpp_namespace(const detail::parse_cont
     DEBUG_ASSERT(cur.kind == CXCursor_Namespace, detail::assert_handler{});
 
     auto builder = make_ns_builder(context, cur);
+    context.comments.match(builder.get(), cur);
     detail::visit_children(cur, [&](const CXCursor& cur) {
         auto entity = parse_entity(context, cur);
         if (entity)
@@ -91,8 +92,10 @@ std::unique_ptr<cpp_entity> detail::parse_cpp_namespace_alias(const detail::pars
         target_name += stream.get().c_str();
 
     auto target = cpp_namespace_ref(parse_ns_target_cursor(cur), std::move(target_name));
-    return cpp_namespace_alias::build(*context.idx, get_entity_id(cur), std::move(name),
-                                      std::move(target));
+    auto result = cpp_namespace_alias::build(*context.idx, get_entity_id(cur), std::move(name),
+                                             std::move(target));
+    context.comments.match(*result, cur);
+    return result;
 }
 
 std::unique_ptr<cpp_entity> detail::parse_cpp_using_directive(const detail::parse_context& context,
@@ -113,7 +116,9 @@ std::unique_ptr<cpp_entity> detail::parse_cpp_using_directive(const detail::pars
         target_name += stream.get().c_str();
 
     auto target = cpp_namespace_ref(parse_ns_target_cursor(cur), std::move(target_name));
-    return cpp_using_directive::build(target);
+    auto result = cpp_using_directive::build(target);
+    context.comments.match(*result, cur);
+    return result;
 }
 
 namespace
@@ -179,5 +184,7 @@ std::unique_ptr<cpp_entity> detail::parse_cpp_using_declaration(
         target_name += stream.get().c_str();
 
     auto target = cpp_entity_ref(parse_entity_target_cursor(cur), std::move(target_name));
-    return cpp_using_declaration::build(std::move(target));
+    auto result = cpp_using_declaration::build(std::move(target));
+    context.comments.match(*result, cur);
+    return result;
 }

src/libclang/parse_functions.cpp

@@ -49,6 +49,24 @@ cpp_storage_class_specifiers detail::get_storage_class(const CXCursor& cur)
     return cpp_storage_class_auto;
 }
 
+void detail::comment_context::match(cpp_entity& e, const CXCursor& cur) const
+{
+    auto     pos = clang_getRangeStart(clang_getCursorExtent(cur));
+    unsigned line;
+    clang_getSpellingLocation(pos, nullptr, &line, nullptr, nullptr);
+
+    match(e, line);
+}
+
+void detail::comment_context::match(cpp_entity& e, unsigned line) const
+{
+    // find comment
+    while (cur_ != end_ && cur_->line + 1 < line)
+        ++cur_;
+    if (cur_ != end_ && cur_->matches(e, line))
+        e.set_comment(std::move(cur_++->comment));
+}
+
 std::unique_ptr<cpp_entity> detail::parse_entity(const detail::parse_context& context,
                                                  const CXCursor&              cur) try
 {
@@ -103,9 +121,10 @@ std::unique_ptr<cpp_entity> detail::parse_entity(const detail::parse_context& co
         break;
     }
 
-    auto msg = format("unhandled cursor of kind '", get_cursor_kind_spelling(cur).c_str(), "'");
+    auto msg = detail::format("unhandled cursor of kind '",
+                              detail::get_cursor_kind_spelling(cur).c_str(), "'");
     context.logger->log("libclang parser",
-                        diagnostic{std::move(msg), make_location(cur), severity::warning});
+                        diagnostic{std::move(msg), detail::make_location(cur), severity::warning});
 
     return nullptr;
 }

src/libclang/parse_functions.hpp

@@ -11,6 +11,7 @@
 #include "raii_wrapper.hpp"
 #include "tokenizer.hpp"   // for convenience
 #include "parse_error.hpp" // for convenience
+#include "preprocessor.hpp"
 
 namespace cppast
 {
@@ -30,12 +31,31 @@ namespace cppast
         // note: does not handle thread_local
         cpp_storage_class_specifiers get_storage_class(const CXCursor& cur);
 
+        class comment_context
+        {
+        public:
+            explicit comment_context(std::vector<pp_doc_comment>& comments)
+            : cur_(comments.data()), end_(comments.data() + comments.size())
+            {
+            }
+
+            // must be called for entities that want an associated comment
+            // must be called *BEFORE* the children are added
+            void match(cpp_entity& e, const CXCursor& cur) const;
+            void match(cpp_entity& e, unsigned line) const;
+
+        private:
+            mutable pp_doc_comment* cur_;
+            pp_doc_comment*         end_;
+        };
+
         struct parse_context
         {
             CXTranslationUnit                              tu;
             CXFile                                         file;
             type_safe::object_ref<const diagnostic_logger> logger;
             type_safe::object_ref<const cpp_entity_index>  idx;
+            comment_context                                comments;
         };
 
         std::unique_ptr<cpp_type> parse_type(const parse_context& context, const CXType& type);

src/libclang/preprocessor.cpp

@@ -134,7 +134,9 @@ namespace
                         [&](const char* str, std::size_t n) {
                             preprocessed.reserve(preprocessed.size() + n);
                             for (auto end = str + n; str != end; ++str)
-                                if (*str != '\r')
+                                if (*str == '\t')
+                                    preprocessed += "  "; // just two spaces because why not
+                                else if (*str != '\r')
                                     preprocessed.push_back(*str);
                         },
                         [&](const char* str, std::size_t n) {
@@ -247,26 +249,155 @@ namespace
         return std::strncmp(p.ptr(), str, std::strlen(str)) == 0;
     }
 
-    bool bump_c_str(position& p)
+    detail::pp_doc_comment parse_c_doc_comment(position& p)
+    {
+        detail::pp_doc_comment result;
+        result.kind = detail::pp_doc_comment::c;
+
+        if (starts_with(p, " "))
+            // skip one whitespace at most
+            p.bump();
+
+        while (!starts_with(p, "*/"))
+        {
+            if (starts_with(p, "\n"))
+            {
+                // remove trailing spaces
+                while (!result.comment.empty() && result.comment.back() == ' ')
+                    result.comment.pop_back();
+                // skip newline
+                p.bump();
+                result.comment += '\n';
+                // skip indentation
+                while (starts_with(p, " "))
+                    p.bump();
+                // skip continuation star, if any
+                if (starts_with(p, "*") && !starts_with(p, "*/"))
+                {
+                    p.bump();
+                    if (starts_with(p, " "))
+                        // skip one whitespace at most
+                        p.bump();
+                }
+            }
+            else
+            {
+                result.comment += *p.ptr();
+                p.bump();
+            }
+        }
+        p.bump(2u);
+
+        // remove trailing star
+        if (!result.comment.empty() && result.comment.back() == '*')
+            result.comment.pop_back();
+        // remove trailing spaces
+        while (!result.comment.empty() && result.comment.back() == ' ')
+            result.comment.pop_back();
+
+        result.line = p.cur_line();
+        return result;
+    }
+
+    bool bump_c_comment(position& p, detail::preprocessor_output& output)
     {
         if (!starts_with(p, "/*"))
             return false;
         p.bump(2u);
 
-        while (!starts_with(p, "*/"))
+        if (starts_with(p, "*"))
+        {
+            // doc comment
             p.bump();
-        p.bump(2u);
+            output.comments.push_back(parse_c_doc_comment(p));
+        }
+        else
+        {
+            while (!starts_with(p, "*/"))
+                p.bump();
+            p.bump(2u);
+        }
+
+        if (!starts_with(p, "\n"))
+            // ensure an additional newline after each C comment
+            // this allows matching documentation comments to entities generated from macros
+            // as the entity corresponding to the documentation comment will be on the next line
+            // otherwise all entities would have the same line number
+            p.write_str("\n");
+
         return true;
     }
 
-    bool bump_cpp_str(position& p)
+    detail::pp_doc_comment parse_cpp_doc_comment(position& p, bool end_of_line)
+    {
+        detail::pp_doc_comment result;
+        result.kind =
+            end_of_line ? detail::pp_doc_comment::end_of_line : detail::pp_doc_comment::cpp;
+        if (starts_with(p, " "))
+            // skip one whitespace at most
+            p.bump();
+
+        while (!starts_with(p, "\n"))
+        {
+            result.comment += *p.ptr();
+            p.bump();
+        }
+        // don't skip newline
+
+        // remove trailing spaces
+        while (!result.comment.empty() && result.comment.back() == ' ')
+            result.comment.pop_back();
+        result.line = p.cur_line();
+        return result;
+    }
+
+    bool can_merge_comment(const detail::pp_doc_comment& comment, unsigned cur_line)
+    {
+        return comment.line + 1 == cur_line
+               && (comment.kind == detail::pp_doc_comment::cpp
+                   || comment.kind == detail::pp_doc_comment::end_of_line);
+    }
+
+    void merge_or_add(detail::preprocessor_output& output, detail::pp_doc_comment comment)
+    {
+        if (output.comments.empty() || !can_merge_comment(output.comments.back(), comment.line))
+            output.comments.push_back(std::move(comment));
+        else
+        {
+            auto& result = output.comments.back();
+            result.comment += "\n" + std::move(comment.comment);
+            if (result.kind != detail::pp_doc_comment::end_of_line)
+                result.line = comment.line;
+        }
+    }
+
+    bool bump_cpp_comment(position& p, detail::preprocessor_output& output)
     {
         if (!starts_with(p, "//"))
             return false;
         p.bump(2u);
 
-        while (!starts_with(p, "\n"))
+        if (starts_with(p, "/") || starts_with(p, "!"))
+        {
+            // C++ style doc comment
             p.bump();
+            auto comment = parse_cpp_doc_comment(p, false);
+            merge_or_add(output, std::move(comment));
+        }
+        else if (starts_with(p, "<"))
+        {
+            // end of line doc comment
+            p.bump();
+            auto comment = parse_cpp_doc_comment(p, true);
+            output.comments.push_back(std::move(comment));
+        }
+        else
+        {
+            while (!starts_with(p, "\n"))
+                p.bump();
+            // don't skip newline
+        }
+
         return true;
     }
 
@@ -276,7 +407,8 @@ namespace
             p.skip();
     }
 
-    std::unique_ptr<cpp_macro_definition> parse_macro(position& p)
+    std::unique_ptr<cpp_macro_definition> parse_macro(position&                    p,
+                                                      detail::preprocessor_output& output)
     {
         // format (at new line): #define <name> [replacement]
         // or: #define <name>(<args>) [replacement]
@@ -303,11 +435,25 @@ namespace
         }
 
         std::string rep;
-        for (skip_spaces(p); !starts_with(p, "\n"); p.skip())
+        auto        in_c_comment = false;
+        for (skip_spaces(p); in_c_comment || !starts_with(p, "\n"); p.skip())
+        {
+            if (starts_with(p, "/*"))
+                in_c_comment = true;
+            else if (in_c_comment && starts_with(p, "*/"))
+                in_c_comment = false;
             rep += *p.ptr();
+        }
         // don't skip newline
 
-        return cpp_macro_definition::build(std::move(name), std::move(args), std::move(rep));
+        auto result = cpp_macro_definition::build(std::move(name), std::move(args), std::move(rep));
+        // match comment directly
+        if (!output.comments.empty() && output.comments.back().matches(*result, p.cur_line()))
+        {
+            result->set_comment(std::move(output.comments.back().comment));
+            output.comments.pop_back();
+        }
+        return result;
     }
 
     ts::optional<std::string> parse_undef(position& p)
@@ -325,7 +471,8 @@ namespace
         return result;
     }
 
-    std::unique_ptr<cpp_include_directive> parse_include(position& p)
+    std::unique_ptr<cpp_include_directive> parse_include(position&                    p,
+                                                         detail::preprocessor_output& output)
     {
         // format (at new line, literal <>): #include <filename>
         // or: #include "filename"
@@ -358,8 +505,14 @@ namespace
         DEBUG_ASSERT(starts_with(p, "\n"), detail::assert_handler{});
         // don't skip newline
 
-        return cpp_include_directive::build(cpp_file_ref(cpp_entity_id(filename), filename),
-                                            include_kind);
+        auto result = cpp_include_directive::build(cpp_file_ref(cpp_entity_id(filename), filename),
+                                                   include_kind);
+        if (!output.comments.empty() && output.comments.back().matches(*result, p.cur_line()))
+        {
+            result->set_comment(std::move(output.comments.back().comment));
+            output.comments.pop_back();
+        }
+        return result;
     }
 
     bool skip_pragma(position& p)
@@ -461,7 +614,7 @@ detail::preprocessor_output detail::preprocess(const libclang_compile_config& co
     std::size_t file_depth = 0u;
     while (p)
     {
-        if (auto macro = parse_macro(p))
+        if (auto macro = parse_macro(p, result))
         {
             if (file_depth == 0u)
                 result.entities.push_back({std::move(macro), p.cur_line()});
@@ -478,7 +631,7 @@ detail::preprocessor_output detail::preprocess(const libclang_compile_config& co
                                           }),
                            result.entities.end());
         }
-        else if (auto include = parse_include(p))
+        else if (auto include = parse_include(p, result))
         {
             if (file_depth == 0u)
                 result.entities.push_back({std::move(include), p.cur_line()});
@@ -516,13 +669,9 @@ detail::preprocessor_output detail::preprocess(const libclang_compile_config& co
                 break;
             }
         }
-        else if (bump_c_str(p))
-            // write an additional newline after each string
-            // this allows matching documentation comments to entities generated from macros
-            // as the entity corresponding to the documentation comment will be on the next line
-            // otherwise all entities would have the same line number
-            p.write_str("\n");
-        else if (bump_cpp_str(p))
+        else if (bump_c_comment(p, result))
+            continue;
+        else if (bump_cpp_comment(p, result))
             continue;
         else
             p.bump();
@@ -530,3 +679,11 @@ detail::preprocessor_output detail::preprocess(const libclang_compile_config& co
 
     return result;
 }
+
+bool detail::pp_doc_comment::matches(const cpp_entity&, unsigned e_line)
+{
+    if (kind == detail::pp_doc_comment::end_of_line)
+        return line == e_line;
+    else
+        return line + 1u == e_line;
+}

src/libclang/preprocessor.hpp

@@ -18,10 +18,25 @@ namespace cppast
             unsigned                    line;
         };
 
+        struct pp_doc_comment
+        {
+            std::string comment;
+            unsigned    line;
+            enum
+            {
+                c,
+                cpp,
+                end_of_line,
+            } kind;
+
+            bool matches(const cpp_entity& e, unsigned line);
+        };
+
         struct preprocessor_output
         {
-            std::string            source;
-            std::vector<pp_entity> entities;
+            std::string                 source;
+            std::vector<pp_entity>      entities;
+            std::vector<pp_doc_comment> comments;
         };
 
         preprocessor_output preprocess(const libclang_compile_config& config, const char* path,

src/libclang/type_parser.cpp

@@ -443,5 +443,8 @@ std::unique_ptr<cpp_entity> detail::parse_cpp_type_alias(const detail::parse_con
 
     auto name = detail::get_cursor_name(cur);
     auto type = parse_type(context, clang_getTypedefDeclUnderlyingType(cur));
-    return cpp_type_alias::build(*context.idx, get_entity_id(cur), name.c_str(), std::move(type));
+    auto result =
+        cpp_type_alias::build(*context.idx, get_entity_id(cur), name.c_str(), std::move(type));
+    context.comments.match(*result, cur);
+    return result;
 }

src/libclang/variable_parser.cpp

@@ -50,15 +50,19 @@ std::unique_ptr<cpp_entity> detail::parse_cpp_variable(const detail::parse_conte
         else if (token.value() == "constexpr")
             is_constexpr = true;
 
+    std::unique_ptr<cpp_variable> result;
     if (clang_isCursorDefinition(cur))
     {
         auto default_value = parse_default_value(context, cur);
-        return cpp_variable::build(*context.idx, get_entity_id(cur), name.c_str(), std::move(type),
-                                   std::move(default_value), storage_class, is_constexpr);
+        result =
+            cpp_variable::build(*context.idx, get_entity_id(cur), name.c_str(), std::move(type),
+                                std::move(default_value), storage_class, is_constexpr);
     }
     else
-        return cpp_variable::build_declaration(get_entity_id(cur), name.c_str(), std::move(type),
-                                               storage_class, is_constexpr);
+        result = cpp_variable::build_declaration(get_entity_id(cur), name.c_str(), std::move(type),
+                                                 storage_class, is_constexpr);
+    context.comments.match(*result, cur);
+    return result;
 }
 
 std::unique_ptr<cpp_entity> detail::parse_cpp_member_variable(const detail::parse_context& context,
@@ -70,15 +74,16 @@ std::unique_ptr<cpp_entity> detail::parse_cpp_member_variable(const detail::pars
     auto type       = parse_type(context, clang_getCursorType(cur));
     auto is_mutable = clang_CXXField_isMutable(cur) != 0u;
 
+    std::unique_ptr<cpp_member_variable_base> result;
     if (clang_Cursor_isBitField(cur))
     {
         auto no_bits = clang_getFieldDeclBitWidth(cur);
         DEBUG_ASSERT(no_bits >= 0, detail::parse_error_handler{}, cur, "invalid number of bits");
         if (name.empty())
-            return cpp_bitfield::build(std::move(type), unsigned(no_bits), is_mutable);
+            result = cpp_bitfield::build(std::move(type), unsigned(no_bits), is_mutable);
         else
-            return cpp_bitfield::build(*context.idx, get_entity_id(cur), name.c_str(),
-                                       std::move(type), unsigned(no_bits), is_mutable);
+            result = cpp_bitfield::build(*context.idx, get_entity_id(cur), name.c_str(),
+                                         std::move(type), unsigned(no_bits), is_mutable);
     }
     else
     {
@@ -92,7 +97,9 @@ std::unique_ptr<cpp_entity> detail::parse_cpp_member_variable(const detail::pars
         auto default_value = parse_raw_expression(context, stream, stream.end(),
                                                   parse_type(context, clang_getCursorType(cur)));
 
-        return cpp_member_variable::build(*context.idx, get_entity_id(cur), name.c_str(),
-                                          std::move(type), std::move(default_value), is_mutable);
+        result = cpp_member_variable::build(*context.idx, get_entity_id(cur), name.c_str(),
+                                            std::move(type), std::move(default_value), is_mutable);
     }
+    context.comments.match(*result, cur);
+    return result;
 }

test/cpp_preprocessor.cpp

@@ -113,3 +113,69 @@ TEST_CASE("cpp_include_directive")
     REQUIRE(count == 2u);
 }
 #endif
+
+TEST_CASE("comment matching")
+{
+    auto code = R"(
+/// u
+
+/// a
+/// a
+struct a {};
+
+/// u
+/** b
+  * b */
+void b(int, float);
+
+/** u */
+//! c
+/// c
+enum class c
+{
+    d, //< d
+    /// d
+    e, //< e
+    /// e
+
+    /** f
+f **/
+    f,
+};
+
+/// g
+/// g
+#define g(name) \
+class name \
+{ \
+    /** i
+        i */ \
+    void i(); \
+};
+
+/// h
+/// h
+g(h)
+
+/// j
+/// j
+using j = int;
+)";
+
+    auto file = parse({}, "comment-matching.cpp", code);
+    visit(*file, [&](const cpp_entity& e, visitor_info) {
+        if (e.kind() == cpp_entity_kind::file_t)
+            return true;
+        else if (e.name().empty())
+            return true;
+
+        INFO(e.name());
+        REQUIRE(e.comment());
+        REQUIRE(e.comment().value() == e.name() + "\n" + e.name());
+        return true;
+    });
+
+    for (auto& comment : file->unmatched_comments())
+        REQUIRE(comment == "u");
+    REQUIRE((file->unmatched_comments().size() == 3u));
+}