Add compilation database parsing and FileParser

Closes #19.

include/cppast/compile_config.hpp

@@ -104,6 +104,9 @@ namespace cppast
         {
         }
 
+        compile_config(const compile_config&) = default;
+        compile_config& operator=(const compile_config&) = default;
+
         ~compile_config() noexcept = default;
 
         void add_flag(std::string flag)

include/cppast/cpp_entity_index.hpp

@@ -17,6 +17,7 @@
 namespace cppast
 {
     class cpp_entity;
+    class cpp_file;
     class cpp_namespace;
 
     /// \exclude
@@ -78,10 +79,17 @@ namespace cppast
         void register_definition(cpp_entity_id                           id,
                                  type_safe::object_ref<const cpp_entity> entity) const;
 
+        /// \effects Registers a new [cppast::cpp_file]().
+        /// \returns `true` if the file was not registered before.
+        /// If it returns `false`, the file was registered before and nothing was changed.
+        /// \requires The entity must live as long as the index lives.
+        /// \notes This operation is thread safe.
+        bool register_file(cpp_entity_id id, type_safe::object_ref<const cpp_file> file) const;
+
         /// \effects Registers a new [cppast::cpp_entity]() which is a declaration.
         /// Only the first declaration will be registered.
         /// \requires The entity must live as long as the index lives.
-        /// \requires The entity must not be a namespace.
+        /// \requires The entity must be forward declarable.
         /// \notes This operation is thread safe.
         void register_forward_declaration(cpp_entity_id                           id,
                                           type_safe::object_ref<const cpp_entity> entity) const;

include/cppast/cpp_file.hpp

@@ -50,11 +50,11 @@ namespace cppast
 
             /// \effects Registers the file in the [cppast::cpp_entity_index]().
             /// It will use the file name as identifier.
-            /// \returns The finished file.
+            /// \returns The finished file, or `nullptr`, if that file was already registered.
             std::unique_ptr<cpp_file> finish(const cpp_entity_index& idx) noexcept
             {
-                idx.register_definition(cpp_entity_id(file_->name()), type_safe::ref(*file_));
+                auto res = idx.register_file(cpp_entity_id(file_->name()), type_safe::ref(*file_));
-                return std::move(file_);
+                return res ? std::move(file_) : nullptr;
             }
 
         private:

include/cppast/detail/intrusive_list.hpp

@@ -14,6 +14,8 @@
 
 namespace cppast
 {
+    class cpp_file;
+
     namespace detail
     {
         template <typename T>
@@ -122,22 +124,20 @@ namespace cppast
             intrusive_list() = default;
 
             //=== modifiers ===//
-            template <typename U>
+            template <
+                typename Dummy = T,
+                typename = typename std::enable_if<std::is_same<Dummy, cpp_file>::value>::type>
+            void push_back(std::unique_ptr<T> obj) noexcept
+            {
+                push_back_impl(std::move(obj));
+            }
+
+            template <
+                typename U,
+                typename = typename std::enable_if<!std::is_same<T, cpp_file>::value, U>::type>
             void push_back(const U& parent, std::unique_ptr<T> obj) noexcept
             {
-                DEBUG_ASSERT(obj != nullptr, detail::assert_handler{});
+                push_back_impl(std::move(obj));
-
-                if (last_)
-                {
-                    auto ptr = intrusive_list_access<T>::set_next(last_.value(), std::move(obj));
-                    last_    = type_safe::ref(*ptr);
-                }
-                else
-                {
-                    first_ = std::move(obj);
-                    last_  = type_safe::opt_ref(first_.get());
-                }
-
                 intrusive_list_access<T>::on_insert(last_.value(), parent);
             }
 
@@ -192,6 +192,22 @@ namespace cppast
             }
 
         private:
+            void push_back_impl(std::unique_ptr<T> obj)
+            {
+                DEBUG_ASSERT(obj != nullptr, detail::assert_handler{});
+
+                if (last_)
+                {
+                    auto ptr = intrusive_list_access<T>::set_next(last_.value(), std::move(obj));
+                    last_    = type_safe::ref(*ptr);
+                }
+                else
+                {
+                    first_ = std::move(obj);
+                    last_  = type_safe::opt_ref(first_.get());
+                }
+            }
+
             std::unique_ptr<T>         first_;
             type_safe::optional_ref<T> last_;
         };

include/cppast/libclang_parser.hpp

@@ -12,6 +12,7 @@
 namespace cppast
 {
     class libclang_compile_config;
+    class libclang_compilation_database;
 
     namespace detail
     {
@@ -23,8 +24,67 @@ namespace cppast
 
             static const std::vector<std::string>& flags(const libclang_compile_config& config);
         };
+
+        void for_each_file(const libclang_compilation_database& database, void* user_data,
+                           void (*callback)(void*, std::string));
     } // namespace detail
 
+    /// The exception thrown when a fatal parse error occurs.
+    class libclang_error final : public std::runtime_error
+    {
+    public:
+        /// \effects Creates it with a message.
+        libclang_error(std::string msg) : std::runtime_error(std::move(msg))
+        {
+        }
+    };
+
+    /// A compilation database.
+    ///
+    /// This represents a `compile_commands.json` file,
+    /// which stores all the commands needed to compile a set of files.
+    /// It can be generated by CMake using the `CMAKE_EXPORT_COMPILE_COMMANDS` option.
+    class libclang_compilation_database
+    {
+    public:
+        /// \effects Creates it giving the directory where the `compile_commands.json` file is located.
+        /// \throws `libclang_error` if the database could not be loaded or found.
+        libclang_compilation_database(const std::string& build_directory);
+
+        libclang_compilation_database(libclang_compilation_database&& other)
+        : database_(other.database_)
+        {
+            other.database_ = nullptr;
+        }
+
+        ~libclang_compilation_database();
+
+        libclang_compilation_database& operator=(libclang_compilation_database&& other)
+        {
+            libclang_compilation_database tmp(std::move(other));
+            std::swap(tmp.database_, database_);
+            return *this;
+        }
+
+        /// \returns Whether or not the database contains information about the given file.
+        /// \group has_config
+        bool has_config(const char* file_name) const;
+
+        /// \group has_config
+        bool has_config(const std::string& file_name) const
+        {
+            return has_config(file_name.c_str());
+        }
+
+    private:
+        using database = void*;
+        database database_;
+
+        friend libclang_compile_config;
+        friend void detail::for_each_file(const libclang_compilation_database& database,
+                                          void* user_data, void (*callback)(void*, std::string));
+    };
+
     /// Compilation config for the [cppast::libclang_parser]().
     class libclang_compile_config final : public compile_config
     {
@@ -36,6 +96,23 @@ namespace cppast
         /// It will also define `__cppast__` with the value `"libclang"` as well as `__cppast_major__` and `__cppast_minor__`.
         libclang_compile_config();
 
+        /// Creates the configuration stored in the database.
+        ///
+        /// \effects It will use the options found in the database for the specified file.
+        /// This does not necessarily need to match the file that is going to be parsed,
+        /// but it should.
+        /// It will also add the default configuration options.
+        /// \notes Header files are not included in the compilation database,
+        /// you need to pass in the file name of the corresponding source file,
+        /// if you want to parse one.
+        /// \notes It will only consider options you could also set by the other functions.
+        /// \notes The file key will include the specified directory in the JSON, if it is not a full path.
+        libclang_compile_config(const libclang_compilation_database& database,
+                                const std::string&                   file);
+
+        libclang_compile_config(const libclang_compile_config& other) = default;
+        libclang_compile_config& operator=(const libclang_compile_config& other) = default;
+
         /// \effects Sets the path to the location of the `clang++` binary and the version of that binary.
         /// \notes It will be used for preprocessing.
         void set_clang_binary(std::string binary, int major, int minor, int patch)
@@ -64,20 +141,21 @@ namespace cppast
         friend detail::libclang_compile_config_access;
     };
 
-    /// The exception thrown when a fatal parse error occurs.
+    /// Finds a configuration for a given file.
-    class libclang_error final : public std::runtime_error
+    ///
-    {
+    /// \returns If the database contains a configuration for the given file, returns that configuration.
-    public:
+    /// Otherwise removes the file extension of the file and tries the same procedure
-        /// \effects Creates it with a message.
+    /// for common C++ header and source file extensions.
-        libclang_error(std::string msg) : std::runtime_error(std::move(msg))
+    /// \notes This function is intended to be used as the basis for a `get_config` function of [cppast::parse_files](standardese://cppast::parse_files_basic/).
-        {
+    type_safe::optional<libclang_compile_config> find_config_for(
-        }
+        const libclang_compilation_database& database, std::string file_name);
-    };
 
     /// A parser that uses libclang.
     class libclang_parser final : public parser
     {
     public:
+        using config = libclang_compile_config;
+
         explicit libclang_parser(type_safe::object_ref<const diagnostic_logger> logger);
         ~libclang_parser() noexcept override;
 
@@ -88,6 +166,55 @@ namespace cppast
         struct impl;
         std::unique_ptr<impl> pimpl_;
     };
+
+    /// Parses multiple files using a [cppast::libclang_parser]() and a compilation database.
+    ///
+    /// \effects Invokes [cppast::parse_files](standardese://parse_files_basic/) passing it the parser and file names,
+    /// and a `get_config` function using [cppast::find_config_for]().
+    ///
+    /// \throws [cppast::libclang_error]() if no configuration for a given file could be found in the database.
+    ///
+    /// \requires `FileParser` must use the libclang parser.
+    /// i.e. `FileParser::parser` must be an alias of [cppast::libclang_parser]().
+    template <class FileParser, class Range>
+    void parse_files(FileParser& parser, Range&& file_names,
+                     const libclang_compilation_database& database)
+    {
+        static_assert(std::is_same<typename FileParser::parser, libclang_parser>::value,
+                      "must use the libclang parser");
+        parse_files(parser, std::forward<Range>(file_names), [&](const std::string& file) {
+            auto config = find_config_for(database, file);
+            if (!config)
+                throw libclang_error("unable to find configuration for file '" + file + "'");
+            return config.value();
+        });
+    }
+
+    /// Parses the files specified in a compilation database using a [cppast::libclang_parser]().
+    ///
+    /// \effects For each file specified in a compilation database,
+    /// uses the `FileParser` to parse the file with the configuration specified in the database.
+    ///
+    /// \requires `FileParser` must have the same requirements as for [cppast::parse_files](standardese://parse_files_basic/).
+    /// It must also use the libclang parser,
+    /// i.e. `FileParser::parser` must be an alias of [cppast::libclang_parser]().
+    template <class FileParser>
+    void parse_database(FileParser& parser, const libclang_compilation_database& database)
+    {
+        static_assert(std::is_same<typename FileParser::parser, libclang_parser>::value,
+                      "must use the libclang parser");
+        struct data_t
+        {
+            FileParser&                          parser;
+            const libclang_compilation_database& database;
+        } data{parser, database};
+        detail::for_each_file(database, &data, [](void* ptr, std::string file) {
+            auto& data = *static_cast<data_t*>(ptr);
+
+            libclang_compile_config config(data.database, file);
+            data.parser.parse(std::move(file), std::move(config));
+        });
+    }
 } // namespace cppast
 
 #endif // CPPAST_LIBCLANG_PARSER_HPP_INCLUDED

include/cppast/parser.hpp

@@ -9,6 +9,7 @@
 
 #include <cppast/compile_config.hpp>
 #include <cppast/cpp_file.hpp>
+#include <cppast/cpp_preprocessor.hpp>
 
 namespace cppast
 {
@@ -21,7 +22,8 @@ namespace cppast
     class diagnostic_logger
     {
     public:
-        diagnostic_logger() noexcept : error_(false)
+        /// \effects Creates it either as verbose or not.
+        explicit diagnostic_logger(bool is_verbose = false) noexcept : verbose_(is_verbose)
         {
         }
 
@@ -34,15 +36,6 @@ namespace cppast
         /// \notes `source` points to a string literal that gives additional context to what generates the message.
         bool log(const char* source, const diagnostic& d) const;
 
-        /// \returns Whether or not a diagnostic of [severity::error]() was logged.
-        /// \notes If an error happens, the parser will still continue to build the AST,
-        /// unless the error is critical
-        /// so the result will be incomplete if this function returns `true`.
-        bool error_logged() const noexcept
-        {
-            return error_;
-        }
-
         /// \effects Sets whether or not the logger prints debugging diagnostics.
         void set_verbose(bool value) noexcept
         {
@@ -58,15 +51,23 @@ namespace cppast
     private:
         virtual bool do_log(const char* source, const diagnostic& d) const = 0;
 
-        mutable std::atomic<bool> error_;
+        bool verbose_;
-        bool                      verbose_ = false;
     };
 
+    /// \returns The default logger object.
+    type_safe::object_ref<const diagnostic_logger> default_logger() noexcept;
+
+    /// \returns The default verbose logger object.
+    type_safe::object_ref<const diagnostic_logger> default_verbose_logger() noexcept;
+
     /// A [cppast::diagnostic_logger]() that logs to `stderr`.
     ///
     /// It prints all diagnostics in an implementation-defined format.
     class stderr_diagnostic_logger final : public diagnostic_logger
     {
+    public:
+        using diagnostic_logger::diagnostic_logger;
+
     private:
         bool do_log(const char* source, const diagnostic& d) const override;
     };
@@ -74,6 +75,9 @@ namespace cppast
     /// Base class for a parser.
     ///
     /// It reads a C++ source file and creates the matching [cppast::cpp_file]().
+    /// Derived classes can implement how the file is parsed.
+    ///
+    /// \requires A derived class must provide an alias `config` which is the corresponding derived class of the [cppast::compile_config]().
     class parser
     {
     public:
@@ -84,16 +88,32 @@ namespace cppast
 
         /// \effects Parses the given file.
         /// \returns The [cppast::cpp_file]() object describing it.
+        /// It can be `nullptr`, if there was an error or the specified file already registered in the index.
         /// \requires The dynamic type of `config` must match the required config type.
+        /// \notes This function is thread safe.
         std::unique_ptr<cpp_file> parse(const cpp_entity_index& idx, std::string path,
                                         const compile_config& config) const
         {
             return do_parse(idx, std::move(path), config);
         }
 
+        /// \returns Whether or not an error occurred during parsing.
+        /// If that happens, the AST might be incomplete.
+        bool error() const noexcept
+        {
+            return error_;
+        }
+
+        /// \effects Resets the error state.
+        void reset_error() noexcept
+        {
+            error_ = false;
+        }
+
     protected:
         /// \effects Creates it giving it a reference to the logger it uses.
-        explicit parser(type_safe::object_ref<const diagnostic_logger> logger) : logger_(logger)
+        explicit parser(type_safe::object_ref<const diagnostic_logger> logger)
+        : logger_(logger), error_(false)
         {
         }
 
@@ -103,14 +123,139 @@ namespace cppast
             return *logger_;
         }
 
+        /// \effects Sets the error state.
+        /// This must be called when an error or critical diagnostic is logged and the AST is incomplete.
+        void set_error() const noexcept
+        {
+            error_ = true;
+        }
+
     private:
         /// \effects Parses the given file.
         /// \returns The [cppast::cpp_file]() object describing it.
+        /// \requires The function must be thread safe.
         virtual std::unique_ptr<cpp_file> do_parse(const cpp_entity_index& idx, std::string path,
                                                    const compile_config& config) const = 0;
 
         type_safe::object_ref<const diagnostic_logger> logger_;
+        mutable std::atomic<bool>                      error_;
     };
+
+    /// A simple `FileParser` that parses all files synchronously.
+    ///
+    /// More advanced parsers could use a thread pool, for example.
+    template <class Parser>
+    class simple_file_parser
+    {
+        static_assert(std::is_base_of<cppast::parser, Parser>::value,
+                      "Parser must be derived from cppast::parser");
+
+    public:
+        using parser = Parser;
+        using config = typename Parser::config;
+
+        /// \effects Creates a file parser populating the given index
+        /// and using the parser created by forwarding the given arguments.
+        template <typename... Args>
+        explicit simple_file_parser(type_safe::object_ref<const cpp_entity_index> idx,
+                                    Args&&... args)
+        : parser_(std::forward<Args>(args)...), idx_(idx)
+        {
+        }
+
+        /// \effects Parses the given file using the given configuration.
+        /// \returns The parsed file or an empty optional, if a fatal error occurred.
+        type_safe::optional_ref<const cpp_file> parse(std::string path, const config& c)
+        {
+            auto file = parser_.parse(*idx_, std::move(path), c);
+            auto ptr  = file.get();
+            if (file)
+                files_.push_back(std::move(file));
+            return type_safe::opt_ref(ptr);
+        }
+
+        /// \returns The result of [cppast::parser::error]().
+        bool error() const noexcept
+        {
+            return parser_.error();
+        }
+
+        /// \effects Calls [cppast::parser::reset_error]().
+        void reset_error() noexcept
+        {
+            parser_.reset_error();
+        }
+
+        /// \returns The index that is being populated.
+        const cpp_entity_index& index() const noexcept
+        {
+            return *idx_;
+        }
+
+        /// \returns An iteratable object iterating over all the files that have been parsed so far.
+        /// \exclude return
+        detail::iteratable_intrusive_list<cpp_file> files() const noexcept
+        {
+            return type_safe::ref(files_);
+        }
+
+    private:
+        Parser                                        parser_;
+        detail::intrusive_list<cpp_file>              files_;
+        type_safe::object_ref<const cpp_entity_index> idx_;
+    };
+
+    /// Parses multiple files using a given `FileParser`.
+    ///
+    /// \effects Will call the `parse()` function for each path specified in the `file_names`,
+    /// using `get_confg` to determine the configuration.
+    ///
+    /// \requires `FileParser` must be a class with the following members:
+    /// * `parser` - A typedef for the parser being used to do the parsing.
+    /// * `config` - The same as `parser::config`.
+    /// * `parse(path, config)` - Parses the given file with the configuration using its parser.
+    /// The parsing can be executed in a separated thread, but then a copy of the configuration and path must be created.
+    /// \requires `Range` must be some type that can be iterated in a range-based for loop.
+    /// \requires `Configuration` must be a function that returns a configuration of type `FileParser::config` when given a path.
+    /// \unique_name parse_files_basic
+    /// \synopsis_return void
+    template <class FileParser, class Range, class Configuration>
+    auto parse_files(FileParser& parser, Range&& file_names, const Configuration& get_config) ->
+        typename std::enable_if<std::is_same<typename std::decay<decltype(get_config(
+                                                 *std::forward<Range>(file_names).begin()))>::type,
+                                             typename FileParser::config>::value>::type
+    {
+        for (auto&& file : std::forward<Range>(file_names))
+        {
+            auto&& config = get_config(file);
+            parser.parse(std::forward<decltype(file)>(file),
+                         std::forward<decltype(config)>(config));
+        }
+    }
+
+    /// Parses multiple files using a given `FileParser` and configuration.
+    /// \effects Invokes [cppast::parse_files](standardese://parse_files_basic/) passing it the parser and file names,
+    /// and a function that returns the same configuration for each file.
+    template <class FileParser, class Range>
+    void parse_files(FileParser& parser, Range&& file_names, typename FileParser::config config)
+    {
+        parse_files(parser, std::forward<Range>(file_names),
+                    [&](const std::string&) { return config; });
+    }
+
+    template <class FileParser>
+    void resolve_includes(FileParser& parser, const cpp_file& file,
+                          typename FileParser::config config)
+    {
+        for (auto& entity : file)
+        {
+            if (entity.kind() == cpp_include_directive::kind())
+            {
+                auto& include = static_cast<const cpp_include_directive&>(entity);
+                parser.parse(include.full_path(), config);
+            }
+        }
+    }
 } // namespace cppast
 
 #endif // CPPAST_PARSER_HPP_INCLUDED

src/cpp_entity_index.cpp

@@ -7,6 +7,7 @@
 #include <cppast/detail/assert.hpp>
 #include <cppast/cpp_entity.hpp>
 #include <cppast/cpp_entity_kind.hpp>
+#include <cppast/cpp_file.hpp>
 
 using namespace cppast;
 
@@ -33,6 +34,13 @@ void cpp_entity_index::register_definition(cpp_entity_id
     }
 }
 
+bool cpp_entity_index::register_file(cpp_entity_id                         id,
+                                     type_safe::object_ref<const cpp_file> file) const
+{
+    std::lock_guard<std::mutex> lock(mutex_);
+    return map_.emplace(std::move(id), value(file, true)).second;
+}
+
 void cpp_entity_index::register_forward_declaration(
     cpp_entity_id id, type_safe::object_ref<const cpp_entity> entity) const
 {

src/libclang/enum_parser.cpp

@@ -94,10 +94,12 @@ std::unique_ptr<cpp_entity> detail::parse_cpp_enum(const detail::parse_context&
         }
         catch (parse_error& ex)
         {
+            context.error = true;
             context.logger->log("libclang parser", ex.get_diagnostic());
         }
         catch (std::logic_error& ex)
         {
+            context.error = true;
             context.logger->log("libclang parser",
                                 diagnostic{ex.what(), make_location(child), severity::error});
         }

src/libclang/function_parser.cpp

@@ -46,10 +46,12 @@ namespace
                 }
                 catch (detail::parse_error& ex)
                 {
+                    context.error = true;
                     context.logger->log("libclang parser", ex.get_diagnostic());
                 }
                 catch (std::logic_error& ex)
                 {
+                    context.error = true;
                     context.logger->log("libclang parser",
                                         diagnostic{ex.what(), detail::make_location(child),
                                                    severity::error});

src/libclang/libclang_parser.cpp

@@ -7,6 +7,8 @@
 #include <cstring>
 #include <vector>
 
+#include <clang-c/CXCompilationDatabase.h>
+
 #include "libclang_visitor.hpp"
 #include "raii_wrapper.hpp"
 #include "parse_error.hpp"
@@ -33,6 +35,31 @@ const std::vector<std::string>& detail::libclang_compile_config_access::flags(
     return config.get_flags();
 }
 
+libclang_compilation_database::libclang_compilation_database(const std::string& build_directory)
+{
+    static_assert(std::is_same<database, CXCompilationDatabase>::value, "forgot to update type");
+
+    auto error = CXCompilationDatabase_NoError;
+    database_  = clang_CompilationDatabase_fromDirectory(build_directory.c_str(), &error);
+    if (error != CXCompilationDatabase_NoError)
+        throw libclang_error("unable to load compilation database");
+}
+
+libclang_compilation_database::~libclang_compilation_database()
+{
+    if (database_)
+        clang_CompilationDatabase_dispose(database_);
+}
+
+bool libclang_compilation_database::has_config(const char* file_name) const
+{
+    auto cxcommands = clang_CompilationDatabase_getCompileCommands(database_, file_name);
+    if (!cxcommands)
+        return false;
+    clang_CompileCommands_dispose(cxcommands);
+    return true;
+}
+
 namespace
 {
     int parse_number(const char*& str)
@@ -65,6 +92,144 @@ libclang_compile_config::libclang_compile_config() : compile_config({})
     define_macro("__cppast_version_minor__", CPPAST_VERSION_MINOR);
 }
 
+namespace
+{
+    struct cxcompile_commands_deleter
+    {
+        void operator()(CXCompileCommands cmds)
+        {
+            clang_CompileCommands_dispose(cmds);
+        }
+    };
+
+    using cxcompile_commands = detail::raii_wrapper<CXCompileCommands, cxcompile_commands_deleter>;
+
+    bool has_drive_prefix(const std::string& file)
+    {
+        return file.size() > 2 && file[1] == ':';
+    }
+
+    std::string get_full_path(const detail::cxstring& dir, const std::string& file)
+    {
+        if (has_drive_prefix(file) || file.front() == '/' || file.front() == '\\')
+            // absolute file
+            return file;
+        else if (dir[dir.length() - 1] != '/' && dir[dir.length() - 1] != '\\')
+            // relative needing separator
+            return dir.std_str() + '/' + file;
+        else
+            // relative w/o separator
+            return dir.std_str() + file;
+    }
+}
+
+void detail::for_each_file(const libclang_compilation_database& database, void* user_data,
+                           void (*callback)(void*, std::string))
+{
+    cxcompile_commands commands(
+        clang_CompilationDatabase_getAllCompileCommands(database.database_));
+    auto no = clang_CompileCommands_getSize(commands.get());
+    for (auto i = 0u; i != no; ++i)
+    {
+        auto cmd = clang_CompileCommands_getCommand(commands.get(), i);
+
+        auto dir = cxstring(clang_CompileCommand_getDirectory(cmd));
+        callback(user_data,
+                 get_full_path(dir, cxstring(clang_CompileCommand_getFilename(cmd)).std_str()));
+    }
+}
+
+namespace
+{
+    bool is_flag(const detail::cxstring& str)
+    {
+        return str.length() > 1u && str[0] == '-';
+    }
+
+    const char* find_flag_arg_sep(const std::string& last_flag)
+    {
+        if (last_flag[1] == 'D')
+            // no  separator, equal is part of the arg
+            return nullptr;
+        return std::strchr(last_flag.c_str(), '=');
+    }
+
+    template <typename Func>
+    void parse_flags(CXCompileCommand cmd, Func callback)
+    {
+        auto        no_args = clang_CompileCommand_getNumArgs(cmd);
+        std::string last_flag;
+        for (auto i = 1u /* 0 is compiler executable */; i != no_args; ++i)
+        {
+            detail::cxstring str(clang_CompileCommand_getArg(cmd, i));
+            if (is_flag(str))
+            {
+                if (!last_flag.empty())
+                {
+                    // process last flag
+                    std::string args;
+                    if (auto ptr = find_flag_arg_sep(last_flag))
+                    {
+                        auto pos = std::size_t(ptr - last_flag.c_str());
+                        ++ptr;
+                        while (*ptr)
+                            args += *ptr++;
+                        last_flag.erase(pos);
+                    }
+                    else if (last_flag.size() > 2u)
+                    {
+                        // assume two character flag
+                        args = last_flag.substr(2u);
+                        last_flag.erase(2u);
+                    }
+
+                    callback(std::move(last_flag), std::move(args));
+                }
+
+                last_flag = str.std_str();
+            }
+            else if (!last_flag.empty())
+            {
+                // we have flags + args
+                callback(std::move(last_flag), str.std_str());
+                last_flag.clear();
+            }
+            // else skip argument
+        }
+    }
+}
+
+libclang_compile_config::libclang_compile_config(const libclang_compilation_database& database,
+                                                 const std::string&                   file)
+: libclang_compile_config()
+{
+    auto cxcommands =
+        clang_CompilationDatabase_getCompileCommands(database.database_, file.c_str());
+    if (cxcommands == nullptr)
+        throw libclang_error(detail::format("no compile commands specified for file '", file, "'"));
+    cxcompile_commands commands(cxcommands);
+
+    auto size = clang_CompileCommands_getSize(commands.get());
+    for (auto i = 0u; i != size; ++i)
+    {
+        auto cmd = clang_CompileCommands_getCommand(commands.get(), i);
+        auto dir = detail::cxstring(clang_CompileCommand_getDirectory(cmd));
+        parse_flags(cmd, [&](std::string flag, std::string args) {
+            if (flag == "-I")
+                add_flag(std::move(flag) + get_full_path(dir, args));
+            else if (flag == "-D" || flag == "-U")
+                // preprocessor options
+                add_flag(std::move(flag) + std::move(args));
+            else if (flag == "-std")
+                // standard
+                add_flag(std::move(flag) + "=" + std::move(args));
+            else if (flag == "-f" && (args == "ms-compatibility" || args == "ms-extensions"))
+                // other options
+                add_flag(std::move(flag) + std::move(args));
+        });
+    }
+}
+
 void libclang_compile_config::do_set_flags(cpp_standard standard, compile_flags flags)
 {
     switch (standard)
@@ -125,6 +290,30 @@ void libclang_compile_config::do_remove_macro_definition(std::string name)
     add_flag("-U" + std::move(name));
 }
 
+type_safe::optional<libclang_compile_config> cppast::find_config_for(
+    const libclang_compilation_database& database, std::string file_name)
+{
+    if (database.has_config(file_name))
+        return libclang_compile_config(database, std::move(file_name));
+
+    auto dot = file_name.rfind('.');
+    if (dot != std::string::npos)
+        file_name.erase(dot);
+
+    if (database.has_config(file_name))
+        return libclang_compile_config(database, std::move(file_name));
+    static const char* extensions[] = {".h",   ".hpp", ".cpp", ".h++", ".c++", ".hxx",
+                                       ".cxx", ".hh",  ".cc",  ".H",   ".C"};
+    for (auto ext : extensions)
+    {
+        auto name = file_name + ext;
+        if (database.has_config(name))
+            return libclang_compile_config(database, std::move(name));
+    }
+
+    return type_safe::nullopt;
+}
+
 struct libclang_parser::impl
 {
     detail::cxindex index;
@@ -264,9 +453,13 @@ std::unique_ptr<cpp_file> libclang_parser::do_parse(const cpp_entity_index& idx,
     auto              macro_iter   = preprocessed.macros.begin();
     auto              include_iter = preprocessed.includes.begin();
 
-    // convert entity hierachies
+    // convert entity hierarchies
-    detail::parse_context context{tu.get(), file, type_safe::ref(logger()), type_safe::ref(idx),
+    detail::parse_context context{tu.get(),
-                                  detail::comment_context(preprocessed.comments)};
+                                  file,
+                                  type_safe::ref(logger()),
+                                  type_safe::ref(idx),
+                                  detail::comment_context(preprocessed.comments),
+                                  false};
     detail::visit_tu(tu, path.c_str(), [&](const CXCursor& cur) {
         if (clang_getCursorKind(cur) == CXCursor_InclusionDirective)
         {
@@ -304,10 +497,14 @@ std::unique_ptr<cpp_file> libclang_parser::do_parse(const cpp_entity_index& idx,
             builder.add_unmatched_comment(std::move(c.comment));
     }
 
+    if (context.error)
+        set_error();
+
     return builder.finish(idx);
 }
 catch (detail::parse_error& ex)
 {
     logger().log("libclang parser", ex.get_diagnostic());
-    return cpp_file::builder(path).finish(idx);
+    set_error();
+    return nullptr;
 }

src/libclang/parse_functions.cpp

@@ -230,11 +230,13 @@ std::unique_ptr<cpp_entity> detail::parse_entity(const detail::parse_context& co
 }
 catch (parse_error& ex)
 {
+    context.error = true;
     context.logger->log("libclang parser", ex.get_diagnostic());
     return nullptr;
 }
 catch (std::logic_error& ex)
 {
+    context.error = true;
     context.logger->log("libclang parser",
                         diagnostic{ex.what(), detail::make_location(cur), severity::error});
     return nullptr;

src/libclang/parse_functions.hpp

@@ -62,6 +62,7 @@ namespace cppast
             type_safe::object_ref<const diagnostic_logger> logger;
             type_safe::object_ref<const cpp_entity_index>  idx;
             comment_context                                comments;
+            mutable bool                                   error;
         };
 
         // parse default value of variable, function parameter...

src/parser.cpp

@@ -13,13 +13,23 @@ using namespace cppast;
 
 bool diagnostic_logger::log(const char* source, const diagnostic& d) const
 {
-    if (d.severity == severity::error || d.severity == severity::critical)
+    if (!verbose_ && d.severity == severity::debug)
-        error_ = true;
-    else if (!verbose_ && d.severity == severity::debug)
         return false;
     return do_log(source, d);
 }
 
+type_safe::object_ref<const diagnostic_logger> cppast::default_logger() noexcept
+{
+    static const stderr_diagnostic_logger logger(false);
+    return type_safe::ref(logger);
+}
+
+type_safe::object_ref<const diagnostic_logger> cppast::default_verbose_logger() noexcept
+{
+    static const stderr_diagnostic_logger logger(true);
+    return type_safe::ref(logger);
+}
+
 bool stderr_diagnostic_logger::do_log(const char* source, const diagnostic& d) const
 {
     auto loc = d.location.to_string();

test/CMakeLists.txt

@@ -9,25 +9,27 @@ if(NOT EXISTS ${CMAKE_CURRENT_BINARY_DIR}/catch.hpp)
 endif()
 
 set(tests
-    code_generator.cpp
+        code_generator.cpp
-    cpp_alias_template.cpp
+        cpp_alias_template.cpp
-    cpp_class.cpp
+        cpp_class.cpp
-    cpp_class_template.cpp
+        cpp_class_template.cpp
-    cpp_enum.cpp
+        cpp_enum.cpp
-    cpp_friend.cpp
+        cpp_friend.cpp
-    cpp_function.cpp
+        cpp_function.cpp
-    cpp_function_template.cpp
+        cpp_function_template.cpp
-    cpp_language_linkage.cpp
+        cpp_language_linkage.cpp
-    cpp_member_function.cpp
+        cpp_member_function.cpp
-    cpp_member_variable.cpp
+        cpp_member_variable.cpp
-    cpp_namespace.cpp
+        cpp_namespace.cpp
-    cpp_preprocessor.cpp
+        cpp_preprocessor.cpp
-    cpp_static_assert.cpp
+        cpp_static_assert.cpp
-    cpp_template_parameter.cpp
+        cpp_template_parameter.cpp
-    cpp_type_alias.cpp
+        cpp_type_alias.cpp
-    cpp_variable.cpp
+        cpp_variable.cpp
-    visitor.cpp
+        integration.cpp
-    integration.cpp)
+        libclang_parser.cpp
+        parser.cpp
+        visitor.cpp)
 
 add_executable(cppast_test test.cpp test_parser.hpp ${tests})
 target_include_directories(cppast_test PUBLIC ${CMAKE_CURRENT_BINARY_DIR})

test/integration.cpp

@@ -8,18 +8,6 @@
 
 using namespace cppast;
 
-void parse_included_files(const cpp_entity_index& idx, const cpp_file& file)
-{
-    for (auto& e : file)
-    {
-        if (e.kind() == cpp_entity_kind::include_directive_t)
-        {
-            auto path = static_cast<const cpp_include_directive&>(e).full_path();
-            parse_file(idx, path.c_str());
-        }
-    }
-}
-
 TEST_CASE("stdlib", "[!hide][integration]")
 {
     auto code = R"(
@@ -111,8 +99,17 @@ TEST_CASE("stdlib", "[!hide][integration]")
 #include <future>
 #include <condition_variable>
 )";
+    write_file("stdlib.cpp", code);
 
-    cpp_entity_index idx;
+    cpp_entity_index                    idx;
-    auto             file = parse(idx, "stdlib.cpp", code);
+    simple_file_parser<libclang_parser> parser(type_safe::ref(idx), default_logger());
-    parse_included_files(idx, *file);
+
+    libclang_compile_config config;
+    config.set_flags(cpp_standard::cpp_latest);
+    auto file = parser.parse("stdlib.cpp", config);
+    REQUIRE(!parser.error());
+    REQUIRE(file);
+
+    resolve_includes(parser, file.value(), config);
+    REQUIRE(!parser.error());
 }

test/libclang_parser.cpp

@@ -0,0 +1,102 @@
+// Copyright (C) 2017 Jonathan Müller <jonathanmueller.dev@gmail.com>
+// This file is subject to the license terms in the LICENSE file
+// found in the top-level directory of this distribution.
+
+#include <catch.hpp>
+
+#include <cppast/libclang_parser.hpp>
+
+#include <fstream>
+
+using namespace cppast;
+
+libclang_compilation_database get_database(const char* json)
+{
+    std::ofstream file("compile_commands.json");
+    file << json;
+    file.close();
+
+    return libclang_compilation_database(".");
+}
+
+void require_flags(const libclang_compile_config& config, const char* flags)
+{
+    std::string result;
+    auto        config_flags = detail::libclang_compile_config_access::flags(config);
+    // skip first 4, those are the default options
+    for (auto iter = config_flags.begin() + 4; iter != config_flags.end(); ++iter)
+        result += *iter + ' ';
+    result.pop_back();
+    REQUIRE(result == flags);
+}
+
+TEST_CASE("libclang_compile_config")
+{
+// only test database parser
+#ifdef _WIN32
+    auto json = R"([
+{
+    "directory": "C:/foo",
+    "command": "/usr/bin/clang++ -Irelative -IC:/absolute -DA=FOO -DB(X)=X -c -o a.o a.cpp",
+    "file": "a.cpp"
+},
+{
+    "directory": "C:/bar/",
+    "command": "/usr/bin/clang++ -Irelative -DA=FOO -c -o b.o b.cpp",
+    "file": "C:/b.cpp",
+},
+{
+    "directory": "C:/bar/",
+    "command": "/usr/bin/clang++ -IC:/absolute -DB(X)=X -c -o b.o b.cpp",
+    "file": "C:/b.cpp",
+},
+{
+    "directory": "",
+    "command": "/usr/bin/clang++ -std=c++14 -fms-extensions -fms-compatibility -c -o c.o c.cpp",
+    "file": "C:/c.cpp",
+}
+])";
+
+#define CPPAST_DETAIL_DRIVE "C:"
+
+#else
+    auto json = R"([
+{
+    "directory": "/foo",
+    "command": "/usr/bin/clang++ -Irelative -I/absolute -DA=FOO -DB(X)=X -c -o a.o a.cpp",
+    "file": "a.cpp"
+},
+{
+    "directory": "/bar/",
+    "command": "/usr/bin/clang++ -Irelative -DA=FOO -c -o b.o b.cpp",
+    "file": "/b.cpp",
+},
+{
+    "directory": "/bar/",
+    "command": "/usr/bin/clang++ -I/absolute -DB(X)=X -c -o b.o b.cpp",
+    "file": "/b.cpp",
+},
+{
+    "directory": "",
+    "command": "/usr/bin/clang++ -std=c++14 -fms-extensions -fms-compatibility -c -o c.o c.cpp",
+    "file": "/c.cpp",
+}
+])";
+
+#define CPPAST_DETAIL_DRIVE
+
+#endif
+
+    auto database = get_database(json);
+
+    libclang_compile_config a(database, CPPAST_DETAIL_DRIVE "/foo/a.cpp");
+    require_flags(a, "-I" CPPAST_DETAIL_DRIVE "/foo/relative -I" CPPAST_DETAIL_DRIVE
+                     "/absolute -DA=FOO -DB(X)=X");
+
+    libclang_compile_config b(database, CPPAST_DETAIL_DRIVE "/b.cpp");
+    require_flags(b, "-I" CPPAST_DETAIL_DRIVE "/bar/relative -DA=FOO -I" CPPAST_DETAIL_DRIVE
+                     "/absolute -DB(X)=X");
+
+    libclang_compile_config c(database, CPPAST_DETAIL_DRIVE "/c.cpp");
+    require_flags(c, "-std=c++14 -fms-extensions -fms-compatibility");
+}

test/parser.cpp

@@ -0,0 +1,71 @@
+// Copyright (C) 2017 Jonathan Müller <jonathanmueller.dev@gmail.com>
+// This file is subject to the license terms in the LICENSE file
+// found in the top-level directory of this distribution.
+
+#include <cppast/parser.hpp>
+
+#include <catch.hpp>
+
+using namespace cppast;
+
+TEST_CASE("parse_files")
+{
+    class null_compile_config : public compile_config
+    {
+    public:
+        null_compile_config() : compile_config({})
+        {
+        }
+
+    private:
+        void do_set_flags(cpp_standard, compile_flags) override
+        {
+        }
+
+        void do_add_include_dir(std::string) override
+        {
+        }
+
+        void do_add_macro_definition(std::string, std::string) override
+        {
+        }
+
+        void do_remove_macro_definition(std::string) override
+        {
+        }
+
+        const char* do_get_name() const noexcept override
+        {
+            return "null";
+        }
+    } config;
+
+    class null_parser : public parser
+    {
+    public:
+        using config = null_compile_config;
+
+        null_parser() : parser(type_safe::ref(logger_))
+        {
+        }
+
+    private:
+        std::unique_ptr<cpp_file> do_parse(const cpp_entity_index& idx, std::string path,
+                                           const compile_config&) const override
+        {
+            return cpp_file::builder(std::move(path)).finish(idx);
+        }
+
+        stderr_diagnostic_logger logger_;
+    };
+
+    cpp_entity_index                idx;
+    simple_file_parser<null_parser> parser(type_safe::ref(idx));
+
+    auto file_names = {"a.cpp", "b.cpp", "c.cpp"};
+    parse_files(parser, file_names, config);
+
+    auto iter = file_names.begin();
+    for (auto& file : parser.files())
+        REQUIRE(file.name() == *iter++);
+}

test/test_parser.hpp

@@ -31,12 +31,11 @@ inline std::unique_ptr<cppast::cpp_file> parse_file(const cppast::cpp_entity_ind
     libclang_compile_config config;
     config.set_flags(cpp_standard::cpp_latest);
 
-    stderr_diagnostic_logger logger;
+    libclang_parser p(default_logger());
-    libclang_parser          p(type_safe::ref(logger));
 
     std::unique_ptr<cppast::cpp_file> result;
     REQUIRE_NOTHROW(result = p.parse(idx, name, config));
-    REQUIRE(!logger.error_logged());
+    REQUIRE(!p.error());
     return result;
 }
 

tool/main.cpp

@@ -180,7 +180,7 @@ std::unique_ptr<cppast::cpp_file> parse_file(const cppast::libclang_compile_conf
     cppast::libclang_parser parser(type_safe::ref(logger));
     // parse the file
     auto file = parser.parse(idx, filename, config);
-    if (fatal_error && logger.error_logged())
+    if (fatal_error && parser.error())
         return nullptr;
     return file;
 }