Simplify tokenizer

Fixes #26.

test/cpp_attribute.cpp

@@ -140,6 +140,7 @@ alignas(type) int var;
     count = test_visit<cpp_variable>(*file,
                                      [&](const cpp_entity& e) {
                                          auto& attributes = e.attributes();
+                                         INFO(e.name());
                                          REQUIRE(attributes.size() == 1u);
                                          auto& attr = attributes.front();
                                          check_attribute(attr, "alignas", type_safe::nullopt, false,
@@ -215,6 +216,18 @@ using o [[o]] = int;
 
 template <typename T>
 using p [[p]] = T;
+
+// constructor
+struct [[q]] q
+{
+    [[q]] q();
+};
+
+struct [[r]] r
+{
+    [[r]]
+    r();
+};
 )";
 
     auto file = parse({}, "cpp_attribute__matching.cpp", code);
@@ -242,5 +255,5 @@ using p [[p]] = T;
 
         return true;
     });
-    REQUIRE(count == 36u);
+    REQUIRE(count == 40u);
 }

test/cpp_preprocessor.cpp

@@ -93,8 +93,7 @@ namespace ns2
     }
 }
 
-// requires clang 4.0
-TEST_CASE("cpp_include_directive", "[!hide][clang4]")
+TEST_CASE("cpp_include_directive")
 {
     write_file("cpp_include_directive-header.hpp", R"(
 #define FOO a\