Fixes to YAML highlighting support, added tests

lib/packages/docutils/highlite.nim

@@ -590,20 +590,21 @@ proc yamlPlainStrLit(g: var GeneralTokenizer, pos: var int) =
 proc yamlPossibleNumber(g: var GeneralTokenizer, pos: var int) =
   g.kind = gtNone
   if g.buf[pos] == '-': inc(pos)
-  if g.buf[pos] == '0':
+  if g.buf[pos] == '0': inc(pos)
-    inc(pos)
   elif g.buf[pos] in '1'..'9':
     inc(pos)
     while g.buf[pos] in {'0'..'9'}: inc(pos)
   else: yamlPlainStrLit(g, pos)
   if g.kind == gtNone:
-    if g.buf[pos] in {'\0', '\x09'..'\x0D', ' '}: g.kind = gtDecNumber
+    if g.buf[pos] in {'\0', '\x09'..'\x0D', ' ', ',', ']', '}'}:
+      g.kind = gtDecNumber
     elif g.buf[pos] == '.':
       inc(pos)
       if g.buf[pos] notin {'0'..'9'}: yamlPlainStrLit(g, pos)
       else:
         while g.buf[pos] in {'0'..'9'}: inc(pos)
-        if g.buf[pos] in {'\0', '\x09'..'\x0D', ' '}: g.kind = gtFloatNumber
+        if g.buf[pos] in {'\0', '\x09'..'\x0D', ' ', ',', ']', '}'}:
+          g.kind = gtFloatNumber
     if g.kind == gtNone:
       if g.buf[pos] in {'e', 'E'}:
         inc(pos)
@@ -611,9 +612,19 @@ proc yamlPossibleNumber(g: var GeneralTokenizer, pos: var int) =
         if g.buf[pos] notin {'0'..'9'}: yamlPlainStrLit(g, pos)
         else:
           while g.buf[pos] in {'0'..'9'}: inc(pos)
-          if g.buf[pos] in {'\0', '\x09'..'\x0D', ' '}: g.kind = gtFloatNumber
+          if g.buf[pos] in {'\0', '\x09'..'\x0D', ' ', ',', ']', '}'}:
+            g.kind = gtFloatNumber
           else: yamlPlainStrLit(g, pos)
       else: yamlPlainStrLit(g, pos)
+  while g.buf[pos] notin {'\0', ',', ']', '}', '\x0A', '\x0D'}:
+    inc(pos)
+    if g.buf[pos] notin {'\x09'..'\x0D', ' ', ',', ']', '}'}:
+      yamlPlainStrLit(g, pos)
+      break
+  # theoretically, we would need to parse indentation (like with block scalars)
+  # because of possible multiline flow scalars that start with number-like
+  # content, but that is far too troublesome. I think it is fine that the
+  # highlighter is sloppy here.
 
 proc yamlNextToken(g: var GeneralTokenizer) =
   const
@@ -660,6 +671,7 @@ proc yamlNextToken(g: var GeneralTokenizer) =
   elif g.state == gtCharLit:
     # abusing gtCharLit as single-quoted string lit
     g.kind = gtStringLit
+    inc(pos) # skip the starting '
     while true:
       case g.buf[pos]
       of '\'':
@@ -807,9 +819,8 @@ proc yamlNextToken(g: var GeneralTokenizer) =
     of '\"':
       inc(pos)
       g.state = gtStringLit
-      g.kind = gtNone
+      g.kind = gtStringLit
     of '\'':
-      inc(pos)
       g.state = gtCharLit
       g.kind = gtNone
     of '!':

tests/stdlib/trstgen.nim

@@ -0,0 +1,139 @@
+# tests for rstgen module.
+
+import ../../lib/packages/docutils/rstgen
+import unittest
+
+suite "YAML syntax highlighting":
+  test "Basics":
+    let input = """.. code-block:: yaml
+    %YAML 1.2
+    ---
+    a string: string
+    a list:
+      - item 1
+      - item 2
+    a map:
+    ? key
+    : value
+    ..."""
+    let output = rstTohtml(input, {}, defaultConfig())
+    assert output == """<pre class = "listing"><span class="Directive">%YAML 1.2</span>
+<span class="Keyword">---</span>
+<span class="StringLit">a string</span><span class="Punctuation">:</span> <span class="StringLit">string</span>
+<span class="StringLit">a list</span><span class="Punctuation">:</span>
+  <span class="Punctuation">-</span> <span class="StringLit">item 1</span>
+  <span class="Punctuation">-</span> <span class="StringLit">item 2</span>
+<span class="StringLit">a map</span><span class="Punctuation">:</span>
+<span class="Punctuation">?</span> <span class="StringLit">key</span>
+<span class="Punctuation">:</span> <span class="StringLit">value</span>
+<span class="Keyword">...</span></pre>"""
+  
+  test "Block scalars":
+    let input = """.. code-block:: yaml
+    a literal block scalar: |
+      some text
+      # not a comment
+     # a comment, since less indented
+      # another comment
+    a folded block scalar: >2
+       some text
+      # not a comment since indented as specified
+     # a comment
+    another literal block scalar:
+      |+ # comment after header
+     allowed, since more indented than parent"""
+    let output = rstToHtml(input, {}, defaultConfig())
+    assert output == """<pre class = "listing"><span class="StringLit">a literal block scalar</span><span class="Punctuation">:</span> <span class="Command">|</span><span class="Command"></span><span class="LongStringLit">
+  some text
+  # not a comment
+ </span><span class="Comment"># a comment, since less indented</span>
+  <span class="Comment"># another comment</span>
+<span class="StringLit">a folded block scalar</span><span class="Punctuation">:</span> <span class="Command">&gt;2</span><span class="Command"></span><span class="LongStringLit">
+   some text
+  # not a comment since indented as specified
+ </span><span class="Comment"># a comment</span>
+<span class="StringLit">another literal block scalar</span><span class="Punctuation">:</span>
+  <span class="Command">|+</span> <span class="Comment"># comment after header</span><span class="LongStringLit">
+ allowed, since more indented than parent</span></pre>"""
+ 
+  test "Directives":
+    let input = """.. code-block:: yaml
+    %YAML 1.2
+    ---
+    %not a directive
+    ...
+    %a directive
+    ...
+    a string
+    % not a directive
+    ...
+    %TAG ! !foo:"""
+    let output = rstToHtml(input, {}, defaultConfig())
+    assert output == """<pre class = "listing"><span class="Directive">%YAML 1.2</span>
+<span class="Keyword">---</span>
+<span class="StringLit">%not a directive</span>
+<span class="Keyword">...</span>
+<span class="Directive">%a directive</span>
+<span class="Keyword">...</span>
+<span class="StringLit">a string</span>
+<span class="StringLit">% not a directive</span>
+<span class="Keyword">...</span>
+<span class="Directive">%TAG ! !foo:</span></pre>"""
+
+  test "Flow Style and Numbers":
+    let input = """.. code-block:: yaml
+    {
+      "quoted string": 42,
+      'single quoted string': false,
+      [ list, "with", 'entries' ]: 73.32e-73,
+      more numbers: [-783, 11e78],
+      not numbers: [ 42e, 0023, +32.37, 8 ball]
+    }"""
+    let output = rstToHtml(input, {}, defaultConfig())
+    assert output == """<pre class = "listing"><span class="Punctuation">{</span>
+  <span class="StringLit">&quot;</span><span class="StringLit">quoted string&quot;</span><span class="Punctuation">:</span> <span class="DecNumber">42</span><span class="Punctuation">,</span>
+  <span class="StringLit">'single quoted string'</span><span class="Punctuation">:</span> <span class="StringLit">false</span><span class="Punctuation">,</span>
+  <span class="Punctuation">[</span> <span class="StringLit">list</span><span class="Punctuation">,</span> <span class="StringLit">&quot;</span><span class="StringLit">with&quot;</span><span class="Punctuation">,</span> <span class="StringLit">'entries'</span> <span class="Punctuation">]</span><span class="Punctuation">:</span> <span class="FloatNumber">73.32e-73</span><span class="Punctuation">,</span>
+  <span class="StringLit">more numbers</span><span class="Punctuation">:</span> <span class="Punctuation">[</span><span class="DecNumber">-783</span><span class="Punctuation">,</span> <span class="FloatNumber">11e78</span><span class="Punctuation">]</span><span class="Punctuation">,</span>
+  <span class="StringLit">not numbers</span><span class="Punctuation">:</span> <span class="Punctuation">[</span> <span class="StringLit">42e</span><span class="Punctuation">,</span> <span class="StringLit">0023</span><span class="Punctuation">,</span> <span class="StringLit">+32.37</span><span class="Punctuation">,</span> <span class="StringLit">8 ball</span><span class="Punctuation">]</span>
+<span class="Punctuation">}</span></pre>"""
+  
+  test "Anchors, Aliases, Tags":
+    let input = """.. code-block:: yaml
+    --- !!map
+    !!str string: !<tag:yaml.org,2002:int> 42
+    ? &anchor !!seq []:
+    : !localtag foo
+    alias: *anchor
+    """
+    let output = rstToHtml(input, {}, defaultConfig())
+    assert output == """<pre class = "listing"><span class="Keyword">---</span> <span class="TagStart">!!map</span>
+<span class="TagStart">!!str</span> <span class="StringLit">string</span><span class="Punctuation">:</span> <span class="TagStart">!&lt;tag:yaml.org,2002:int&gt;</span> <span class="DecNumber">42</span>
+<span class="Punctuation">?</span> <span class="Label">&amp;anchor</span> <span class="TagStart">!!seq</span> <span class="Punctuation">[</span><span class="Punctuation">]</span><span class="Punctuation">:</span>
+<span class="Punctuation">:</span> <span class="TagStart">!localtag</span> <span class="StringLit">foo</span>
+<span class="StringLit">alias</span><span class="Punctuation">:</span> <span class="Reference">*anchor</span></pre>"""
+
+  test "Edge cases":
+    let input = """.. code-block:: yaml
+    ...
+     %a string:
+      a:string:not:a:map
+    ...
+    not a list:
+      -2
+      -3
+      -4
+    example.com/not/a#comment:
+      ?not a map key
+    """
+    let output = rstToHtml(input, {}, defaultConfig())
+    assert output == """<pre class = "listing"><span class="Keyword">...</span>
+ <span class="StringLit">%a string</span><span class="Punctuation">:</span>
+  <span class="StringLit">a:string:not:a:map</span>
+<span class="Keyword">...</span>
+<span class="StringLit">not a list</span><span class="Punctuation">:</span>
+  <span class="DecNumber">-2</span>
+  <span class="DecNumber">-3</span>
+  <span class="DecNumber">-4</span>
+<span class="StringLit">example.com/not/a#comment</span><span class="Punctuation">:</span>
+  <span class="StringLit">?not a map key</span></pre>"""