add RST highlighting for command line / shells (also fixes #16858) (#17789)

config/nimdoc.tex.cfg

@@ -137,6 +137,8 @@ bottomline=false}
 \newcommand{\spanReference}[1]{#1}
 \newcommand{\spanOther}[1]{#1}
 \newcommand{\spantok}[1]{\frame{#1}}
+\newcommand{\spanprogram}[1]{\textbf{\underline{#1}}}
+\newcommand{\spanoption}[1]{\textbf{#1}}
 
 $content
 \end{document}

doc/contributing.rst

@@ -1,9 +1,10 @@
-.. default-role:: code
-
 ============
 Contributing
 ============
 
+.. default-role:: code
+.. include:: rstcommon.rst
+
 .. contents::
 
 
@@ -19,21 +20,23 @@ Writing tests
 
 There are 4 types of tests:
 
-1. `runnableExamples` documentation comment tests, ran by `nim doc mymod.nim`
+1. `runnableExamples` documentation comment tests, ran by `nim doc mymod.nim`:cmd:
    These end up in documentation and ensure documentation stays in sync with code.
 
-2. separate test files, e.g.: `tests/stdlib/tos.nim`.
+2. separate test files, e.g.: ``tests/stdlib/tos.nim``.
-   In nim repo, `testament` (see below) runs all `$nim/tests/*/t*.nim` test files;
+   In nim repo, `testament`:cmd: (see below) runs all
+   ``$nim/tests/*/t*.nim`` test files;
    for nimble packages, see https://github.com/nim-lang/nimble#tests.
 
-3. (deprecated) tests in `when isMainModule:` block, ran by `nim r mymod.nim`.
+3. (deprecated) tests in `when isMainModule:` block, ran by `nim r mymod.nim`:cmd:.
-   `nimble test` can run those in nimble packages when specified in a
+   `nimble test`:cmd: can run those in nimble packages when specified in a
    `task "test"`.
 
-4. (not preferred) `.. code-block:: nim` RST snippets; these should only be used in rst sources,
+4. (not preferred) ``.. code-block:: nim`` RST snippets;
+   these should only be used in rst sources,
    in nim sources `runnableExamples` should now always be preferred to those for
    several reasons (cleaner syntax, syntax highlights, batched testing, and
-   `rdoccmd` allows customization).
+   parameter `rdoccmd` allows customization).
 
 Not all the tests follow the convention here, feel free to change the ones
 that don't. Always leave the code cleaner than you found it.
@@ -41,8 +44,8 @@ that don't. Always leave the code cleaner than you found it.
 Stdlib
 ------
 
-Each stdlib module (anything under `lib/`, e.g. `lib/pure/os.nim`) should
+Each stdlib module (anything under ``lib/``, e.g. ``lib/pure/os.nim``) should
-preferably have a corresponding separate test file, e.g. `tests/stdlib/tos.nim`.
+preferably have a corresponding separate test file, e.g. ``tests/stdlib/tos.nim``.
 The old convention was to add a `when isMainModule:` block in the source file,
 which only gets executed when the tester is building the file.
 
@@ -71,36 +74,36 @@ Sample test:
     # doAssert with `not` can now be done as follows:
     doAssert not (1 == 2)
 
-Always refer to a GitHub issue using the following exact syntax: `bug #1234` as shown
+Always refer to a GitHub issue using the following exact syntax: ``bug #1234`` as shown
 above, so that it's consistent and easier to search or for tooling. Some browser
 extensions (e.g. https://github.com/sindresorhus/refined-github) will even turn those
 in clickable links when it works.
 
 Rationale for using a separate test file instead of `when isMainModule:` block:
 * allows custom compiler flags or testing options (see details below)
-* faster CI since they can be joined in `megatest` (combined into a single test)
+* faster CI since they can be joined in ``megatest`` (combined into a single test)
 * avoids making the parser do un-necessary work when a source file is merely imported
 * avoids mixing source and test code when reporting line of code statistics or code coverage
 
 Compiler
 --------
 
-The tests for the compiler use a testing tool called `testament`. They are all
+The tests for the compiler use a testing tool called `testament`:cmd:. They are all
-located in `tests/` (e.g.: `tests/destructor/tdestructor3.nim`).
+located in ``tests/`` (e.g.: ``tests/destructor/tdestructor3.nim``).
 Each test has its own file. All test files are prefixed with `t`. If you want
 to create a file for import into another test only, use the prefix `m`.
 
 At the beginning of every test is the expected behavior of the test.
 Possible keys are:
 
-- `cmd`: A compilation command template e.g. `nim $target --threads:on $options $file`
+- `cmd`: A compilation command template e.g. `nim $target --threads:on $options $file`:cmd:
 - `output`: The expected output (stdout + stderr), most likely via `echo`
 - `exitcode`: Exit code of the test (via `exit(number)`)
 - `errormsg`: The expected compiler error message
 - `file`: The file the errormsg was produced at
 - `line`: The line the errormsg was produced at
 
-For a full spec, see here: `testament/specs.nim`
+For a full spec, see here: ``testament/specs.nim``
 
 An example of a test:
 
@@ -124,51 +127,51 @@ Running tests
 
 You can run the tests with
 
-::
+.. code-block:: cmd
 
   ./koch tests
 
 which will run a good subset of tests. Some tests may fail. If you
 only want to see the output of failing tests, go for
 
-::
+```cmd
-
   ./koch tests --failing all
+```
 
 You can also run only a single category of tests. A category is a subdirectory
-in the `tests` directory. There are a couple of special categories; for a
+in the ``tests/`` directory. There are a couple of special categories; for a
-list of these, see `testament/categories.nim`, at the bottom.
+list of these, see ``testament/categories.nim``, at the bottom.
 
-::
+.. code:: cmd
 
   ./koch tests c lib # compiles / runs stdlib modules, including `isMainModule` tests
   ./koch tests c megatest # runs a set of tests that can be combined into 1
 
 To run a single test:
 
-::
+.. code:: cmd
 
   ./koch test run <category>/<name>    # e.g.: tuples/ttuples_issues
   ./koch test run tests/stdlib/tos.nim # can also provide relative path
 
 For reproducible tests (to reproduce an environment more similar to the one
 run by Continuous Integration on travis/appveyor), you may want to disable your
-local configuration (e.g. in `~/.config/nim/nim.cfg`) which may affect some
+local configuration (e.g. in ``~/.config/nim/nim.cfg``) which may affect some
 tests; this can also be achieved by using
-`export XDG_CONFIG_HOME=pathtoAlternateConfig` before running `./koch`
+`export XDG_CONFIG_HOME=pathtoAlternateConfig`:cmd: before running `./koch`:cmd:
 commands.
 
 
 Comparing tests
 ===============
 
-Test failures can be grepped using `Failure:`.
+Test failures can be grepped using ``Failure:``.
 
 The tester can compare two test runs. First, you need to create a
 reference test. You'll also need to the commit id, because that's what
 the tester needs to know in order to compare the two.
 
-::
+.. code:: cmd
 
   git checkout devel
   DEVEL_COMMIT=$(git rev-parse HEAD)
@@ -176,15 +179,15 @@ the tester needs to know in order to compare the two.
 
 Then switch over to your changes and run the tester again.
 
-::
+.. code:: cmd
 
   git checkout your-changes
   ./koch tests
 
-Then you can ask the tester to create a `testresults.html` which will
+Then you can ask the tester to create a ``testresults.html`` which will
 tell you if any new tests passed/failed.
 
-::
+.. code:: cmd
 
   ./koch tests --print html $DEVEL_COMMIT
 
@@ -219,14 +222,14 @@ Documentation
 
 When contributing new procs, be sure to add documentation, especially if
 the proc is public. Even private procs benefit from documentation and can be
-viewed using `nim doc --docInternal foo.nim`.
+viewed using `nim doc --docInternal foo.nim`:cmd:.
 Documentation begins on the line
 following the `proc` definition, and is prefixed by `##` on each line.
 
 Runnable code examples are also encouraged, to show typical behavior with a few
 test cases (typically 1 to 3 `assert` statements, depending on complexity).
-These `runnableExamples` are automatically run by `nim doc mymodule.nim`
+These `runnableExamples` are automatically run by `nim doc mymodule.nim`:cmd:
-as well as `testament` and guarantee they stay in sync.
+as well as `testament`:cmd: and guarantee they stay in sync.
 
 .. code-block:: nim
   proc addBar*(a: string): string =
@@ -238,7 +241,7 @@ as well as `testament` and guarantee they stay in sync.
 See `parentDir <os.html#parentDir,string>`_ example.
 
 The RestructuredText Nim uses has a special syntax for including code snippets
-embedded in documentation; these are not run by `nim doc` and therefore are
+embedded in documentation; these are not run by `nim doc`:cmd: and therefore are
 not guaranteed to stay in sync, so `runnableExamples` is almost always preferred:
 
 .. code-block:: nim
@@ -250,9 +253,9 @@ not guaranteed to stay in sync, so `runnableExamples` is almost always preferred
     ##  echo someProc() # "something"
     result = "something" # single-hash comments do not produce documentation
 
-The `.. code-block:: nim` followed by a newline and an indentation instructs the
+The ``.. code-block:: nim`` followed by a newline and an indentation instructs the
-`nim doc` command to produce syntax-highlighted example code with the
+`nim doc`:cmd: command to produce syntax-highlighted example code with the
-documentation (`.. code-block::` is sufficient from inside a nim module).
+documentation (``.. code-block::`` is sufficient from inside a nim module).
 
 When forward declaration is used, the documentation should be included with the
 first appearance of the proc.
@@ -298,10 +301,10 @@ example below) from `Nim Index`_ can be used in doc comment this way:
 Inline monospaced text can be input using \`single backticks\` or
 \`\`double backticks\`\`. The former are syntactically highlighted,
 the latter are not.
-To avoid accidental highlighting follow this rule in `*.nim` files:
+To avoid accidental highlighting follow this rule in ``*.nim`` files:
 
 * use single backticks for fragments of code in Nim and other
-  programming languages, including identifiers, in `*.nim` files.
+  programming languages, including identifiers, in ``*.nim`` files.
 
   For languages other than Nim add a role after final backtick,
   e.g. for C++ inline highlighting::
@@ -313,18 +316,20 @@ To avoid accidental highlighting follow this rule in `*.nim` files:
 
     `SELECT * FROM <table_name>;`:code:
 
+  Highlight shell commands by ``:cmd:`` role; for command line options use
+  ``:option:`` role, e.g.: \`--docInternal\`:option:.
+
 * prefer double backticks otherwise:
 
   * for file names: \`\`os.nim\`\`
   * for fragments of strings **not** enclosed by `"` and `"` and not
     related to code, e.g. text of compiler messages
-  * for command line options: \`\`--docInternal\`\`
   * also when code ends with a standalone ``\`` (otherwise a combination of
     ``\`` and a final \` would get escaped)
 
-.. Note:: `*.rst` files have `:literal:` as their default role.
+.. Note:: ``*.rst`` files have ``:literal:`` as their default role.
-          So for them the rule above is only applicable if the `:nim:` role
+          So for them the rule above is only applicable if the ``:nim:`` role
-          is set up manually as the default::
+          is set up manually as the default [*]_::
 
             .. role:: nim(code)
                :language: nim
@@ -333,6 +338,8 @@ To avoid accidental highlighting follow this rule in `*.nim` files:
           The first 2 lines are for other RST implementations,
           including Github one.
 
+          .. [*] this is fulfilled when ``doc/rstcommon.rst`` is included.
+
 Best practices
 ==============
 
@@ -350,7 +357,7 @@ to avoid name conflicts across packages.
   # if in nim sources
   when defined(allocStats): discard # bad, can cause conflicts
   when defined(nimAllocStats): discard # preferred
-  # if in a pacakge `cligen`:
+  # if in a package `cligen`:
   when defined(debug): discard # bad, can cause conflicts
   when defined(cligenDebug): discard # preferred
 
@@ -372,7 +379,7 @@ Design with method call syntax chaining in mind
   # can be called as: `getLines().foo(false)`
 
 .. _avoid_quit:
-Use exceptions (including assert / doAssert) instead of `quit`
+Use exceptions (including `assert` / `doAssert`) instead of `quit`
 rationale: https://forum.nim-lang.org/t/4089
 
 .. code-block:: nim
@@ -382,7 +389,7 @@ rationale: https://forum.nim-lang.org/t/4089
 
 .. _tests_use_doAssert:
 Use `doAssert` (or `unittest.check`, `unittest.require`), not `assert` in all
-tests so they'll be enabled even with `--assertions:off`.
+tests so they'll be enabled even with `--assertions:off`:option:.
 
 .. code-block:: nim
 
@@ -391,10 +398,10 @@ tests so they'll be enabled even with `--assertions:off`.
     doAssert foo() # preferred
 
 .. _runnableExamples_use_assert:
-An exception to the above rule is `runnableExamples` and `code-block` rst blocks
+An exception to the above rule is `runnableExamples` and ``code-block`` rst blocks
 intended to be used as `runnableExamples`, which for brevity use `assert`
-instead of `doAssert`. Note that `nim doc -d:danger main` won't pass `-d:danger` to the
+instead of `doAssert`. Note that `nim doc -d:danger main`:cmd: won't pass `-d:danger`:option: to the
-`runnableExamples`, but `nim doc --doccmd:-d:danger main` would, and so would the
+`runnableExamples`, but `nim doc --doccmd:-d:danger main`:cmd: would, and so would the
 second example below:
 
 .. code-block:: nim
@@ -437,8 +444,8 @@ https://github.com/nim-lang/Nim/pull/9335 and https://forum.nim-lang.org/t/4089
   doAssert foo() == [1, 2] # preferred, except when not possible to do so.
 
 
-The Git stuff
+The `git`:cmd: stuff
-=============
+====================
 
 General commit rules
 --------------------
@@ -446,12 +453,12 @@ General commit rules
 1. Important, critical bugfixes that have a tiny chance of breaking
    somebody's code should be backported to the latest stable release
    branch (currently 1.4.x) and maybe also all the way back to the 1.0.x branch.
-   The commit message should contain the tag `[backport]` for "backport to all
+   The commit message should contain the tag ``[backport]`` for "backport to all
-   stable releases" and the tag `[backport:$VERSION]` for backporting to the
+   stable releases" and the tag ``[backport:$VERSION]`` for backporting to the
    given $VERSION.
 
 2. If you introduce changes which affect backward compatibility,
-   make breaking changes, or have PR which is tagged as `[feature]`,
+   make breaking changes, or have PR which is tagged as ``[feature]``,
    the changes should be mentioned in `the changelog
    <https://github.com/nim-lang/Nim/blob/devel/changelog.md>`_.
 
@@ -462,29 +469,29 @@ General commit rules
    your editor reformatted automatically the code or whatever different reason,
    this should be excluded from the commit.
 
-   *Tip:* Never commit everything as is using `git commit -a`, but review
+   *Tip:* Never commit everything as is using `git commit -a`:cmd:, but review
-   carefully your changes with `git add -p`.
+   carefully your changes with `git add -p`:cmd:.
 
 4. Changes should not introduce any trailing whitespace.
 
-   Always check your changes for whitespace errors using `git diff --check`
+   Always check your changes for whitespace errors using `git diff --check`:cmd:
-   or add the following `pre-commit` hook:
+   or add the following ``pre-commit`` hook:
 
-   .. code-block:: sh
+   .. code:: cmd
 
       #!/bin/sh
       git diff --check --cached || exit $?
 5. Describe your commit and use your common sense.
-   Example commit message:
+   Example commit message::
 
-   `Fixes #123; refs #124`
+     Fixes #123; refs #124
 
-   indicates that issue `#123` is completely fixed (GitHub may automatically
+   indicates that issue ``#123`` is completely fixed (GitHub may automatically
-   close it when the PR is committed), wheres issue `#124` is referenced
+   close it when the PR is committed), wheres issue ``#124`` is referenced
    (e.g.: partially fixed) and won't close the issue when committed.
 
 6. PR body (not just PR title) should contain references to fixed/referenced github
-   issues, e.g.: `fix #123` or `refs #123`. This is so that you get proper cross
+   issues, e.g.: ``fix #123`` or ``refs #123``. This is so that you get proper cross
    referencing from linked issue to the PR (github won't make those links with just
    PR title, and commit messages aren't always sufficient to ensure that, e.g.
    can't be changed after a PR is merged).
@@ -492,7 +499,7 @@ General commit rules
 7. Commits should be always be rebased against devel (so a fast forward
    merge can happen)
 
-   e.g.: use `git pull --rebase origin devel`. This is to avoid messing up
+   e.g.: use `git pull --rebase origin devel`:cmd:. This is to avoid messing up
    git history.
    Exceptions should be very rare: when rebase gives too many conflicts, simply
    squash all commits using the script shown in
@@ -508,7 +515,7 @@ Continuous Integration (CI)
 
 1. Continuous Integration is by default run on every push in a PR; this clogs
    the CI pipeline and affects other PR's; if you don't need it (e.g. for WIP or
-   documentation only changes), add `[skip ci]` to your commit message title.
+   documentation only changes), add ``[skip ci]`` to your commit message title.
    This convention is supported by our github actions pipelines and our azure pipeline
    as well as our former other pipelines:
    `Appveyor <https://www.appveyor.com/docs/how-to/filtering-commits/#skip-directive-in-commit-message>`_
@@ -532,7 +539,7 @@ Debugging CI failures, flaky tests, etc
    follow these instructions to only restart the jobs that failed:
 
    * Azure: if on your own fork, it's possible from inside azure console
-    (e.g. `dev.azure.com/username/username/_build/results?buildId=1430&view=results`) via `rerun failed jobs` on top.
+     (e.g. ``dev.azure.com/username/username/_build/results?buildId=1430&view=results``) via ``rerun failed jobs`` on top.
      If either on you own fork or in Nim repo, it's possible from inside GitHub UI
      under checks tab, see https://github.com/timotheecour/Nim/issues/211#issuecomment-629751569
    * GitHub actions: under "Checks" tab, click "Re-run jobs" in the right.
@@ -554,15 +561,15 @@ Code reviews
    doesn't help much as it doesn't highlight moves. Instead, you can use something
    like this, see visual results `here <https://github.com/nim-lang/Nim/pull/10431#issuecomment-456968196>`_:
 
-   .. code-block:: sh
+   .. code:: cmd
 
       git fetch origin pull/10431/head && git checkout FETCH_HEAD
       git diff --color-moved-ws=allow-indentation-change --color-moved=blocks HEAD^
 
 3. In addition, you can view GitHub-like diffs locally to identify what was changed
-   within a code block using `diff-highlight` or `diff-so-fancy`, e.g.:
+   within a code block using `diff-highlight`:cmd: or `diff-so-fancy`:cmd:, e.g.:
 
-   .. code-block:: sh
+   ::
 
       # put this in ~/.gitconfig:
       [core]
@@ -651,15 +658,15 @@ to existing modules is acceptable. For two reasons:
 
 Conventions
 -----------
-1. New stdlib modules should go under `Nim/lib/std/`. The rationale is to
+1. New stdlib modules should go under ``Nim/lib/std/``. The rationale is to
    require users to import via `import std/foo` instead of `import foo`,
    which would cause potential conflicts with nimble packages.
    Note that this still applies for new modules in existing logical
-   directories, e.g.: use `lib/std/collections/foo.nim`,
+   directories, e.g.: use ``lib/std/collections/foo.nim``,
-   not `lib/pure/collections/foo.nim`.
+   not ``lib/pure/collections/foo.nim``.
 
 2. New module names should prefer plural form whenever possible, e.g.:
-   `std/sums.nim` instead of `std/sum.nim`. In particular, this reduces
+   ``std/sums.nim`` instead of ``std/sum.nim``. In particular, this reduces
    chances of conflicts between module name and the symbols it defines.
    Furthermore, module names should use `snake_case` and not use capital
    letters, which cause issues when going from an OS without case

doc/docstyle.rst

@@ -6,17 +6,17 @@ General Guidelines
 
 * See also `nep1<https://nim-lang.github.io/Nim/nep1.html>`_ which should probably be merged here.
 * Authors should document anything that is exported; documentation for private
-  procs can be useful too (visible via `nim doc --docInternal foo.nim`).
+  procs can be useful too (visible via `nim doc --docInternal foo.nim`:cmd:).
 * Within documentation, a period (`.`) should follow each sentence (or sentence fragment) in a comment block.
   The documentation may be limited to one sentence fragment, but if multiple sentences are within the documentation,
   each sentence after the first should be complete and in present tense.
 * Documentation is parsed as a custom ReStructuredText (RST) with partial markdown support.
 * In nim sources, prefer single backticks to double backticks since it's simpler
-  and `nim doc` supports it. Likewise with rst files: `nim rst2html` will render those as monospace, and
+  and `nim doc`:cmd: supports it. Likewise with ``rst`` files: `nim rst2html`:cmd: will render those as monospace, and
-  adding `.. default-role:: code` to an rst file will also make those render as monospace when rendered directly
+  adding ``.. default-role:: code`` to an ``rst`` file will also make those render as monospace when rendered directly
   in tools such as github.
-* In nim sources, for links, prefer `[link text](link.html)` to `` `link text<link.html>`_ ``
+* (debatable) In nim sources, for links, prefer ``[link text](link.html)`` to `\`link text<link.html>\`_`:code:
-  since the syntax is simpler and markdown is more common (likewise, `nim rst2html` also supports it in rst files).
+  since the syntax is simpler and markdown is more common (likewise, `nim rst2html`:cmd: also supports it in ``rst`` files).
 
 .. code-block:: nim
 
@@ -29,7 +29,7 @@ Module-level documentation
 --------------------------
 
 Documentation of a module is placed at the top of the module itself. Each line of documentation begins with double hashes (`##`).
-Sometimes `##[ multiline docs containing code ]##` is preferable, see `lib/pure/times.nim`.
+Sometimes `##[ multiline docs containing code ]##` is preferable, see ``lib/pure/times.nim``.
 Code samples are encouraged, and should follow the general RST syntax:
 
 .. code-block:: Nim
@@ -76,11 +76,11 @@ Whenever an example of usage would be helpful to the user, you should include on
     ##  echo execCmdEx("git pull")
     ##  drawOnScreen()
     runnableExamples:
-      # `runnableExamples` is usually preferred to `code-block`, when possible.
+      # `runnableExamples` is usually preferred to ``code-block``, when possible.
       doAssert addThree(3, 125, 6) == -122
     result = x +% y +% z
 
-The command `nim doc` will then correctly syntax highlight the Nim code within the documentation.
+The command `nim doc`:cmd: will then correctly syntax highlight the Nim code within the documentation.
 
 Types
 -----

doc/nimdoc.css

@@ -35,6 +35,8 @@ Modified by Boyd Greenfield and narimiran
   --escapeSequence: #c4891b;
   --number: #252dbe;
   --literal: #a4255b;
+  --program: #6060c0;
+  --option: #508000;
   --raw-data: #a4255b;
 }
 
@@ -63,6 +65,8 @@ Modified by Boyd Greenfield and narimiran
   --escapeSequence: #bd93f9;
   --number: #bd93f9;
   --literal: #f1fa8c;
+  --program: #9090c0;
+  --option: #90b010;
   --raw-data: #8be9fd;
 }
 
@@ -527,7 +531,6 @@ div.option-list-label {
   margin-left: -11.5em;
   margin-right: 0em;
   min-width: 11.5em;
-  font-weight: bolder;
   display: inline-block;
   vertical-align: top;
 }
@@ -546,7 +549,7 @@ blockquote {
   border-left: 5px solid #bbc;
 }
 
-.pre {
+.pre, span.tok {
   font-family: "Source Code Pro", Monaco, Menlo, Consolas, "Courier New", monospace;
   font-weight: 500;
   font-size: 0.85em;
@@ -557,6 +560,12 @@ blockquote {
   border-radius: 4px;
 }
 
+span.tok {
+  border: 1px solid #808080;
+  padding-bottom: 0.1em;
+  margin-right: 0.2em;
+}
+
 pre {
   font-family: "Source Code Pro", Monaco, Menlo, Consolas, "Courier New", monospace;
   color: var(--text);
@@ -844,9 +853,6 @@ span.classifier {
 span.classifier-delimiter {
   font-weight: bold; }
 
-span.option {
-  white-space: nowrap; }
-
 span.problematic {
   color: #b30000; }
 
@@ -926,6 +932,21 @@ span.Preprocessor {
 span.Directive {
   color: #252dbe; }
 
+span.option {
+  font-weight: bold;
+  font-family: "Source Code Pro", Monaco, Menlo, Consolas, "Courier New", monospace;
+  color: var(--option);
+}
+
+span.program {
+  font-weight: bold;
+  color: var(--program);
+  text-decoration: underline;
+  text-decoration-color: var(--hint);
+  text-decoration-thickness: 0.05em;
+  text-underline-offset: 0.15em;
+}
+
 span.Command, span.Rule, span.Hyperlink, span.Label, span.Reference,
 span.Other {
   color: var(--other); }

lib/packages/docutils/highlite.nim

@@ -37,6 +37,18 @@
 ## .. code:: Nim
 ##   for l in ["C", "c++", "jAvA", "Nim", "c#"]: echo getSourceLanguage(l)
 ##
+## There is also a `Cmd` pseudo-language supported, which is a simple generic
+## shell/cmdline tokenizer (UNIX shell/Powershell/Windows Command):
+## no escaping, no programming language constructs besides variable definition
+## at the beginning of line. It supports these operators:
+##
+## .. code:: Cmd
+##    &  &&  |  ||  (  )  ''  ""  ;  # for comments
+##
+## Instead of escaping always use quotes like here
+## `nimgrep --ext:'nim|nims' file.name`:cmd: shows how to input ``|``.
+## Any argument that contains ``.`` or ``/`` or ``\`` will be treated
+## as a file or directory.
 
 import
   strutils
@@ -45,7 +57,7 @@ from algorithm import binarySearch
 type
   SourceLanguage* = enum
     langNone, langNim, langCpp, langCsharp, langC, langJava,
-    langYaml, langPython
+    langYaml, langPython, langCmd
   TokenClass* = enum
     gtEof, gtNone, gtWhitespace, gtDecNumber, gtBinNumber, gtHexNumber,
     gtOctNumber, gtFloatNumber, gtIdentifier, gtKeyword, gtStringLit,
@@ -53,7 +65,7 @@ type
     gtOperator, gtPunctuation, gtComment, gtLongComment, gtRegularExpression,
     gtTagStart, gtTagEnd, gtKey, gtValue, gtRawData, gtAssembler,
     gtPreprocessor, gtDirective, gtCommand, gtRule, gtHyperlink, gtLabel,
-    gtReference, gtOther
+    gtReference, gtProgram, gtOption, gtOther
   GeneralTokenizer* = object of RootObj
     kind*: TokenClass
     start*, length*: int
@@ -64,14 +76,17 @@ type
 
 const
   sourceLanguageToStr*: array[SourceLanguage, string] = ["none",
-    "Nim", "C++", "C#", "C", "Java", "Yaml", "Python"]
+    "Nim", "C++", "C#", "C", "Java", "Yaml", "Python", "Cmd"]
   tokenClassToStr*: array[TokenClass, string] = ["Eof", "None", "Whitespace",
     "DecNumber", "BinNumber", "HexNumber", "OctNumber", "FloatNumber",
     "Identifier", "Keyword", "StringLit", "LongStringLit", "CharLit",
     "EscapeSequence", "Operator", "Punctuation", "Comment", "LongComment",
     "RegularExpression", "TagStart", "TagEnd", "Key", "Value", "RawData",
     "Assembler", "Preprocessor", "Directive", "Command", "Rule", "Hyperlink",
-    "Label", "Reference", "Other"]
+    "Label", "Reference",
+    # start from lower-case if there is a corresponding RST role (see rst.nim)
+    "program", "option",
+    "Other"]
 
   # The following list comes from doc/keywords.txt, make sure it is
   # synchronized with this array by running the module itself as a test case.
@@ -898,6 +913,65 @@ proc pythonNextToken(g: var GeneralTokenizer) =
       "with", "yield"]
   nimNextToken(g, keywords)
 
+proc cmdNextToken(g: var GeneralTokenizer) =
+  var pos = g.pos
+  g.start = g.pos
+  if g.state == low(TokenClass):
+    g.state = gtProgram
+  case g.buf[pos]
+  of ' ', '\t'..'\r':
+    g.kind = gtWhitespace
+    while g.buf[pos] in {' ', '\t'..'\r'}:
+      if g.buf[pos] == '\n':
+        g.state = gtProgram
+      inc(pos)
+  of '\'', '"':
+    g.kind = gtOption
+    let q = g.buf[pos]
+    inc(pos)
+    while g.buf[pos] notin {q, '\0'}:
+      inc(pos)
+    if g.buf[pos] == q: inc(pos)
+  of '#':
+    g.kind = gtComment
+    while g.buf[pos] notin {'\n', '\0'}:
+      inc(pos)
+  of '&', '|':
+    g.kind = gtOperator
+    inc(pos)
+    if g.buf[pos] == g.buf[pos-1]: inc(pos)
+    g.state = gtProgram
+  of '(':
+    g.kind = gtOperator
+    g.state = gtProgram
+    inc(pos)
+  of ')':
+    g.kind = gtOperator
+    inc(pos)
+  of ';':
+    g.state = gtProgram
+    g.kind = gtOperator
+    inc(pos)
+  of '\0': g.kind = gtEof
+  else:
+    if g.state == gtProgram:
+      g.kind = gtProgram
+      g.state = gtOption
+    else:
+      g.kind = gtOption
+    while g.buf[pos] notin {' ', '\t'..'\r', '&', '|', '(', ')', '\'', '"', '\0'}:
+      if g.buf[pos] == ';' and g.buf[pos+1] == ' ':
+        # (check space because ';' can be used inside arguments in Win bat)
+        break
+      if g.kind == gtOption and g.buf[pos] in {'/', '\\', '.'}:
+        g.kind = gtIdentifier  # for file/dir name
+      elif g.kind == gtProgram and g.buf[pos] == '=':
+        g.kind = gtIdentifier  # for env variable setting at beginning of line
+        g.state = gtProgram
+      inc(pos)
+  g.length = pos - g.pos
+  g.pos = pos
+
 proc getNextToken*(g: var GeneralTokenizer, lang: SourceLanguage) =
   g.lang = lang
   case lang
@@ -909,6 +983,7 @@ proc getNextToken*(g: var GeneralTokenizer, lang: SourceLanguage) =
   of langJava: javaNextToken(g)
   of langYaml: yamlNextToken(g)
   of langPython: pythonNextToken(g)
+  of langCmd: cmdNextToken(g)
 
 when isMainModule:
   var keywords: seq[string]

lib/packages/docutils/rst.nim

@@ -23,10 +23,10 @@
 ##
 ## Nim can output the result to HTML [#html]_ or Latex [#latex]_.
 ##
-## .. [#html] commands ``nim doc`` for ``*.nim`` files and
+## .. [#html] commands `nim doc`:cmd: for ``*.nim`` files and
-##    ``nim rst2html`` for ``*.rst`` files
+##    `nim rst2html`:cmd: for ``*.rst`` files
 ##
-## .. [#latex] command ``nim rst2tex`` for ``*.rst``.
+## .. [#latex] command `nim rst2tex`:cmd: for ``*.rst``.
 ##
 ## If you are new to RST please consider reading the following:
 ##
@@ -78,7 +78,8 @@
 ##
 ## * directives: ``code-block`` [cmp:Sphinx]_, ``title``,
 ##   ``index`` [cmp:Sphinx]_
-## * predefined roles ``:nim:`` (default), ``:c:`` (C programming language),
+## * predefined roles
+##   - ``:nim:`` (default), ``:c:`` (C programming language),
 ##     ``:python:``, ``:yaml:``, ``:java:``, ``:cpp:`` (C++), ``:csharp`` (C#).
 ##     That is every language that `highlite <highlite.html>`_ supports.
 ##     They turn on appropriate syntax highlighting in inline code.
@@ -86,6 +87,12 @@
 ##     .. Note:: default role for Nim files is ``:nim:``,
 ##               for ``*.rst`` it's currently ``:literal:``.
 ##
+##   - generic command line highlighting roles:
+##     - ``:cmd:`` for commands and common shells syntax
+##     - ``:program:`` for executable names [cmp:Sphinx]_
+##       (one can just use ``:cmd:`` on single word)
+##     - ``:option:`` for command line options [cmp:Sphinx]_
+##   - ``:tok:``, a role for highlighting of programming language tokens
 ## * ***triple emphasis*** (bold and italic) using \*\*\*
 ## * ``:idx:`` role for \`interpreted text\` to include the link to this
 ##   text into an index (example: `Nim index`_).
@@ -95,11 +102,11 @@
 ##     //compile   compile the project
 ##     //doc       generate documentation
 ##
-##   Here the dummy `//` will disappear, while options ``compile``
+##   Here the dummy `//` will disappear, while options `compile`:option:
-##   and ``doc`` will be left in the final document.
+##   and `doc`:option: will be left in the final document.
 ##
 ## .. [cmp:Sphinx] similar but different from the directives of
-##    Python `Sphinx directives`_ extensions
+##    Python `Sphinx directives`_ and `Sphinx roles`_ extensions
 ##
 ## .. _`extra features`:
 ##
@@ -144,7 +151,7 @@
 ## -----
 ##
 ## See `Nim DocGen Tools Guide <docgen.html>`_ for the details about
-## ``nim doc``, ``nim rst2html`` and ``nim rst2tex`` commands.
+## `nim doc`:cmd:, `nim rst2html`:cmd: and `nim rst2tex`:cmd: commands.
 ##
 ## See `packages/docutils/rstgen module <rstgen.html>`_ to know how to
 ## generate HTML or Latex strings to embed them into your documents.
@@ -156,6 +163,7 @@
 ## .. _RST roles list: https://docutils.sourceforge.io/docs/ref/rst/roles.html
 ## .. _Nim index: https://nim-lang.org/docs/theindex.html
 ## .. _Sphinx directives: https://www.sphinx-doc.org/en/master/usage/restructuredtext/directives.html
+## .. _Sphinx roles: https://www.sphinx-doc.org/en/master/usage/restructuredtext/roles.html
 
 import
   os, strutils, rstast, std/enumutils, algorithm, lists, sequtils,
@@ -530,7 +538,7 @@ proc defaultRole(options: RstParseOptions): string =
 
 # mirror highlite.nim sourceLanguageToStr with substitutions c++ cpp, c# csharp
 const supportedLanguages = ["nim", "yaml", "python", "java", "c",
-                            "cpp", "csharp"]
+                            "cpp", "csharp", "cmd"]
 
 proc whichRoleAux(sym: string): RstNodeKind =
   let r = sym.toLowerAscii
@@ -543,6 +551,7 @@ proc whichRoleAux(sym: string): RstNodeKind =
   of "sup", "superscript": result = rnSup
   # literal and code are the same in our implementation
   of "code": result = rnInlineLiteral
+  of "program", "option", "tok": result = rnCodeFragment
   # c++ currently can be spelled only as cpp, c# only as csharp
   elif r in supportedLanguages:
     result = rnInlineCode
@@ -1113,10 +1122,10 @@ proc toInlineCode(n: PRstNode, language: string): PRstNode =
   lb.add newLeaf(s)
   result.add lb
 
-proc toUnknownRole(n: PRstNode, roleName: string): PRstNode =
+proc toOtherRole(n: PRstNode, kind: RstNodeKind, roleName: string): PRstNode =
   let newN = newRstNode(rnInner, n.sons)
   let newSons = @[newN, newLeaf(roleName)]
-  result = newRstNode(rnUnknownRole, newSons)
+  result = newRstNode(kind, newSons)
 
 proc parsePostfix(p: var RstParser, n: PRstNode): PRstNode =
   var newKind = n.kind
@@ -1144,8 +1153,8 @@ proc parsePostfix(p: var RstParser, n: PRstNode): PRstNode =
     # a role:
     let (roleName, lastIdx) = getRefname(p, p.idx+1)
     newKind = whichRole(p, roleName)
-    if newKind == rnUnknownRole:
+    if newKind in {rnUnknownRole, rnCodeFragment}:
-      result = n.toUnknownRole(roleName)
+      result = n.toOtherRole(newKind, roleName)
     elif newKind == rnInlineCode:
       result = n.toInlineCode(language=roleName)
     else:
@@ -1417,8 +1426,8 @@ proc parseInline(p: var RstParser, father: PRstNode) =
       if k == rnInlineCode:
         n = n.toInlineCode(language=roleName)
       parseUntil(p, n, "`", false) # bug #17260
-      if k == rnUnknownRole:
+      if k in {rnUnknownRole, rnCodeFragment}:
-        n = n.toUnknownRole(roleName)
+        n = n.toOtherRole(k, roleName)
       father.add(n)
     elif isInlineMarkupStart(p, "`"):
       var n = newRstNode(rnInterpretedText)

lib/packages/docutils/rstast.nim

@@ -56,7 +56,9 @@ type
                               #     * `file#id <file#id>'_
     rnSubstitutionDef,        # a definition of a substitution
     # Inline markup:
-    rnInlineCode,
+    rnInlineCode,             # interpreted text with code in a known language
+    rnCodeFragment,           # inline code for highlighting with the specified
+                              # class (which cannot be inferred from context)
     rnUnknownRole,            # interpreted text with an unknown role
     rnSub, rnSup, rnIdx,
     rnEmphasis,               # "*"

lib/packages/docutils/rstgen.nim

@@ -1198,7 +1198,8 @@ proc renderRstToOut(d: PDoc, n: PRstNode, result: var string) =
         "$1", result)
   of rnOptionGroup:
     renderAux(d, n,
-        "<div class=\"option-list-label\">$1</div>",
+        "<div class=\"option-list-label\"><tt><span class=\"option\">" &
+        "$1</span></tt></div>",
         "\\item[$1]", result)
   of rnDescription:
     renderAux(d, n, "<div class=\"option-list-description\">$1</div>",
@@ -1319,13 +1320,22 @@ proc renderRstToOut(d: PDoc, n: PRstNode, result: var string) =
     renderAux(d, n, "|$1|", "|$1|", result)
   of rnDirective:
     renderAux(d, n, "", "", result)
-  of rnUnknownRole:
+  of rnUnknownRole, rnCodeFragment:
     var tmp0 = ""
     var tmp1 = ""
     renderRstToOut(d, n.sons[0], tmp0)
     renderRstToOut(d, n.sons[1], tmp1)
+    var class = tmp1
+    # don't allow missing role break latex compilation:
+    if d.target == outLatex and n.kind == rnUnknownRole: class = "Other"
+    if n.kind == rnCodeFragment:
+      dispA(d.target, result,
+            "<tt class=\"docutils literal\"><span class=\"pre $2\">" &
+              "$1</span></tt>",
+            "\\texttt{\\span$2{$1}}", [tmp0, class])
+    else:  # rnUnknownRole, not necessarily code/monospace font
       dispA(d.target, result, "<span class=\"$2\">$1</span>", "\\span$2{$1}",
-          [tmp0, tmp1])
+            [tmp0, class])
   of rnSub: renderAux(d, n, "<sub>$1</sub>", "\\rstsub{$1}", result)
   of rnSup: renderAux(d, n, "<sup>$1</sup>", "\\rstsup{$1}", result)
   of rnEmphasis: renderAux(d, n, "<em>$1</em>", "\\emph{$1}", result)

nimdoc/testproject/expected/nimdoc.out.css

@@ -35,6 +35,8 @@ Modified by Boyd Greenfield and narimiran
   --escapeSequence: #c4891b;
   --number: #252dbe;
   --literal: #a4255b;
+  --program: #6060c0;
+  --option: #508000;
   --raw-data: #a4255b;
 }
 
@@ -63,6 +65,8 @@ Modified by Boyd Greenfield and narimiran
   --escapeSequence: #bd93f9;
   --number: #bd93f9;
   --literal: #f1fa8c;
+  --program: #9090c0;
+  --option: #90b010;
   --raw-data: #8be9fd;
 }
 
@@ -527,7 +531,6 @@ div.option-list-label {
   margin-left: -11.5em;
   margin-right: 0em;
   min-width: 11.5em;
-  font-weight: bolder;
   display: inline-block;
   vertical-align: top;
 }
@@ -546,7 +549,7 @@ blockquote {
   border-left: 5px solid #bbc;
 }
 
-.pre {
+.pre, span.tok {
   font-family: "Source Code Pro", Monaco, Menlo, Consolas, "Courier New", monospace;
   font-weight: 500;
   font-size: 0.85em;
@@ -557,6 +560,12 @@ blockquote {
   border-radius: 4px;
 }
 
+span.tok {
+  border: 1px solid #808080;
+  padding-bottom: 0.1em;
+  margin-right: 0.2em;
+}
+
 pre {
   font-family: "Source Code Pro", Monaco, Menlo, Consolas, "Courier New", monospace;
   color: var(--text);
@@ -844,9 +853,6 @@ span.classifier {
 span.classifier-delimiter {
   font-weight: bold; }
 
-span.option {
-  white-space: nowrap; }
-
 span.problematic {
   color: #b30000; }
 
@@ -926,6 +932,21 @@ span.Preprocessor {
 span.Directive {
   color: #252dbe; }
 
+span.option {
+  font-weight: bold;
+  font-family: "Source Code Pro", Monaco, Menlo, Consolas, "Courier New", monospace;
+  color: var(--option);
+}
+
+span.program {
+  font-weight: bold;
+  color: var(--program);
+  text-decoration: underline;
+  text-decoration-color: var(--hint);
+  text-decoration-thickness: 0.05em;
+  text-underline-offset: 0.15em;
+}
+
 span.Command, span.Rule, span.Hyperlink, span.Label, span.Reference,
 span.Other {
   color: var(--other); }

tests/stdlib/trstgen.nim

@@ -40,6 +40,10 @@ proc toHtml(input: string,
 proc id(str: string): string = """<span class="Identifier">"""  & str & "</span>"
 proc op(str: string): string = """<span class="Operator">"""    & str & "</span>"
 proc pu(str: string): string = """<span class="Punctuation">""" & str & "</span>"
+proc optionListLabel(opt: string): string =
+  """<div class="option-list-label"><tt><span class="option">""" &
+  opt &
+  "</span></tt></div>"
 
 suite "YAML syntax highlighting":
   test "Basics":
@@ -1382,10 +1386,10 @@ Test1
     check(output.count("<ul") == 1)
     check(output.count("<li>") == 2)
     check(output.count("<div class=\"option-list\"") == 1)
-    check("""<div class="option-list-label">-m</div>""" &
+    check(optionListLabel("-m") &
           """<div class="option-list-description">desc</div></div>""" in
           output)
-    check("""<div class="option-list-label">-n</div>""" &
+    check(optionListLabel("-n") &
           """<div class="option-list-description">very long desc</div></div>""" in
           output)
 
@@ -1400,13 +1404,13 @@ Test1
     let output = input.toHtml
     check(output.count("<ul") == 1)
     check output.count("<div class=\"option-list\"") == 2
-    check("""<div class="option-list-label">-m</div>""" &
+    check(optionListLabel("-m") &
           """<div class="option-list-description">desc</div></div>""" in
           output)
-    check("""<div class="option-list-label">-n</div>""" &
+    check(optionListLabel("-n") &
           """<div class="option-list-description">very long desc</div></div>""" in
           output)
-    check("""<div class="option-list-label">-d</div>""" &
+    check(optionListLabel("-d") &
           """<div class="option-list-description">option</div></div>""" in
           output)
     check "<p>option</p>" notin output
@@ -1421,13 +1425,13 @@ Test1
     let output = input.toHtml
     check(output.count("<ul") == 1)
     check output.count("<div class=\"option-list\"") == 2
-    check("""<div class="option-list-label">compile</div>""" &
+    check(optionListLabel("compile") &
           """<div class="option-list-description">compile1</div></div>""" in
           output)
-    check("""<div class="option-list-label">doc</div>""" &
+    check(optionListLabel("doc") &
           """<div class="option-list-description">doc1 cont</div></div>""" in
           output)
-    check("""<div class="option-list-label">-d</div>""" &
+    check(optionListLabel("-d") &
           """<div class="option-list-description">option</div></div>""" in
           output)
     check "<p>option</p>" notin output