stdlib organization & documentation improvements (#20971)

* stdlib organization & documentation improvements

* fix CI

* Update doc/lib.md

Co-authored-by: Juan Carlos <juancarlospaco@gmail.com>

* fix ci, remove jshttpcore, export in jsfetch instead

* fix alphabetical order violations

* add cmdline, db_odbc

Co-authored-by: Juan Carlos <juancarlospaco@gmail.com>

compiler/options.nim

@@ -865,7 +865,8 @@ const stdlibDirs* = [
   "pure/concurrency",
   "pure/unidecode", "impure",
   "wrappers", "wrappers/linenoise",
-  "windows", "posix", "js"]
+  "windows", "posix", "js",
+  "deprecated/pure"]
 
 const
   pkgPrefix = "pkg/"

doc/lib.md

@@ -46,15 +46,19 @@ Core
 * [bitops](bitops.html)
   Provides a series of low-level methods for bit manipulation.
 
+* [compilesettings](compilesettings.html)
+  This module allows querying the compiler about diverse configuration settings.
+
 * [cpuinfo](cpuinfo.html)
   This module implements procs to determine the number of CPUs / cores.
 
+* [effecttraits](effecttraits.html)
+  This module provides access to the inferred .raises effects
+  for Nim's macro system.
+
 * [endians](endians.html)
   This module contains helpers that deal with different byte orders.
 
-* [lenientops](lenientops.html)
-  Provides binary operators for mixed integer/float expressions for convenience.
-
 * [locks](locks.html)
   Locks and condition variables for Nim.
 
@@ -122,9 +126,19 @@ Collections
 * [packedsets](packedsets.html)
   Efficient implementation of a set of ordinals as a sparse bit set.
 
+* [ropes](ropes.html)
+  This module contains support for a *rope* data type.
+  Ropes can represent very long strings efficiently;
+  in particular, concatenation is done in O(1) instead of O(n).
+
 * [sets](sets.html)
   Nim hash set support.
 
+* [strtabs](strtabs.html)
+  The `strtabs` module implements an efficient hash table that is a mapping
+  from strings to strings. Supports a case-sensitive, case-insensitive and
+  style-insensitive modes.
+
 * [tables](tables.html)
   Nim hash table support. Contains tables, ordered tables, and count tables.
 
@@ -143,20 +157,15 @@ String handling
   Converts between different character encodings. On UNIX, this uses
   the `iconv` library, on Windows the Windows API.
 
-* [parseutils](parseutils.html)
+* [formatfloat](formatfloat.html)
-  This module contains helpers for parsing tokens, numbers, identifiers, etc.
+  This module implements formatting floats as strings.
 
-* [pegs](pegs.html)
+* [objectdollar](objectdollar.html)
-  This module contains procedures and operators for handling PEGs.
+  This module implements a generic `$` operator to convert objects to strings.
 
 * [punycode](punycode.html)
   Implements a representation of Unicode with the limited ASCII character subset.
 
-* [ropes](ropes.html)
-  This module contains support for a *rope* data type.
-  Ropes can represent very long strings efficiently;
-  in particular, concatenation is done in O(1) instead of O(n).
-
 * [strbasics](strbasics.html)
   This module provides some high performance string operations.
 
@@ -171,11 +180,6 @@ String handling
 * [strscans](strscans.html)
   This module contains a `scanf` macro for convenient parsing of mini languages.
 
-* [strtabs](strtabs.html)
-  The `strtabs` module implements an efficient hash table that is a mapping
-  from strings to strings. Supports a case-sensitive, case-insensitive and
-  style-insensitive modes.
-
 * [strutils](strutils.html)
   This module contains common string handling operations like changing
   case of a string, splitting a string into substrings, searching for
@@ -188,6 +192,9 @@ String handling
   It provides a single proc that does Unicode to ASCII transliterations.
   Based on Python's Unidecode module.
 
+* [widestrs](widestrs.html)
+  Nim support for C/C++'s wide strings.
+
 * [wordwrap](wordwrap.html)
   This module contains an algorithm to wordwrap a Unicode string.
 
@@ -205,6 +212,16 @@ Time handling
 Generic Operating System Services
 ---------------------------------
 
+* [appdirs](appdirs.html)
+  This module implements helpers for determining special directories used by apps.
+
+* [cmdline](cmdline.html)
+  This module contains system facilities for reading command
+  line parameters.
+
+* [dirs](dirs.html)
+  This module implements directory handling.
+
 * [distros](distros.html)
   This module implements the basics for OS distribution ("distro") detection
   and the OS's native package manager.
@@ -216,9 +233,14 @@ Generic Operating System Services
 * [dynlib](dynlib.html)
   This module implements the ability to access symbols from shared libraries.
 
-* [marshal](marshal.html)
+* [envvars](envvars.html)
-  Contains procs for serialization and deserialization of arbitrary Nim
+  This module implements environment variable handling.
-  data structures.
+
+* [exitprocs](exitprocs.html)
+  This module allows adding hooks to program exit.
+
+* [files](files.html)
+  This module implements file handling.
 
 * [memfiles](memfiles.html)
   This module provides support for memory-mapped files (Posix's `mmap`)
@@ -229,15 +251,31 @@ Generic Operating System Services
   reading command line arguments, working with directories, running shell
   commands, etc.
 
+* [oserrors](oserrors.html)
+  This module implements OS error reporting.
+
 * [osproc](osproc.html)
   Module for process communication beyond `os.execShellCmd`.
 
+* [paths](paths.html)
+  This module implements path handling.
+
+* [reservedmem](reservedmem.html)
+  This module provides utilities for reserving portions of the
+  address space of a program without consuming physical memory.
+
 * [streams](streams.html)
   This module provides a stream interface and two implementations thereof:
   the `FileStream` and the `StringStream` which implement the stream
   interface for Nim file objects (`File`) and strings. Other modules
   may provide other implementations for this standard stream interface.
 
+* [symlinks](symlinks.html)
+  This module implements symlink handling.
+
+* [syncio](syncio.html)
+  This module implements various synchronized I/O operations.
+
 * [terminal](terminal.html)
   This module contains a few procedures to control the *terminal*
   (also called *console*). The implementation simply uses ANSI escape
@@ -258,6 +296,9 @@ Math libraries
   Floating-point environment. Handling of floating-point rounding and
   exceptions (overflow, zero-divide, etc.).
 
+* [lenientops](lenientops.html)
+  Provides binary operators for mixed integer/float expressions for convenience.
+
 * [math](math.html)
   Mathematical operations like cosine, square root.
 
@@ -270,9 +311,6 @@ Math libraries
 * [stats](stats.html)
   Statistical analysis.
 
-* [sums](sums.html)
-  Accurate summation functions.
-
 * [sysrand](sysrand.html)
   Cryptographically secure pseudorandom number generator.
 
@@ -280,6 +318,9 @@ Math libraries
 Internet Protocols and Support
 ------------------------------
 
+* [async](async.html)
+  Exports `asyncmacro` and `asyncfutures` for native backends, and `asyncjs` on the JS backend. 
+
 * [asyncdispatch](asyncdispatch.html)
   This module implements an asynchronous dispatcher for IO operations.
 
@@ -295,6 +336,9 @@ Internet Protocols and Support
   This module implements an asynchronous HTTP server using the `asyncnet`
   module.
 
+* [asyncmacro](asyncmacro.html)
+  Implements the `async` and `multisync` macros for `asyncdispatch`.
+
 * [asyncnet](asyncnet.html)
   This module implements asynchronous sockets based on the `asyncdispatch`
   module.
@@ -326,6 +370,9 @@ Internet Protocols and Support
   This module implements a selector API with backends specific to each OS.
   Currently, epoll on Linux and select on other operating systems.
 
+* [socketstreams](socketstreams.html)
+  This module provides an implementation of the streams interface for sockets.
+
 * [uri](uri.html)
   This module provides functions for working with URIs.
 
@@ -333,12 +380,20 @@ Internet Protocols and Support
 Threading
 ---------
 
-* [threads](threads.html)
+* [isolation](isolation.html)
-  Basic Nim thread support.
+  This module implements the `Isolated[T]` type for
+  safe construction of isolated subgraphs that can be
+  passed efficiently to different channels and threads.
+
+* [tasks](tasks.html)
+  This module provides basic primitives for creating parallel programs.
 
 * [threadpool](threadpool.html)
   Implements Nim's [spawn](manual_experimental.html#parallel-amp-spawn).
 
+* [typedthreads](typedthreads.html)
+  Basic Nim thread support.
+
 
 Parsers
 -------
@@ -349,9 +404,6 @@ Parsers
 * [json](json.html)
   High-performance JSON parser.
 
-* [jsonutils](jsonutils.html)
-  This module implements a hookable (de)serialization for arbitrary types.
-
 * [lexbase](lexbase.html)
   This is a low-level module that implements an extremely efficient buffering
   scheme for lexers and parsers. This is used by the diverse parsing modules.
@@ -375,12 +427,18 @@ Parsers
 * [parsesql](parsesql.html)
   The `parsesql` module implements a simple high-performance SQL parser.
 
+* [parseutils](parseutils.html)
+  This module contains helpers for parsing tokens, numbers, identifiers, etc.
+
 * [parsexml](parsexml.html)
   The `parsexml` module implements a simple high performance XML/HTML parser.
   The only encoding that is supported is UTF-8. The parser has been designed
   to be somewhat error-correcting, so that even some "wild HTML" found on the
   web can be parsed with it.
 
+* [pegs](pegs.html)
+  This module contains procedures and operators for handling PEGs.
+
 
 Docutils
 --------
@@ -415,6 +473,9 @@ XML Processing
 Generators
 ----------
 
+* [genasts](genasts.html)
+  This module implements AST generation using captured variables for macros.
+
 * [htmlgen](htmlgen.html)
   This module implements a simple XML and HTML code
   generator. Each commonly used HTML tag has a corresponding macro
@@ -443,9 +504,24 @@ Hashing
   This module implements the SHA-1 checksum algorithm.
 
 
+Serialization
+-------------
+
+* [jsonutils](jsonutils.html)
+  This module implements a hookable (de)serialization for arbitrary types
+  using JSON.
+
+* [marshal](marshal.html)
+  Contains procs for serialization and deserialization of arbitrary Nim
+  data structures.
+
+
 Miscellaneous
 -------------
 
+* [assertions](assertions.html)
+  This module implements assertion handling.
+
 * [browsers](browsers.html)
   This module implements procs for opening URLs with the user's default
   browser.
@@ -456,9 +532,15 @@ Miscellaneous
 * [coro](coro.html)
   This module implements experimental coroutines in Nim.
 
+* [decls](decls.html)
+  This module implements syntax sugar for some declarations.
+
 * [enumerate](enumerate.html)
   This module implements `enumerate` syntactic sugar based on Nim's macro system.
 
+* [importutils](importutils.html)
+  Utilities related to import and symbol resolution.
+
 * [logging](logging.html)
   This module implements a simple logger.
 
@@ -477,9 +559,12 @@ Miscellaneous
 * [with](with.html)
   This module implements the `with` macro for easy function chaining.
 
+* [wrapnils](wrapnils.html)
+  This module allows evaluating expressions safely against nil dereferences.
 
-Modules for the JS backend
+
---------------------------
+Modules for the JavaScript backend
+----------------------------------
 
 * [asyncjs](asyncjs.html)
   Types and macros for writing asynchronous procedures in JavaScript.
@@ -497,9 +582,15 @@ Modules for the JS backend
   The wrapper of core JavaScript functions. For most purposes, you should be using
   the `math`, `json`, and `times` stdlib modules instead of this module.
 
+* [jsfetch](jshttp.html)
+  Wrapper for `fetch`.
+
 * [jsffi](jsffi.html)
   Types and macros for easier interaction with JavaScript.
 
+* [jsre](jsre.html)
+  Regular Expressions for the JavaScript target.
+
 
 Impure libraries
 ================
@@ -515,14 +606,18 @@ Regular expressions
 Database support
 ----------------
 
-* [db_postgres](db_postgres.html)
-  A higher level PostgreSQL database wrapper. The same interface is implemented
-  for other databases too.
-
 * [db_mysql](db_mysql.html)
   A higher level MySQL database wrapper. The same interface is implemented
   for other databases too.
 
+* [db_odbc](db_odbc.html)
+  A higher level ODBC database wrapper. The same interface is implemented
+  for other databases too.
+
+* [db_postgres](db_postgres.html)
+  A higher level PostgreSQL database wrapper. The same interface is implemented
+  for other databases too.
+
 * [db_sqlite](db_sqlite.html)
   A higher level SQLite database wrapper. The same interface is implemented
   for other databases too.
@@ -570,14 +665,14 @@ Regular expressions
 Database support
 ----------------
 
-* [postgres](postgres.html)
-  Contains a wrapper for the PostgreSQL API.
 * [mysql](mysql.html)
   Contains a wrapper for the mySQL API.
-* [sqlite3](sqlite3.html)
-  Contains a wrapper for the SQLite 3 API.
 * [odbcsql](odbcsql.html)
   interface to the ODBC driver.
+* [postgres](postgres.html)
+  Contains a wrapper for the PostgreSQL API.
+* [sqlite3](sqlite3.html)
+  Contains a wrapper for the SQLite 3 API.
 
 
 Network Programming and Internet Protocols

doc/nims.md

@@ -76,7 +76,6 @@ At least the following standard library modules are available:
 * [htmlgen](htmlgen.html)
 * [httpcore](httpcore.html)
 * [lenientops](lenientops.html)
-* [mersenne](mersenne.html)
 * [options](options.html)
 * [parseutils](parseutils.html)
 * [punycode](punycode.html)
@@ -89,7 +88,6 @@ At least the following standard library modules are available:
 * [uri](uri.html)
 * [std/editdistance](editdistance.html)
 * [std/wordwrap](wordwrap.html)
-* [std/sums](sums.html)
 * [parsecsv](parsecsv.html)
 * [parsecfg](parsecfg.html)
 * [parsesql](parsesql.html)

lib/pure/async.nim

@@ -1,3 +1,6 @@
+## Exports [asyncmacro](asyncmacro.html) and [asyncfutures](asyncfutures.html) for native backends,
+## and [asyncjs](asyncjs.html) on the JS backend. 
+
 when defined(js):
   import asyncjs
   export asyncjs

lib/pure/asyncdispatch.nim

@@ -1983,7 +1983,8 @@ proc send*(socket: AsyncFD, data: string,
   return retFuture
 
 # -- Await Macro
-include asyncmacro
+import asyncmacro
+export asyncmacro
 
 proc readAll*(future: FutureStream[string]): owned(Future[string]) {.async.} =
   ## Returns a future that will complete when all the string data from the

lib/pure/asyncmacro.nim

@@ -7,7 +7,7 @@
 #    distribution, for details about the copyright.
 #
 
-## `asyncdispatch` module depends on the `asyncmacro` module to work properly.
+## Implements the `async` and `multisync` macros for `asyncdispatch`.
 
 import macros, strutils, asyncfutures
 

lib/pure/math.nim

@@ -47,7 +47,6 @@ runnableExamples:
 ## * `fenv module <fenv.html>`_ for handling of floating-point rounding
 ##   and exceptions (overflow, zero-divide, etc.)
 ## * `random module <random.html>`_ for a fast and tiny random number generator
-## * `mersenne module <mersenne.html>`_ for the Mersenne Twister random number generator
 ## * `stats module <stats.html>`_ for statistical analysis
 ## * `strformat module <strformat.html>`_ for formatting floats for printing
 ## * `system module <system.html>`_ for some very basic and trivial math operators

lib/pure/random.nim

@@ -66,7 +66,6 @@ runnableExamples:
 ## See also
 ## ========
 ## * `std/sysrand module <sysrand.html>`_ for a cryptographically secure pseudorandom number generator
-## * `mersenne module <mersenne.html>`_ for the Mersenne Twister random number generator
 ## * `math module <math.html>`_ for basic math routines
 ## * `stats module <stats.html>`_ for statistical analysis
 ## * `list of cryptographic and hashing modules <lib.html#pure-libraries-hashing>`_

lib/pure/reservedmem.nim

@@ -9,7 +9,7 @@
 
 ## :Authors: Zahary Karadjov
 ##
-## This module provides utilities for reserving a portions of the
+## This module provides utilities for reserving portions of the
 ## address space of a program without consuming physical memory.
 ## It can be used to implement a dynamically resizable buffer that
 ## is guaranteed to remain in the same memory location. The buffer

lib/std/appdirs.nim

@@ -1,3 +1,5 @@
+## This module implements helpers for determining special directories used by apps.
+
 from std/private/osappdirs import nil
 import std/paths
 import std/envvars

lib/std/decls.nim

@@ -1,5 +1,4 @@
-# see `semLowerLetVarCustomPragma` for compiler support that enables these
+## This module implements syntax sugar for some declarations.
-# lowerings
 
 import macros
 

lib/std/dirs.nim

@@ -1,3 +1,5 @@
+## This module implements directory handling.
+
 from paths import Path, ReadDirEffect, WriteDirEffect
 
 from std/private/osdirs import dirExists, createDir, existsOrCreateDir, removeDir,

lib/std/envvars.nim

@@ -8,7 +8,7 @@
 #
 
 
-## The `std/envvars` module implements environment variables handling.
+## The `std/envvars` module implements environment variable handling.
 import std/oserrors
 
 type

lib/std/exitprocs.nim

@@ -7,6 +7,8 @@
 #    distribution, for details about the copyright.
 #
 
+## This module allows adding hooks to program exit.
+
 import locks
 
 type

lib/std/files.nim

@@ -1,3 +1,5 @@
+## This module implements file handling.
+
 from paths import Path, ReadDirEffect, WriteDirEffect
 
 from std/private/osfiles import fileExists, removeFile,

lib/std/formatfloat.nim

@@ -7,6 +7,8 @@
 #    distribution, for details about the copyright.
 #
 
+## This module implements formatting floats as strings.
+
 when defined(nimPreviewSlimSystem):
   import std/assertions
 else:

lib/std/genasts.nim

@@ -1,3 +1,5 @@
+## This module implements AST generation using captured variables for macros.
+
 import macros
 
 type GenAstOpt* = enum

lib/std/jsfetch.nim

@@ -2,7 +2,8 @@
 when not defined(js):
   {.fatal: "Module jsfetch is designed to be used with the JavaScript backend.".}
 
-import std/[asyncjs, jsheaders, jsformdata]
+import std/[asyncjs, jsformdata, jsheaders]
+export jsformdata, jsheaders
 from std/httpcore import HttpMethod
 from std/jsffi import JsObject
 
@@ -116,7 +117,7 @@ func `$`*(self: Request | Response | FetchOptions): string = $toCstring(self)
 
 
 runnableExamples("-r:off"):
-  import std/[asyncjs, jsconsole, jsheaders, jsformdata]
+  import std/[asyncjs, jsconsole, jsformdata, jsheaders]
   from std/httpcore import HttpMethod
   from std/jsffi import JsObject
   from std/sugar import `=>`

lib/std/jsonutils.nim

@@ -1,5 +1,5 @@
 ##[
-This module implements a hookable (de)serialization for arbitrary types.
+This module implements a hookable (de)serialization for arbitrary types using JSON.
 Design goal: avoid importing modules where a custom serialization is needed;
 see strtabs.fromJsonHook,toJsonHook for an example.
 ]##

lib/std/objectdollar.nim

@@ -1,3 +1,5 @@
+## This module implements a generic `$` operator to convert objects to strings.
+
 import std/private/miscdollars
 
 proc `$`*[T: object](x: T): string =

lib/std/paths.nim

@@ -1,3 +1,5 @@
+## This module implements path handling.
+
 import std/private/osseps
 export osseps
 

lib/std/symlinks.nim

@@ -1,3 +1,5 @@
+## This module implements symlink (symbolic link) handling.
+
 from paths import Path, ReadDirEffect
 
 from std/private/ossymlinks import symlinkExists, createSymlink, expandSymlink

lib/std/widestrs.nim

@@ -7,7 +7,7 @@
 #    distribution, for details about the copyright.
 #
 
-# Nim support for C/C++'s `wide strings`:idx:.
+## Nim support for C/C++'s `wide strings`:idx:.
 
 #when not declared(ThisIsSystem):
 #  {.error: "You must not import this module explicitly".}