jyapayne/Nim
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge branch 'devel' into async-improvements

Browse source
This commit is contained in:
Andreas Rumpf 2017-12-11 15:12:45 +01:00 • committed by GitHub
commit 5e43e0d789
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
95 changed files with 9598 additions and 948 deletions
Download patch file Download diff file Expand all files Collapse all files

5
lib/core/macros.nim
View file

@ -1226,3 +1226,8 @@ when not defined(booting):
macro payload: untyped {.gensym.} =
result = parseStmt(e)
payload()
macro unpackVarargs*(callee: untyped; args: varargs[untyped]): untyped =
result = newCall(callee)
for i in 0 ..< args.len:
result.add args[i]

20
lib/impure/re.nim
View file

@ -49,9 +49,6 @@ type
RegexError* = object of ValueError
## is raised if the pattern is no valid regular expression.
{.deprecated: [TRegexFlag: RegexFlag, TRegexDesc: RegexDesc, TRegex: Regex,
EInvalidRegEx: RegexError].}
proc raiseInvalidRegex(msg: string) {.noinline, noreturn.} =
var e: ref RegexError
new(e)
@ -470,8 +467,8 @@ proc replacef*(s: string, sub: Regex, by: string): string =
prev = match.last + 1
add(result, substr(s, prev))
proc parallelReplace*(s: string, subs: openArray[
tuple[pattern: Regex, repl: string]]): string =
proc multiReplace*(s: string, subs: openArray[
tuple[pattern: Regex, repl: string]]): string =
## Returns a modified copy of ``s`` with the substitutions in ``subs``
## applied in parallel.
result = ""
@ -490,13 +487,20 @@ proc parallelReplace*(s: string, subs: openArray[
# copy the rest:
add(result, substr(s, i))
proc parallelReplace*(s: string, subs: openArray[
tuple[pattern: Regex, repl: string]]): string {.deprecated.} =
## Returns a modified copy of ``s`` with the substitutions in ``subs``
## applied in parallel.
## **Deprecated since version 0.18.0**: Use ``multiReplace`` instead.
result = multiReplace(s, subs)
proc transformFile*(infile, outfile: string,
subs: openArray[tuple[pattern: Regex, repl: string]]) =
## reads in the file ``infile``, performs a parallel replacement (calls
## ``parallelReplace``) and writes back to ``outfile``. Raises ``IOError`` if an
## error occurs. This is supposed to be used for quick scripting.
var x = readFile(infile).string
writeFile(outfile, x.parallelReplace(subs))
writeFile(outfile, x.multiReplace(subs))
iterator split*(s: string, sep: Regex): string =
## Splits the string ``s`` into substrings.
@ -579,12 +583,12 @@ const ## common regular expressions
## describes an URL
when isMainModule:
doAssert match("(a b c)", re"\( .* \)")
doAssert match("(a b c)", rex"\( .* \)")
doAssert match("WHiLe", re("while", {reIgnoreCase}))
doAssert "0158787".match(re"\d+")
doAssert "ABC 0232".match(re"\w+\s+\d+")
doAssert "ABC".match(re"\d+ | \w+")
doAssert "ABC".match(rex"\d+ | \w+")
{.push warnings:off.}
doAssert matchLen("key", re(reIdentifier)) == 3

22
lib/js/jsffi.nim
View file

@ -177,7 +177,7 @@ proc `==`*(x, y: JsRoot): bool {. importcpp: "(# === #)" .}
## and not strings or numbers, this is a *comparison of references*.
{. experimental .}
macro `.`*(obj: JsObject, field: static[cstring]): JsObject =
macro `.`*(obj: JsObject, field: untyped): JsObject =
## Experimental dot accessor (get) for type JsObject.
## Returns the value of a property of name `field` from a JsObject `x`.
##
@ -196,14 +196,14 @@ macro `.`*(obj: JsObject, field: static[cstring]): JsObject =
helper(`obj`)
else:
if not mangledNames.hasKey($field):
mangledNames[$field] = $mangleJsName(field)
mangledNames[$field] = $mangleJsName($field)
let importString = "#." & mangledNames[$field]
result = quote do:
proc helper(o: JsObject): JsObject
{. importcpp: `importString`, gensym .}
helper(`obj`)
macro `.=`*(obj: JsObject, field: static[cstring], value: untyped): untyped =
macro `.=`*(obj: JsObject, field, value: untyped): untyped =
## Experimental dot accessor (set) for type JsObject.
## Sets the value of a property of name `field` in a JsObject `x` to `value`.
if validJsName($field):
@ -214,7 +214,7 @@ macro `.=`*(obj: JsObject, field: static[cstring], value: untyped): untyped =
helper(`obj`, `value`)
else:
if not mangledNames.hasKey($field):
mangledNames[$field] = $mangleJsName(field)
mangledNames[$field] = $mangleJsName($field)
let importString = "#." & mangledNames[$field] & " = #"
result = quote do:
proc helper(o: JsObject, v: auto)
@ -222,7 +222,7 @@ macro `.=`*(obj: JsObject, field: static[cstring], value: untyped): untyped =
helper(`obj`, `value`)
macro `.()`*(obj: JsObject,
field: static[cstring],
field: untyped,
args: varargs[JsObject, jsFromAst]): JsObject =
## Experimental "method call" operator for type JsObject.
## Takes the name of a method of the JavaScript object (`field`) and calls
@ -245,7 +245,7 @@ macro `.()`*(obj: JsObject,
importString = "#." & $field & "(@)"
else:
if not mangledNames.hasKey($field):
mangledNames[$field] = $mangleJsName(field)
mangledNames[$field] = $mangleJsName($field)
importString = "#." & mangledNames[$field] & "(@)"
result = quote:
proc helper(o: JsObject): JsObject
@ -257,7 +257,7 @@ macro `.()`*(obj: JsObject,
result[1].add args[idx].copyNimTree
macro `.`*[K: string | cstring, V](obj: JsAssoc[K, V],
field: static[cstring]): V =
field: untyped): V =
## Experimental dot accessor (get) for type JsAssoc.
## Returns the value of a property of name `field` from a JsObject `x`.
var importString: string
@ -265,7 +265,7 @@ macro `.`*[K: string | cstring, V](obj: JsAssoc[K, V],
importString = "#." & $field
else:
if not mangledNames.hasKey($field):
mangledNames[$field] = $mangleJsName(field)
mangledNames[$field] = $mangleJsName($field)
importString = "#." & mangledNames[$field]
result = quote do:
proc helper(o: type(`obj`)): `obj`.V
@ -273,7 +273,7 @@ macro `.`*[K: string | cstring, V](obj: JsAssoc[K, V],
helper(`obj`)
macro `.=`*[K: string | cstring, V](obj: JsAssoc[K, V],
field: static[cstring],
field: untyped,
value: V): untyped =
## Experimental dot accessor (set) for type JsAssoc.
## Sets the value of a property of name `field` in a JsObject `x` to `value`.
@ -282,7 +282,7 @@ macro `.=`*[K: string | cstring, V](obj: JsAssoc[K, V],
importString = "#." & $field & " = #"
else:
if not mangledNames.hasKey($field):
mangledNames[$field] = $mangleJsName(field)
mangledNames[$field] = $mangleJsName($field)
importString = "#." & mangledNames[$field] & " = #"
result = quote do:
proc helper(o: type(`obj`), v: `obj`.V)
@ -290,7 +290,7 @@ macro `.=`*[K: string | cstring, V](obj: JsAssoc[K, V],
helper(`obj`, `value`)
macro `.()`*[K: string | cstring, V: proc](obj: JsAssoc[K, V],
field: static[cstring],
field: untyped,
args: varargs[untyped]): auto =
## Experimental "method call" operator for type JsAssoc.
## Takes the name of a method of the JavaScript object (`field`) and calls

2
lib/nimbase.h
View file

@ -70,7 +70,7 @@ __clang__
#if defined(_MSC_VER)
# pragma warning(disable: 4005 4100 4101 4189 4191 4200 4244 4293 4296 4309)
# pragma warning(disable: 4310 4365 4456 4477 4514 4574 4611 4668 4702 4706)
# pragma warning(disable: 4710 4711 4774 4800 4820 4996 4090)
# pragma warning(disable: 4710 4711 4774 4800 4820 4996 4090 4297)
#endif
/* ------------------------------------------------------------------------- */

9
lib/packages/docutils/highlite.nim
View file

@ -31,14 +31,12 @@ type
state: TokenClass
SourceLanguage* = enum
langNone, langNim, langNimrod, langCpp, langCsharp, langC, langJava,
langNone, langNim, langCpp, langCsharp, langC, langJava,
langYaml
{.deprecated: [TSourceLanguage: SourceLanguage, TTokenClass: TokenClass,
TGeneralTokenizer: GeneralTokenizer].}
const
sourceLanguageToStr*: array[SourceLanguage, string] = ["none",
"Nim", "Nimrod", "C++", "C#", "C", "Java", "Yaml"]
"Nim", "C++", "C#", "C", "Java", "Yaml"]
tokenClassToStr*: array[TokenClass, string] = ["Eof", "None", "Whitespace",
"DecNumber", "BinNumber", "HexNumber", "OctNumber", "FloatNumber",
"Identifier", "Keyword", "StringLit", "LongStringLit", "CharLit",
@ -398,7 +396,6 @@ type
TokenizerFlag = enum
hasPreprocessor, hasNestedComments
TokenizerFlags = set[TokenizerFlag]
{.deprecated: [TTokenizerFlag: TokenizerFlag, TTokenizerFlags: TokenizerFlags].}
proc clikeNextToken(g: var GeneralTokenizer, keywords: openArray[string],
flags: TokenizerFlags) =
@ -888,7 +885,7 @@ proc yamlNextToken(g: var GeneralTokenizer) =
proc getNextToken*(g: var GeneralTokenizer, lang: SourceLanguage) =
case lang
of langNone: assert false
of langNim, langNimrod: nimNextToken(g)
of langNim: nimNextToken(g)
of langCpp: cppNextToken(g)
of langCsharp: csharpNextToken(g)
of langC: cNextToken(g)

13
lib/packages/docutils/rst.nim
View file

@ -45,8 +45,6 @@ type
MsgHandler* = proc (filename: string, line, col: int, msgKind: MsgKind,
arg: string) {.nimcall.} ## what to do in case of an error
FindFileHandler* = proc (filename: string): string {.nimcall.}
{.deprecated: [TRstParseOptions: RstParseOptions, TRstParseOption: RstParseOption,
TMsgKind: MsgKind].}
const
messages: array[MsgKind, string] = [
@ -127,8 +125,6 @@ type
bufpos*: int
line*, col*, baseIndent*: int
skipPounds*: bool
{.deprecated: [TTokType: TokType, TToken: Token, TTokenSeq: TokenSeq,
TLexer: Lexer].}
proc getThing(L: var Lexer, tok: var Token, s: set[char]) =
tok.kind = tkWord
@ -288,10 +284,6 @@ type
hasToc*: bool
EParseError* = object of ValueError
{.deprecated: [TLevelMap: LevelMap, TSubstitution: Substitution,
TSharedState: SharedState, TRstParser: RstParser,
TMsgHandler: MsgHandler, TFindFileHandler: FindFileHandler,
TMsgClass: MsgClass].}
proc whichMsgClass*(k: MsgKind): MsgClass =
## returns which message class `k` belongs to.
@ -341,11 +333,6 @@ proc rstMessage(p: RstParser, msgKind: MsgKind) =
p.col + p.tok[p.idx].col, msgKind,
p.tok[p.idx].symbol)
when false:
proc corrupt(p: RstParser) =
assert p.indentStack[0] == 0
for i in 1 .. high(p.indentStack): assert p.indentStack[i] < 1_000
proc currInd(p: RstParser): int =
result = p.indentStack[high(p.indentStack)]

26
lib/packages/docutils/rstgen.nim
View file

@ -46,7 +46,7 @@ type
target*: OutputTarget
config*: StringTableRef
splitAfter*: int # split too long entries in the TOC
listingCounter: int
listingCounter*: int
tocPart*: seq[TocEntry]
hasToc*: bool
theIndex: string # Contents of the index file to be dumped at the end.
@ -61,6 +61,9 @@ type
seenIndexTerms: Table[string, int] ## \
## Keeps count of same text index terms to generate different identifiers
## for hyperlinks. See renderIndexTerm proc for details.
id*: int ## A counter useful for generating IDs.
onTestSnippet*: proc (d: var RstGenerator; filename, cmd: string; status: int;
content: string)
PDoc = var RstGenerator ## Alias to type less.
@ -69,8 +72,9 @@ type
startLine: int ## The starting line of the code block, by default 1.
langStr: string ## Input string used to specify the language.
lang: SourceLanguage ## Type of highlighting, by default none.
{.deprecated: [TRstGenerator: RstGenerator, TTocEntry: TocEntry,
TOutputTarget: OutputTarget, TMetaEnum: MetaEnum].}
filename: string
testCmd: string
status: int
proc init(p: var CodeBlockParams) =
## Default initialisation of CodeBlockParams to sane values.
@ -133,6 +137,7 @@ proc initRstGenerator*(g: var RstGenerator, target: OutputTarget,
g.options = options
g.findFile = findFile
g.currentSection = ""
g.id = 0
let fileParts = filename.splitFile
if fileParts.ext == ".nim":
g.currentSection = "Module " & fileParts.name
@ -368,7 +373,6 @@ type
##
## The value indexed by this IndexEntry is a sequence with the real index
## entries found in the ``.idx`` file.
{.deprecated: [TIndexEntry: IndexEntry, TIndexedDocs: IndexedDocs].}
proc cmp(a, b: IndexEntry): int =
## Sorts two ``IndexEntry`` first by `keyword` field, then by `link`.
@ -823,13 +827,20 @@ proc parseCodeBlockField(d: PDoc, n: PRstNode, params: var CodeBlockParams) =
var number: int
if parseInt(n.getFieldValue, number) > 0:
params.startLine = number
of "file":
of "file", "filename":
# The ``file`` option is a Nim extension to the official spec, it acts
# like it would for other directives like ``raw`` or ``cvs-table``. This
# field is dealt with in ``rst.nim`` which replaces the existing block with
# the referenced file, so we only need to ignore it here to avoid incorrect
# warning messages.
discard
params.filename = n.getFieldValue.strip
of "test":
params.testCmd = n.getFieldValue.strip
if params.testCmd.len == 0: params.testCmd = "nim c -r $1"
of "status":
var status: int
if parseInt(n.getFieldValue, status) > 0:
params.status = status
of "default-language":
params.langStr = n.getFieldValue.strip
params.lang = params.langStr.getSourceLanguage
@ -901,6 +912,9 @@ proc renderCodeBlock(d: PDoc, n: PRstNode, result: var string) =
var m = n.sons[2].sons[0]
assert m.kind == rnLeaf
if params.testCmd.len > 0 and d.onTestSnippet != nil:
d.onTestSnippet(d, params.filename, params.testCmd, params.status, m.text)
let (blockStart, blockEnd) = buildLinesHTMLTable(d, params, m.text)
dispA(d.target, result, blockStart, "\\begin{rstpre}\n", [])

3
lib/pure/asyncdispatch.nim
View file

@ -59,9 +59,10 @@ export asyncfutures, asyncstreams
##
## .. code-block::nim
## var future = socket.recv(100)
## future.callback =
## future.addCallback(
## proc () =
## echo(future.read)
## )
##
## All asynchronous functions returning a ``Future`` will not block. They
## will not however return immediately. An asynchronous function will have

4
lib/pure/asyncfutures.nim
View file

@ -334,7 +334,7 @@ proc all*[T](futs: varargs[Future[T]]): auto =
let totalFutures = len(futs)
for fut in futs:
fut.callback = proc(f: Future[T]) =
fut.addCallback proc (f: Future[T]) =
inc(completedFutures)
if not retFuture.finished:
if f.failed:
@ -356,7 +356,7 @@ proc all*[T](futs: varargs[Future[T]]): auto =
for i, fut in futs:
proc setCallback(i: int) =
fut.callback = proc(f: Future[T]) =
fut.addCallback proc (f: Future[T]) =
inc(completedFutures)
if not retFuture.finished:
if f.failed:

5
lib/pure/asynchttpserver.nim
View file

@ -275,10 +275,7 @@ proc processClient(server: AsyncHttpServer, client: AsyncSocket, address: string
lineFut.mget() = newStringOfCap(80)
while not client.isClosed:
try:
await processRequest(server, request, client, address, lineFut, callback)
except:
asyncCheck request.mget().respondError(Http500)
await processRequest(server, request, client, address, lineFut, callback)
proc serve*(server: AsyncHttpServer, port: Port,
callback: proc (request: Request): Future[void] {.closure,gcsafe.},

2
lib/pure/bitops.nim
View file

@ -181,7 +181,7 @@ elif useICC_builtins:
proc countSetBits*(x: SomeInteger): int {.inline, nosideeffect.} =
## Counts the set bits in integer. (also called Hamming weight.)
## Counts the set bits in integer. (also called `Hamming weight`:idx:.)
# TODO: figure out if ICC support _popcnt32/_popcnt64 on platform without POPCNT.
# like GCC and MSVC
when nimvm:

44
lib/pure/cgi.nim
View file

@ -29,21 +29,8 @@
## writeLine(stdout, "your password: " & myData["password"])
## writeLine(stdout, "</body></html>")
import strutils, os, strtabs, cookies
proc encodeUrl*(s: string): string =
## Encodes a value to be HTTP safe: This means that characters in the set
## ``{'A'..'Z', 'a'..'z', '0'..'9', '_'}`` are carried over to the result,
## a space is converted to ``'+'`` and every other character is encoded as
## ``'%xx'`` where ``xx`` denotes its hexadecimal value.
result = newStringOfCap(s.len + s.len shr 2) # assume 12% non-alnum-chars
for i in 0..s.len-1:
case s[i]
of 'a'..'z', 'A'..'Z', '0'..'9', '_': add(result, s[i])
of ' ': add(result, '+')
else:
add(result, '%')
add(result, toHex(ord(s[i]), 2))
import strutils, os, strtabs, cookies, uri
export uri.encodeUrl, uri.decodeUrl
proc handleHexChar(c: char, x: var int) {.inline.} =
case c
@ -52,28 +39,6 @@ proc handleHexChar(c: char, x: var int) {.inline.} =
of 'A'..'F': x = (x shl 4) or (ord(c) - ord('A') + 10)
else: assert(false)
proc decodeUrl*(s: string): string =
## Decodes a value from its HTTP representation: This means that a ``'+'``
## is converted to a space, ``'%xx'`` (where ``xx`` denotes a hexadecimal
## value) is converted to the character with ordinal number ``xx``, and
## and every other character is carried over.
result = newString(s.len)
var i = 0
var j = 0
while i < s.len:
case s[i]
of '%':
var x = 0
handleHexChar(s[i+1], x)
handleHexChar(s[i+2], x)
inc(i, 2)
result[j] = chr(x)
of '+': result[j] = ' '
else: result[j] = s[i]
inc(i)
inc(j)
setLen(result, j)
proc addXmlChar(dest: var string, c: char) {.inline.} =
case c
of '&': add(dest, "&amp;")
@ -390,8 +355,3 @@ proc existsCookie*(name: string): bool =
## Checks if a cookie of `name` exists.
if gcookies == nil: gcookies = parseCookies(getHttpCookie())
result = hasKey(gcookies, name)
when isMainModule:
const test1 = "abc\L+def xyz"
assert encodeUrl(test1) == "abc%0A%2Bdef+xyz"
assert decodeUrl(encodeUrl(test1)) == test1

4
lib/pure/collections/sequtils.nim
View file

@ -66,7 +66,7 @@ proc cycle*[T](s: openArray[T], n: Natural): seq[T] =
##
## Example:
##
## .. code-block:
## .. code-block::
##
## let
## s = @[1, 2, 3]
@ -84,7 +84,7 @@ proc repeat*[T](x: T, n: Natural): seq[T] =
##
## Example:
##
## .. code-block:
## .. code-block::
##
## let
## total = repeat(5, 3)

4
lib/pure/htmlgen.nim
View file

@ -59,8 +59,8 @@ proc xmlCheckedTag*(e: NimNode, tag: string, optAttr = "", reqAttr = "",
# copy the attributes; when iterating over them these lists
# will be modified, so that each attribute is only given one value
var req = split(reqAttr)
var opt = split(optAttr)
var req = splitWhitespace(reqAttr)
var opt = splitWhitespace(optAttr)
result = newNimNode(nnkBracket, e)
result.add(newStrLitNode("<"))
result.add(newStrLitNode(tag))

28
lib/pure/ioselects/ioselectors_select.nim
View file

@ -279,8 +279,9 @@ proc updateHandle*[T](s: Selector[T], fd: int | SocketHandle,
inc(s.count)
pkey.events = events
proc unregister*[T](s: Selector[T], fd: int | SocketHandle) =
proc unregister*[T](s: Selector[T], fd: SocketHandle|int) =
s.withSelectLock():
let fd = fd.SocketHandle
var pkey = s.getKey(fd)
if Event.Read in pkey.events:
IOFD_CLR(fd, addr s.rSet)
@ -438,18 +439,19 @@ template withData*[T](s: Selector[T], fd: SocketHandle|int, value,
body1, body2: untyped) =
mixin withSelectLock
s.withSelectLock():
var value: ptr T
let fdi = int(fd)
var i = 0
while i < FD_SETSIZE:
if s.fds[i].ident == fdi:
value = addr(s.fds[i].data)
break
inc(i)
if i != FD_SETSIZE:
body1
else:
body2
block:
var value: ptr T
let fdi = int(fd)
var i = 0
while i < FD_SETSIZE:
if s.fds[i].ident == fdi:
value = addr(s.fds[i].data)
break
inc(i)
if i != FD_SETSIZE:
body1
else:
body2
proc getFd*[T](s: Selector[T]): int =

164
lib/pure/json.nim
View file

@ -1346,6 +1346,16 @@ proc createJsonIndexer(jsonNode: NimNode,
indexNode
)
proc transformJsonIndexer(jsonNode: NimNode): NimNode =
case jsonNode.kind
of nnkBracketExpr:
result = newNimNode(nnkCurlyExpr)
else:
result = jsonNode.copy()
for child in jsonNode:
result.add(transformJsonIndexer(child))
template verifyJsonKind(node: JsonNode, kinds: set[JsonNodeKind],
ast: string) =
if node.kind notin kinds:
@ -1524,6 +1534,35 @@ proc processObjField(field, jsonNode: NimNode): seq[NimNode] =
doAssert result.len > 0
proc processFields(obj: NimNode,
jsonNode: NimNode): seq[NimNode] {.compileTime.} =
## Process all the fields of an ``ObjectTy`` and any of its
## parent type's fields (via inheritance).
result = @[]
case obj.kind
of nnkObjectTy:
expectKind(obj[2], nnkRecList)
for field in obj[2]:
let nodes = processObjField(field, jsonNode)
result.add(nodes)
# process parent type fields
case obj[1].kind
of nnkBracketExpr:
assert $obj[1][0] == "ref"
result.add(processFields(getType(obj[1][1]), jsonNode))
of nnkSym:
result.add(processFields(getType(obj[1]), jsonNode))
else:
discard
of nnkTupleTy:
for identDefs in obj:
expectKind(identDefs, nnkIdentDefs)
let nodes = processObjField(identDefs[0], jsonNode)
result.add(nodes)
else:
doAssert false, "Unable to process field type: " & $obj.kind
proc processType(typeName: NimNode, obj: NimNode,
jsonNode: NimNode, isRef: bool): NimNode {.compileTime.} =
## Process a type such as ``Sym "float"`` or ``ObjectTy ...``.
@ -1533,20 +1572,21 @@ proc processType(typeName: NimNode, obj: NimNode,
## .. code-block::plain
## ObjectTy
## Empty
## Empty
## InheritanceInformation
## RecList
## Sym "events"
case obj.kind
of nnkObjectTy:
of nnkObjectTy, nnkTupleTy:
# Create object constructor.
result = newNimNode(nnkObjConstr)
result.add(typeName) # Name of the type to construct.
result =
if obj.kind == nnkObjectTy: newNimNode(nnkObjConstr)
else: newNimNode(nnkPar)
# Process each object field and add it as an exprColonExpr
expectKind(obj[2], nnkRecList)
for field in obj[2]:
let nodes = processObjField(field, jsonNode)
result.add(nodes)
if obj.kind == nnkObjectTy:
result.add(typeName) # Name of the type to construct.
# Process each object/tuple field and add it as an exprColonExpr
result.add(processFields(obj, jsonNode))
# Object might be null. So we need to check for that.
if isRef:
@ -1569,25 +1609,14 @@ proc processType(typeName: NimNode, obj: NimNode,
`getEnumCall`
)
of nnkSym:
case ($typeName).normalize
of "float":
result = quote do:
(
verifyJsonKind(`jsonNode`, {JFloat, JInt}, astToStr(`jsonNode`));
if `jsonNode`.kind == JFloat: `jsonNode`.fnum else: `jsonNode`.num.float
)
let name = ($typeName).normalize
case name
of "string":
result = quote do:
(
verifyJsonKind(`jsonNode`, {JString, JNull}, astToStr(`jsonNode`));
if `jsonNode`.kind == JNull: nil else: `jsonNode`.str
)
of "int":
result = quote do:
(
verifyJsonKind(`jsonNode`, {JInt}, astToStr(`jsonNode`));
`jsonNode`.num.int
)
of "biggestint":
result = quote do:
(
@ -1601,12 +1630,36 @@ proc processType(typeName: NimNode, obj: NimNode,
`jsonNode`.bval
)
else:
doAssert false, "Unable to process nnkSym " & $typeName
if name.startsWith("int") or name.startsWith("uint"):
result = quote do:
(
verifyJsonKind(`jsonNode`, {JInt}, astToStr(`jsonNode`));
`jsonNode`.num.`obj`
)
elif name.startsWith("float"):
result = quote do:
(
verifyJsonKind(`jsonNode`, {JInt, JFloat}, astToStr(`jsonNode`));
if `jsonNode`.kind == JFloat: `jsonNode`.fnum.`obj` else: `jsonNode`.num.`obj`
)
else:
doAssert false, "Unable to process nnkSym " & $typeName
else:
doAssert false, "Unable to process type: " & $obj.kind
doAssert(not result.isNil(), "processType not initialised.")
import options
proc workaroundMacroNone[T](): Option[T] =
none(T)
proc depth(n: NimNode, current = 0): int =
result = 1
for child in n:
let d = 1 + child.depth(current + 1)
if d > result:
result = d
proc createConstructor(typeSym, jsonNode: NimNode): NimNode =
## Accepts a type description, i.e. "ref Type", "seq[Type]", "Type" etc.
##
@ -1616,10 +1669,50 @@ proc createConstructor(typeSym, jsonNode: NimNode): NimNode =
# echo("--createConsuctor-- \n", treeRepr(typeSym))
# echo()
if depth(jsonNode) > 150:
error("The `to` macro does not support ref objects with cycles.", jsonNode)
case typeSym.kind
of nnkBracketExpr:
var bracketName = ($typeSym[0]).normalize
case bracketName
of "option":
# TODO: Would be good to verify that this is Option[T] from
# options module I suppose.
let lenientJsonNode = transformJsonIndexer(jsonNode)
let optionGeneric = typeSym[1]
let value = createConstructor(typeSym[1], jsonNode)
let workaround = bindSym("workaroundMacroNone") # TODO: Nim Bug: This shouldn't be necessary.
result = quote do:
(
if `lenientJsonNode`.isNil: `workaround`[`optionGeneric`]() else: some[`optionGeneric`](`value`)
)
of "table", "orderedtable":
let tableKeyType = typeSym[1]
if ($tableKeyType).cmpIgnoreStyle("string") != 0:
error("JSON doesn't support keys of type " & $tableKeyType)
let tableValueType = typeSym[2]
let forLoopKey = genSym(nskForVar, "key")
let indexerNode = createJsonIndexer(jsonNode, forLoopKey)
let constructorNode = createConstructor(tableValueType, indexerNode)
let tableInit =
if bracketName == "table":
bindSym("initTable")
else:
bindSym("initOrderedTable")
# Create a statement expression containing a for loop.
result = quote do:
(
var map = `tableInit`[`tableKeyType`, `tableValueType`]();
verifyJsonKind(`jsonNode`, {JObject}, astToStr(`jsonNode`));
for `forLoopKey` in keys(`jsonNode`.fields): map[`forLoopKey`] = `constructorNode`;
map
)
of "ref":
# Ref type.
var typeName = $typeSym[1]
@ -1663,12 +1756,23 @@ proc createConstructor(typeSym, jsonNode: NimNode): NimNode =
let obj = getType(typeSym)
result = processType(typeSym, obj, jsonNode, false)
of nnkSym:
# Handle JsonNode.
if ($typeSym).cmpIgnoreStyle("jsonnode") == 0:
return jsonNode
# Handle all other types.
let obj = getType(typeSym)
if obj.kind == nnkBracketExpr:
# When `Sym "Foo"` turns out to be a `ref object`.
result = createConstructor(obj, jsonNode)
else:
result = processType(typeSym, obj, jsonNode, false)
of nnkTupleTy:
result = processType(typeSym, typeSym, jsonNode, false)
of nnkPar:
# TODO: The fact that `jsonNode` here works to give a good line number
# is weird. Specifying typeSym should work but doesn't.
error("Use a named tuple instead of: " & $toStrLit(typeSym), jsonNode)
else:
doAssert false, "Unable to create constructor for: " & $typeSym.kind
@ -1796,10 +1900,18 @@ macro to*(node: JsonNode, T: typedesc): untyped =
expectKind(typeNode, nnkBracketExpr)
doAssert(($typeNode[0]).normalize == "typedesc")
result = createConstructor(typeNode[1], node)
# TODO: Rename postProcessValue and move it (?)
result = postProcessValue(result)
# Create `temp` variable to store the result in case the user calls this
# on `parseJson` (see bug #6604).
result = newNimNode(nnkStmtListExpr)
let temp = genSym(nskLet, "temp")
result.add quote do:
let `temp` = `node`
let constructor = createConstructor(typeNode[1], temp)
# TODO: Rename postProcessValue and move it (?)
result.add(postProcessValue(constructor))
# echo(treeRepr(result))
# echo(toStrLit(result))
when false:

8
lib/pure/logging.nim
View file

@ -202,13 +202,17 @@ when not defined(js):
proc countLogLines(logger: RollingFileLogger): int =
result = 0
for line in logger.file.lines():
let fp = open(logger.baseName, fmRead)
for line in fp.lines():
result.inc()
fp.close()
proc countFiles(filename: string): int =
# Example: file.log.1
result = 0
let (dir, name, ext) = splitFile(filename)
var (dir, name, ext) = splitFile(filename)
if dir == "":
dir = "."
for kind, path in walkDir(dir):
if kind == pcFile:
let llfn = name & ext & ExtSep

9
lib/pure/mimetypes.nim
View file

@ -491,6 +491,8 @@ const mimes* = {
"vrml": "x-world/x-vrml",
"wrl": "x-world/x-vrml"}
from strutils import startsWith
proc newMimetypes*(): MimeDB =
## Creates a new Mimetypes database. The database will contain the most
## common mimetypes.
@ -498,8 +500,11 @@ proc newMimetypes*(): MimeDB =
proc getMimetype*(mimedb: MimeDB, ext: string, default = "text/plain"): string =
## Gets mimetype which corresponds to ``ext``. Returns ``default`` if ``ext``
## could not be found.
result = mimedb.mimes.getOrDefault(ext)
## could not be found. ``ext`` can start with an optional dot which is ignored.
if ext.startsWith("."):
result = mimedb.mimes.getOrDefault(ext.substr(1))
else:
result = mimedb.mimes.getOrDefault(ext)
if result == "":
return default

14
lib/pure/net.nim
View file

@ -145,7 +145,7 @@ type
SOBool* = enum ## Boolean socket options.
OptAcceptConn, OptBroadcast, OptDebug, OptDontRoute, OptKeepAlive,
OptOOBInline, OptReuseAddr, OptReusePort
OptOOBInline, OptReuseAddr, OptReusePort, OptNoDelay
ReadLineResult* = enum ## result for readLineAsync
ReadFullLine, ReadPartialLine, ReadDisconnected, ReadNone
@ -865,6 +865,11 @@ proc close*(socket: Socket) =
socket.fd.close()
when defined(posix):
from posix import TCP_NODELAY
else:
from winlean import TCP_NODELAY
proc toCInt*(opt: SOBool): cint =
## Converts a ``SOBool`` into its Socket Option cint representation.
case opt
@ -876,6 +881,7 @@ proc toCInt*(opt: SOBool): cint =
of OptOOBInline: SO_OOBINLINE
of OptReuseAddr: SO_REUSEADDR
of OptReusePort: SO_REUSEPORT
of OptNoDelay: TCP_NODELAY
proc getSockOpt*(socket: Socket, opt: SOBool, level = SOL_SOCKET): bool {.
tags: [ReadIOEffect].} =
@ -898,6 +904,12 @@ proc getPeerAddr*(socket: Socket): (string, Port) =
proc setSockOpt*(socket: Socket, opt: SOBool, value: bool, level = SOL_SOCKET) {.
tags: [WriteIOEffect].} =
## Sets option ``opt`` to a boolean value specified by ``value``.
##
## .. code-block:: Nim
## var socket = newSocket()
## socket.setSockOpt(OptReusePort, true)
## socket.setSockOpt(OptNoDelay, true, level=IPPROTO_TCP.toInt)
##
var valuei = cint(if value: 1 else: 0)
setSockOptInt(socket.fd, cint(level), toCInt(opt), valuei)

2
lib/pure/os.nim
View file

@ -630,7 +630,7 @@ proc execShellCmd*(command: string): int {.rtl, extern: "nos$1",
## the process has finished. To execute a program without having a
## shell involved, use the `execProcess` proc of the `osproc`
## module.
when defined(linux):
when defined(posix):
result = c_system(command) shr 8
else:
result = c_system(command)

15
lib/pure/ospaths.nim
View file

@ -602,14 +602,13 @@ proc quoteShellPosix*(s: string): string {.noSideEffect, rtl, extern: "nosp$1".}
else:
return "'" & s.replace("'", "'\"'\"'") & "'"
proc quoteShell*(s: string): string {.noSideEffect, rtl, extern: "nosp$1".} =
## Quote ``s``, so it can be safely passed to shell.
when defined(Windows):
return quoteShellWindows(s)
elif defined(posix):
return quoteShellPosix(s)
else:
{.error:"quoteShell is not supported on your system".}
when defined(windows) or defined(posix):
proc quoteShell*(s: string): string {.noSideEffect, rtl, extern: "nosp$1".} =
## Quote ``s``, so it can be safely passed to shell.
when defined(windows):
return quoteShellWindows(s)
else:
return quoteShellPosix(s)
when isMainModule:
assert quoteShellWindows("aaa") == "aaa"

177
lib/pure/osproc.nim
View file

@ -41,6 +41,8 @@ type
## Windows: Named pipes are used so that you can peek
## at the process' output streams.
poDemon ## Windows: The program creates no Window.
## Unix: Start the program as a demon. This is still
## work in progress!
ProcessObj = object of RootObj
when defined(windows):
@ -167,8 +169,7 @@ proc waitForExit*(p: Process, timeout: int = -1): int {.rtl,
## On posix, if the process has exited because of a signal, 128 + signal
## number will be returned.
proc peekExitCode*(p: Process): int {.tags: [].}
proc peekExitCode*(p: Process): int {.rtl, extern: "nosp$1", tags: [].}
## return -1 if the process is still running. Otherwise the process' exit code
##
## On posix, if the process has exited because of a signal, 128 + signal
@ -231,55 +232,79 @@ proc execProcesses*(cmds: openArray[string],
## executes the commands `cmds` in parallel. Creates `n` processes
## that execute in parallel. The highest return value of all processes
## is returned. Runs `beforeRunEvent` before running each command.
when false:
# poParentStreams causes problems on Posix, so we simply disable it:
var options = options - {poParentStreams}
assert n > 0
if n > 1:
var q: seq[Process]
newSeq(q, n)
var i = 0
var q = newSeq[Process](n)
var m = min(n, cmds.len)
for i in 0..m-1:
when defined(windows):
var w: WOHandleArray
var wcount = m
while i < m:
if beforeRunEvent != nil:
beforeRunEvent(i)
q[i] = startProcess(cmds[i], options=options + {poEvalCommand})
when defined(noBusyWaiting):
var r = 0
for i in m..high(cmds):
when defined(debugExecProcesses):
var err = ""
var outp = outputStream(q[r])
while running(q[r]) or not atEnd(outp):
err.add(outp.readLine())
err.add("\n")
echo(err)
result = max(waitForExit(q[r]), result)
if afterRunEvent != nil: afterRunEvent(r, q[r])
if q[r] != nil: close(q[r])
if beforeRunEvent != nil:
beforeRunEvent(i)
q[r] = startProcess(cmds[i], options=options + {poEvalCommand})
r = (r + 1) mod n
else:
var i = m
while i <= high(cmds):
sleep(50)
for r in 0..n-1:
q[i] = startProcess(cmds[i], options = options + {poEvalCommand})
when defined(windows):
w[i] = q[i].fProcessHandle
inc(i)
var ecount = len(cmds)
while ecount > 0:
when defined(windows):
# waiting for all children, get result if any child exits
var ret = waitForMultipleObjects(int32(wcount), addr(w), 0'i32,
INFINITE)
if ret == WAIT_TIMEOUT:
# must not be happen
discard
elif ret == WAIT_FAILED:
raiseOSError(osLastError())
else:
var status : cint = 1
# waiting for all children, get result if any child exits
let res = waitpid(-1, status, 0)
if res > 0:
for r in 0..m-1:
if not isNil(q[r]) and q[r].id == res:
# we updating `exitStatus` manually, so `running()` can work.
if WIFEXITED(status) or WIFSIGNALED(status):
q[r].exitStatus = status
break
else:
let err = osLastError()
if err == OSErrorCode(ECHILD):
# some child exits, we need to check our childs exit codes
discard
elif err == OSErrorCode(EINTR):
# signal interrupted our syscall, lets repeat it
continue
else:
# all other errors are exceptions
raiseOSError(err)
for r in 0..m-1:
if not isNil(q[r]):
if not running(q[r]):
#echo(outputStream(q[r]).readLine())
result = max(waitForExit(q[r]), result)
result = max(result, q[r].peekExitCode())
if afterRunEvent != nil: afterRunEvent(r, q[r])
if q[r] != nil: close(q[r])
if beforeRunEvent != nil:
beforeRunEvent(i)
q[r] = startProcess(cmds[i], options=options + {poEvalCommand})
inc(i)
if i > high(cmds): break
for j in 0..m-1:
result = max(waitForExit(q[j]), result)
if afterRunEvent != nil: afterRunEvent(j, q[j])
if q[j] != nil: close(q[j])
close(q[r])
if i < len(cmds):
if beforeRunEvent != nil: beforeRunEvent(i)
q[r] = startProcess(cmds[i],
options = options + {poEvalCommand})
when defined(windows):
w[r] = q[r].fProcessHandle
inc(i)
else:
q[r] = nil
when defined(windows):
for c in r..MAXIMUM_WAIT_OBJECTS - 2:
w[c] = w[c + 1]
dec(wcount)
dec(ecount)
else:
for i in 0..high(cmds):
if beforeRunEvent != nil:
@ -321,6 +346,8 @@ when not defined(useNimRtl):
elif not running(p): break
close(p)
template streamAccess(p) =
assert poParentStreams notin p.options, "API usage error: stream access not allowed when you use poParentStreams"
when defined(Windows) and not defined(useNimRtl):
# We need to implement a handle stream for Windows:
@ -581,12 +608,15 @@ when defined(Windows) and not defined(useNimRtl):
return res
proc inputStream(p: Process): Stream =
streamAccess(p)
result = newFileHandleStream(p.inHandle)
proc outputStream(p: Process): Stream =
streamAccess(p)
result = newFileHandleStream(p.outHandle)
proc errorStream(p: Process): Stream =
streamAccess(p)
result = newFileHandleStream(p.errHandle)
proc execCmd(command: string): int =
@ -682,9 +712,7 @@ elif not defined(useNimRtl):
sysEnv: cstringArray
workingDir: cstring
pStdin, pStdout, pStderr, pErrorPipe: array[0..1, cint]
optionPoUsePath: bool
optionPoParentStreams: bool
optionPoStdErrToStdOut: bool
options: set[ProcessOption]
{.deprecated: [TStartProcessData: StartProcessData].}
const useProcessAuxSpawn = declared(posix_spawn) and not defined(useFork) and
@ -749,10 +777,8 @@ elif not defined(useNimRtl):
data.pStdin = pStdin
data.pStdout = pStdout
data.pStderr = pStderr
data.optionPoParentStreams = poParentStreams in options
data.optionPoUsePath = poUsePath in options
data.optionPoStdErrToStdOut = poStdErrToStdOut in options
data.workingDir = workingDir
data.options = options
when useProcessAuxSpawn:
var currentDir = getCurrentDir()
@ -801,19 +827,22 @@ elif not defined(useNimRtl):
var mask: Sigset
chck sigemptyset(mask)
chck posix_spawnattr_setsigmask(attr, mask)
chck posix_spawnattr_setpgroup(attr, 0'i32)
if poDemon in data.options:
chck posix_spawnattr_setpgroup(attr, 0'i32)
chck posix_spawnattr_setflags(attr, POSIX_SPAWN_USEVFORK or
POSIX_SPAWN_SETSIGMASK or
POSIX_SPAWN_SETPGROUP)
var flags = POSIX_SPAWN_USEVFORK or
POSIX_SPAWN_SETSIGMASK
if poDemon in data.options:
flags = flags or POSIX_SPAWN_SETPGROUP
chck posix_spawnattr_setflags(attr, flags)
if not data.optionPoParentStreams:
if not (poParentStreams in data.options):
chck posix_spawn_file_actions_addclose(fops, data.pStdin[writeIdx])
chck posix_spawn_file_actions_adddup2(fops, data.pStdin[readIdx], readIdx)
chck posix_spawn_file_actions_addclose(fops, data.pStdout[readIdx])
chck posix_spawn_file_actions_adddup2(fops, data.pStdout[writeIdx], writeIdx)
chck posix_spawn_file_actions_addclose(fops, data.pStderr[readIdx])
if data.optionPoStdErrToStdOut:
if (poStdErrToStdOut in data.options):
chck posix_spawn_file_actions_adddup2(fops, data.pStdout[writeIdx], 2)
else:
chck posix_spawn_file_actions_adddup2(fops, data.pStderr[writeIdx], 2)
@ -823,7 +852,7 @@ elif not defined(useNimRtl):
setCurrentDir($data.workingDir)
var pid: Pid
if data.optionPoUsePath:
if (poUsePath in data.options):
res = posix_spawnp(pid, data.sysCommand, fops, attr, data.sysArgs, data.sysEnv)
else:
res = posix_spawn(pid, data.sysCommand, fops, attr, data.sysArgs, data.sysEnv)
@ -885,7 +914,7 @@ elif not defined(useNimRtl):
# Warning: no GC here!
# Or anything that touches global structures - all called nim procs
# must be marked with stackTrace:off. Inspect C code after making changes.
if not data.optionPoParentStreams:
if not (poParentStreams in data.options):
discard close(data.pStdin[writeIdx])
if dup2(data.pStdin[readIdx], readIdx) < 0:
startProcessFail(data)
@ -893,7 +922,7 @@ elif not defined(useNimRtl):
if dup2(data.pStdout[writeIdx], writeIdx) < 0:
startProcessFail(data)
discard close(data.pStderr[readIdx])
if data.optionPoStdErrToStdOut:
if (poStdErrToStdOut in data.options):
if dup2(data.pStdout[writeIdx], 2) < 0:
startProcessFail(data)
else:
@ -907,7 +936,7 @@ elif not defined(useNimRtl):
discard close(data.pErrorPipe[readIdx])
discard fcntl(data.pErrorPipe[writeIdx], F_SETFD, FD_CLOEXEC)
if data.optionPoUsePath:
if (poUsePath in data.options):
when defined(uClibc) or defined(linux):
# uClibc environment (OpenWrt included) doesn't have the full execvpe
let exe = findExe(data.sysCommand)
@ -939,19 +968,22 @@ elif not defined(useNimRtl):
if kill(p.id, SIGCONT) != 0'i32: raiseOsError(osLastError())
proc running(p: Process): bool =
var ret : int
var status : cint = 1
ret = waitpid(p.id, status, WNOHANG)
if ret == int(p.id):
if isExitStatus(status):
p.exitStatus = status
return false
else:
return true
elif ret == 0:
return true # Can't establish status. Assume running.
else:
if p.exitStatus != -3:
return false
else:
var ret : int
var status : cint = 1
ret = waitpid(p.id, status, WNOHANG)
if ret == int(p.id):
if isExitStatus(status):
p.exitStatus = status
return false
else:
return true
elif ret == 0:
return true # Can't establish status. Assume running.
else:
raiseOSError(osLastError())
proc terminate(p: Process) =
if kill(p.id, SIGTERM) != 0'i32:
@ -1152,16 +1184,19 @@ elif not defined(useNimRtl):
stream = newFileStream(f)
proc inputStream(p: Process): Stream =
streamAccess(p)
if p.inStream == nil:
createStream(p.inStream, p.inHandle, fmWrite)
return p.inStream
proc outputStream(p: Process): Stream =
streamAccess(p)
if p.outStream == nil:
createStream(p.outStream, p.outHandle, fmRead)
return p.outStream
proc errorStream(p: Process): Stream =
streamAccess(p)
if p.errStream == nil:
createStream(p.errStream, p.errHandle, fmRead)
return p.errStream

5
lib/pure/parsesql.nim
View file

@ -956,6 +956,7 @@ proc parseInsert(p: var SqlParser): SqlNode =
if p.tok.kind == tkParLe:
var n = newNode(nkColumnList)
parseParIdentList(p, n)
result.add n
else:
result.add(nil)
if isKeyw(p, "default"):
@ -1160,7 +1161,7 @@ proc ra(n: SqlNode, s: var string, indent: int) =
else:
s.add("\"" & replace(n.strVal, "\"", "\"\"") & "\"")
of nkStringLit:
s.add(escape(n.strVal, "e'", "'"))
s.add(escape(n.strVal, "'", "'"))
of nkBitStringLit:
s.add("b'" & n.strVal & "'")
of nkHexStringLit:
@ -1240,7 +1241,7 @@ proc ra(n: SqlNode, s: var string, indent: int) =
if n.sons[2].kind == nkDefault:
s.add("default values")
else:
s.add("\nvalues ")
s.add("\n")
ra(n.sons[2], s, indent)
s.add(';')
of nkUpdate:

2
lib/pure/selectors.nim
View file

@ -310,6 +310,8 @@ else:
include ioselects/ioselectors_select
elif defined(solaris):
include ioselects/ioselectors_poll # need to replace it with event ports
elif defined(genode):
include ioselects/ioselectors_select # TODO: use the native VFS layer
else:
include ioselects/ioselectors_poll

253
lib/pure/strutils.nim
View file

@ -32,10 +32,6 @@ when defined(nimOldSplit):
else:
{.pragma: deprecatedSplit.}
type
CharSet* {.deprecated.} = set[char] # for compatibility with Nim
{.deprecated: [TCharSet: CharSet].}
const
Whitespace* = {' ', '\t', '\v', '\r', '\l', '\f'}
## All the characters that count as whitespace.
@ -78,40 +74,40 @@ proc isAlphaAscii*(c: char): bool {.noSideEffect, procvar,
return c in Letters
proc isAlphaNumeric*(c: char): bool {.noSideEffect, procvar,
rtl, extern: "nsuIsAlphaNumericChar".}=
rtl, extern: "nsuIsAlphaNumericChar".} =
## Checks whether or not `c` is alphanumeric.
##
## This checks a-z, A-Z, 0-9 ASCII characters only.
return c in Letters or c in Digits
return c in Letters+Digits
proc isDigit*(c: char): bool {.noSideEffect, procvar,
rtl, extern: "nsuIsDigitChar".}=
rtl, extern: "nsuIsDigitChar".} =
## Checks whether or not `c` is a number.
##
## This checks 0-9 ASCII characters only.
return c in Digits
proc isSpaceAscii*(c: char): bool {.noSideEffect, procvar,
rtl, extern: "nsuIsSpaceAsciiChar".}=
rtl, extern: "nsuIsSpaceAsciiChar".} =
## Checks whether or not `c` is a whitespace character.
return c in Whitespace
proc isLowerAscii*(c: char): bool {.noSideEffect, procvar,
rtl, extern: "nsuIsLowerAsciiChar".}=
rtl, extern: "nsuIsLowerAsciiChar".} =
## Checks whether or not `c` is a lower case character.
##
## This checks ASCII characters only.
return c in {'a'..'z'}
proc isUpperAscii*(c: char): bool {.noSideEffect, procvar,
rtl, extern: "nsuIsUpperAsciiChar".}=
rtl, extern: "nsuIsUpperAsciiChar".} =
## Checks whether or not `c` is an upper case character.
##
## This checks ASCII characters only.
return c in {'A'..'Z'}
proc isAlphaAscii*(s: string): bool {.noSideEffect, procvar,
rtl, extern: "nsuIsAlphaAsciiStr".}=
rtl, extern: "nsuIsAlphaAsciiStr".} =
## Checks whether or not `s` is alphabetical.
##
## This checks a-z, A-Z ASCII characters only.
@ -123,10 +119,10 @@ proc isAlphaAscii*(s: string): bool {.noSideEffect, procvar,
result = true
for c in s:
result = c.isAlphaAscii() and result
if not c.isAlphaAscii(): return false
proc isAlphaNumeric*(s: string): bool {.noSideEffect, procvar,
rtl, extern: "nsuIsAlphaNumericStr".}=
rtl, extern: "nsuIsAlphaNumericStr".} =
## Checks whether or not `s` is alphanumeric.
##
## This checks a-z, A-Z, 0-9 ASCII characters only.
@ -142,7 +138,7 @@ proc isAlphaNumeric*(s: string): bool {.noSideEffect, procvar,
return false
proc isDigit*(s: string): bool {.noSideEffect, procvar,
rtl, extern: "nsuIsDigitStr".}=
rtl, extern: "nsuIsDigitStr".} =
## Checks whether or not `s` is a numeric value.
##
## This checks 0-9 ASCII characters only.
@ -158,7 +154,7 @@ proc isDigit*(s: string): bool {.noSideEffect, procvar,
return false
proc isSpaceAscii*(s: string): bool {.noSideEffect, procvar,
rtl, extern: "nsuIsSpaceAsciiStr".}=
rtl, extern: "nsuIsSpaceAsciiStr".} =
## Checks whether or not `s` is completely whitespace.
##
## Returns true if all characters in `s` are whitespace
@ -172,7 +168,7 @@ proc isSpaceAscii*(s: string): bool {.noSideEffect, procvar,
return false
proc isLowerAscii*(s: string): bool {.noSideEffect, procvar,
rtl, extern: "nsuIsLowerAsciiStr".}=
rtl, extern: "nsuIsLowerAsciiStr".} =
## Checks whether or not `s` contains all lower case characters.
##
## This checks ASCII characters only.
@ -187,7 +183,7 @@ proc isLowerAscii*(s: string): bool {.noSideEffect, procvar,
true
proc isUpperAscii*(s: string): bool {.noSideEffect, procvar,
rtl, extern: "nsuIsUpperAsciiStr".}=
rtl, extern: "nsuIsUpperAsciiStr".} =
## Checks whether or not `s` contains all upper case characters.
##
## This checks ASCII characters only.
@ -506,16 +502,15 @@ template splitCommon(s, sep, maxsplit, sepLen) =
var last = 0
var splits = maxsplit
if len(s) > 0:
while last <= len(s):
var first = last
while last < len(s) and not stringHasSep(s, last, sep):
inc(last)
if splits == 0: last = len(s)
yield substr(s, first, last-1)
if splits == 0: break
dec(splits)
inc(last, sepLen)
while last <= len(s):
var first = last
while last < len(s) and not stringHasSep(s, last, sep):
inc(last)
if splits == 0: last = len(s)
yield substr(s, first, last-1)
if splits == 0: break
dec(splits)
inc(last, sepLen)
template oldSplit(s, seps, maxsplit) =
var last = 0
@ -673,30 +668,29 @@ template rsplitCommon(s, sep, maxsplit, sepLen) =
splits = maxsplit
startPos = 0
if len(s) > 0:
# go to -1 in order to get separators at the beginning
while first >= -1:
while first >= 0 and not stringHasSep(s, first, sep):
dec(first)
if splits == 0:
# No more splits means set first to the beginning
first = -1
if first == -1:
startPos = 0
else:
startPos = first + sepLen
yield substr(s, startPos, last)
if splits == 0:
break
dec(splits)
# go to -1 in order to get separators at the beginning
while first >= -1:
while first >= 0 and not stringHasSep(s, first, sep):
dec(first)
last = first
if splits == 0:
# No more splits means set first to the beginning
first = -1
if first == -1:
startPos = 0
else:
startPos = first + sepLen
yield substr(s, startPos, last)
if splits == 0:
break
dec(splits)
dec(first)
last = first
iterator rsplit*(s: string, seps: set[char] = Whitespace,
maxsplit: int = -1): string =
@ -824,12 +818,18 @@ proc split*(s: string, seps: set[char] = Whitespace, maxsplit: int = -1): seq[st
noSideEffect, rtl, extern: "nsuSplitCharSet".} =
## The same as the `split iterator <#split.i,string,set[char],int>`_, but is a
## proc that returns a sequence of substrings.
runnableExamples:
doAssert "a,b;c".split({',', ';'}) == @["a", "b", "c"]
doAssert "".split({' '}) == @[""]
accumulateResult(split(s, seps, maxsplit))
proc split*(s: string, sep: char, maxsplit: int = -1): seq[string] {.noSideEffect,
rtl, extern: "nsuSplitChar".} =
## The same as the `split iterator <#split.i,string,char,int>`_, but is a proc
## that returns a sequence of substrings.
runnableExamples:
doAssert "a,b,c".split(',') == @["a", "b", "c"]
doAssert "".split(' ') == @[""]
accumulateResult(split(s, sep, maxsplit))
proc split*(s: string, sep: string, maxsplit: int = -1): seq[string] {.noSideEffect,
@ -838,6 +838,13 @@ proc split*(s: string, sep: string, maxsplit: int = -1): seq[string] {.noSideEff
##
## Substrings are separated by the string `sep`. This is a wrapper around the
## `split iterator <#split.i,string,string,int>`_.
runnableExamples:
doAssert "a,b,c".split(",") == @["a", "b", "c"]
doAssert "a man a plan a canal panama".split("a ") == @["", "man ", "plan ", "canal panama"]
doAssert "".split("Elon Musk") == @[""]
doAssert "a largely spaced sentence".split(" ") == @["a", "", "largely", "", "", "", "spaced", "sentence"]
doAssert "a largely spaced sentence".split(" ", maxsplit=1) == @["a", " largely spaced sentence"]
doAssert(sep.len > 0)
accumulateResult(split(s, sep, maxsplit))
@ -906,6 +913,13 @@ proc rsplit*(s: string, sep: string, maxsplit: int = -1): seq[string]
## .. code-block:: nim
## @["Root#Object#Method", "Index"]
##
runnableExamples:
doAssert "a largely spaced sentence".rsplit(" ", maxsplit=1) == @["a largely spaced", "sentence"]
doAssert "a,b,c".rsplit(",") == @["a", "b", "c"]
doAssert "a man a plan a canal panama".rsplit("a ") == @["", "man ", "plan ", "canal panama"]
doAssert "".rsplit("Elon Musk") == @[""]
doAssert "a largely spaced sentence".rsplit(" ") == @["a", "", "largely", "", "", "", "spaced", "sentence"]
accumulateResult(rsplit(s, sep, maxsplit))
result.reverse()
@ -1305,14 +1319,13 @@ proc addSep*(dest: var string, sep = ", ", startLen: Natural = 0)
## This is often useful for generating some code where the items need to
## be *separated* by `sep`. `sep` is only added if `dest` is longer than
## `startLen`. The following example creates a string describing
## an array of integers:
##
## .. code-block:: nim
## var arr = "["
## for x in items([2, 3, 5, 7, 11]):
## addSep(arr, startLen=len("["))
## add(arr, $x)
## add(arr, "]")
## an array of integers.
runnableExamples:
var arr = "["
for x in items([2, 3, 5, 7, 11]):
addSep(arr, startLen=len("["))
add(arr, $x)
add(arr, "]")
if dest.len > startLen: add(dest, sep)
proc allCharsInSet*(s: string, theSet: set[char]): bool =
@ -1730,7 +1743,9 @@ proc insertSep*(s: string, sep = '_', digits = 3): string {.noSideEffect,
##
## Even though the algorithm works with any string `s`, it is only useful
## if `s` contains a number.
## Example: ``insertSep("1000000") == "1_000_000"``
runnableExamples:
doAssert insertSep("1000000") == "1_000_000"
var L = (s.len-1) div digits + s.len
result = newString(L)
var j = 0
@ -1818,6 +1833,8 @@ proc validIdentifier*(s: string): bool {.noSideEffect,
##
## A valid identifier starts with a character of the set `IdentStartChars`
## and is followed by any number of characters of the set `IdentChars`.
runnableExamples:
doAssert "abc_def08".validIdentifier
if s[0] in IdentStartChars:
for i in 1..s.len-1:
if s[i] notin IdentChars: return false
@ -1828,7 +1845,7 @@ proc editDistance*(a, b: string): int {.noSideEffect,
## Returns the edit distance between `a` and `b`.
##
## This uses the `Levenshtein`:idx: distance algorithm with only a linear
## memory overhead. This implementation is highly optimized!
## memory overhead.
var len1 = a.len
var len2 = b.len
if len1 > len2:
@ -2007,16 +2024,11 @@ proc formatFloat*(f: float, format: FloatFormatMode = ffDefault,
## after the decimal point for Nim's ``float`` type.
##
## If ``precision == -1``, it tries to format it nicely.
##
## Examples:
##
## .. code-block:: nim
##
## let x = 123.456
## doAssert x.formatFloat() == "123.4560000000000"
## doAssert x.formatFloat(ffDecimal, 4) == "123.4560"
## doAssert x.formatFloat(ffScientific, 2) == "1.23e+02"
##
runnableExamples:
let x = 123.456
doAssert x.formatFloat() == "123.4560000000000"
doAssert x.formatFloat(ffDecimal, 4) == "123.4560"
doAssert x.formatFloat(ffScientific, 2) == "1.23e+02"
result = formatBiggestFloat(f, format, precision, decimalSep)
proc trimZeros*(x: var string) {.noSideEffect.} =
@ -2051,18 +2063,13 @@ proc formatSize*(bytes: int64,
##
## `includeSpace` can be set to true to include the (SI preferred) space
## between the number and the unit (e.g. 1 KiB).
##
## Examples:
##
## .. code-block:: nim
##
## formatSize((1'i64 shl 31) + (300'i64 shl 20)) == "2.293GiB"
## formatSize((2.234*1024*1024).int) == "2.234MiB"
## formatSize(4096, includeSpace=true) == "4 KiB"
## formatSize(4096, prefix=bpColloquial, includeSpace=true) == "4 kB"
## formatSize(4096) == "4KiB"
## formatSize(5_378_934, prefix=bpColloquial, decimalSep=',') == "5,13MB"
##
runnableExamples:
doAssert formatSize((1'i64 shl 31) + (300'i64 shl 20)) == "2.293GiB"
doAssert formatSize((2.234*1024*1024).int) == "2.234MiB"
doAssert formatSize(4096, includeSpace=true) == "4 KiB"
doAssert formatSize(4096, prefix=bpColloquial, includeSpace=true) == "4 kB"
doAssert formatSize(4096) == "4KiB"
doAssert formatSize(5_378_934, prefix=bpColloquial, decimalSep=',') == "5,13MB"
const iecPrefixes = ["", "Ki", "Mi", "Gi", "Ti", "Pi", "Ei", "Zi", "Yi"]
const collPrefixes = ["", "k", "M", "G", "T", "P", "E", "Z", "Y"]
var
@ -2156,7 +2163,7 @@ proc formatEng*(f: BiggestFloat,
## formatEng(4100, unit="V") == "4.1e3 V"
## formatEng(4100, unit="") == "4.1e3 " # Space with unit=""
##
## `decimalSep` is used as the decimal separator
## `decimalSep` is used as the decimal separator.
var
absolute: BiggestFloat
significand: BiggestFloat
@ -2369,17 +2376,16 @@ proc removeSuffix*(s: var string, chars: set[char] = Newlines) {.
rtl, extern: "nsuRemoveSuffixCharSet".} =
## Removes all characters from `chars` from the end of the string `s`
## (in-place).
##
## .. code-block:: nim
## var userInput = "Hello World!*~\r\n"
## userInput.removeSuffix
## doAssert userInput == "Hello World!*~"
## userInput.removeSuffix({'~', '*'})
## doAssert userInput == "Hello World!"
##
## var otherInput = "Hello!?!"
## otherInput.removeSuffix({'!', '?'})
## doAssert otherInput == "Hello"
runnableExamples:
var userInput = "Hello World!*~\r\n"
userInput.removeSuffix
doAssert userInput == "Hello World!*~"
userInput.removeSuffix({'~', '*'})
doAssert userInput == "Hello World!"
var otherInput = "Hello!?!"
otherInput.removeSuffix({'!', '?'})
doAssert otherInput == "Hello"
if s.len == 0: return
var last = s.high
while last > -1 and s[last] in chars: last -= 1
@ -2390,24 +2396,23 @@ proc removeSuffix*(s: var string, c: char) {.
## Removes all occurrences of a single character (in-place) from the end
## of a string.
##
## .. code-block:: nim
## var table = "users"
## table.removeSuffix('s')
## doAssert table == "user"
##
## var dots = "Trailing dots......."
## dots.removeSuffix('.')
## doAssert dots == "Trailing dots"
runnableExamples:
var table = "users"
table.removeSuffix('s')
doAssert table == "user"
var dots = "Trailing dots......."
dots.removeSuffix('.')
doAssert dots == "Trailing dots"
removeSuffix(s, chars = {c})
proc removeSuffix*(s: var string, suffix: string) {.
rtl, extern: "nsuRemoveSuffixString".} =
## Remove the first matching suffix (in-place) from a string.
##
## .. code-block:: nim
## var answers = "yeses"
## answers.removeSuffix("es")
## doAssert answers == "yes"
runnableExamples:
var answers = "yeses"
answers.removeSuffix("es")
doAssert answers == "yes"
var newLen = s.len
if s.endsWith(suffix):
newLen -= len(suffix)
@ -2418,16 +2423,16 @@ proc removePrefix*(s: var string, chars: set[char] = Newlines) {.
## Removes all characters from `chars` from the start of the string `s`
## (in-place).
##
## .. code-block:: nim
## var userInput = "\r\n*~Hello World!"
## userInput.removePrefix
## doAssert userInput == "*~Hello World!"
## userInput.removePrefix({'~', '*'})
## doAssert userInput == "Hello World!"
##
## var otherInput = "?!?Hello!?!"
## otherInput.removePrefix({'!', '?'})
## doAssert otherInput == "Hello!?!"
runnableExamples:
var userInput = "\r\n*~Hello World!"
userInput.removePrefix
doAssert userInput == "*~Hello World!"
userInput.removePrefix({'~', '*'})
doAssert userInput == "Hello World!"
var otherInput = "?!?Hello!?!"
otherInput.removePrefix({'!', '?'})
doAssert otherInput == "Hello!?!"
var start = 0
while start < s.len and s[start] in chars: start += 1
if start > 0: s.delete(0, start - 1)
@ -2437,20 +2442,20 @@ proc removePrefix*(s: var string, c: char) {.
## Removes all occurrences of a single character (in-place) from the start
## of a string.
##
## .. code-block:: nim
## var ident = "pControl"
## ident.removePrefix('p')
## doAssert ident == "Control"
runnableExamples:
var ident = "pControl"
ident.removePrefix('p')
doAssert ident == "Control"
removePrefix(s, chars = {c})
proc removePrefix*(s: var string, prefix: string) {.
rtl, extern: "nsuRemovePrefixString".} =
## Remove the first matching prefix (in-place) from a string.
##
## .. code-block:: nim
## var answers = "yesyes"
## answers.removePrefix("yes")
## doAssert answers == "yes"
runnableExamples:
var answers = "yesyes"
answers.removePrefix("yes")
doAssert answers == "yes"
if s.startsWith(prefix):
s.delete(0, prefix.len - 1)

12
lib/pure/unicode.nim
View file

@ -293,33 +293,33 @@ proc runeSubStr*(s: string, pos:int, len:int = int.high): string =
if pos < 0:
let (o, rl) = runeReverseOffset(s, -pos)
if len >= rl:
result = s[o.. s.len-1]
result = s.substr(o, s.len-1)
elif len < 0:
let e = rl + len
if e < 0:
result = ""
else:
result = s[o.. runeOffset(s, e-(rl+pos) , o)-1]
result = s.substr(o, runeOffset(s, e-(rl+pos) , o)-1)
else:
result = s[o.. runeOffset(s, len, o)-1]
result = s.substr(o, runeOffset(s, len, o)-1)
else:
let o = runeOffset(s, pos)
if o < 0:
result = ""
elif len == int.high:
result = s[o.. s.len-1]
result = s.substr(o, s.len-1)
elif len < 0:
let (e, rl) = runeReverseOffset(s, -len)
discard rl
if e <= 0:
result = ""
else:
result = s[o.. e-1]
result = s.substr(o, e-1)
else:
var e = runeOffset(s, len, o)
if e < 0:
e = s.len
result = s[o.. e-1]
result = s.substr(o, e-1)
const
alphaRanges = [

48
lib/pure/uri.nim
View file

@ -47,6 +47,49 @@ proc add*(url: var Url, a: Url) {.deprecated.} =
url = url / a
{.pop.}
proc encodeUrl*(s: string): string =
## Encodes a value to be HTTP safe: This means that characters in the set
## ``{'A'..'Z', 'a'..'z', '0'..'9', '_'}`` are carried over to the result,
## a space is converted to ``'+'`` and every other character is encoded as
## ``'%xx'`` where ``xx`` denotes its hexadecimal value.
result = newStringOfCap(s.len + s.len shr 2) # assume 12% non-alnum-chars
for i in 0..s.len-1:
case s[i]
of 'a'..'z', 'A'..'Z', '0'..'9', '_': add(result, s[i])
of ' ': add(result, '+')
else:
add(result, '%')
add(result, toHex(ord(s[i]), 2))
proc decodeUrl*(s: string): string =
## Decodes a value from its HTTP representation: This means that a ``'+'``
## is converted to a space, ``'%xx'`` (where ``xx`` denotes a hexadecimal
## value) is converted to the character with ordinal number ``xx``, and
## and every other character is carried over.
proc handleHexChar(c: char, x: var int) {.inline.} =
case c
of '0'..'9': x = (x shl 4) or (ord(c) - ord('0'))
of 'a'..'f': x = (x shl 4) or (ord(c) - ord('a') + 10)
of 'A'..'F': x = (x shl 4) or (ord(c) - ord('A') + 10)
else: assert(false)
result = newString(s.len)
var i = 0
var j = 0
while i < s.len:
case s[i]
of '%':
var x = 0
handleHexChar(s[i+1], x)
handleHexChar(s[i+2], x)
inc(i, 2)
result[j] = chr(x)
of '+': result[j] = ' '
else: result[j] = s[i]
inc(i)
inc(j)
setLen(result, j)
proc parseAuthority(authority: string, result: var Uri) =
var i = 0
var inPort = false
@ -327,6 +370,11 @@ proc `$`*(u: Uri): string =
result.add(u.anchor)
when isMainModule:
block:
const test1 = "abc\L+def xyz"
doAssert encodeUrl(test1) == "abc%0A%2Bdef+xyz"
doAssert decodeUrl(encodeUrl(test1)) == test1
block:
let str = "http://localhost"
let test = parseUri(str)

47
lib/system.nim
View file

@ -1439,7 +1439,11 @@ const
## is the value that should be passed to `quit <#quit>`_ to indicate
## failure.
var programResult* {.exportc: "nim_program_result".}: int
when defined(nodejs):
var programResult* {.importc: "process.exitCode".}: int
programResult = 0
else:
var programResult* {.exportc: "nim_program_result".}: int
## modify this variable to specify the exit code of the program
## under normal circumstances. When the program is terminated
## prematurely using ``quit``, this value is ignored.
@ -3525,7 +3529,10 @@ when hasAlloc or defined(nimscript):
## .. code-block:: nim
## var s = "abcdef"
## assert s[1..3] == "bcd"
result = s.substr(s ^^ x.a, s ^^ x.b)
let a = s ^^ x.a
let L = (s ^^ x.b) - a + 1
result = newString(L)
for i in 0 ..< L: result[i] = s[i + a]
proc `[]=`*[T, U](s: var string, x: HSlice[T, U], b: string) =
## slice assignment for strings. If
@ -3752,6 +3759,7 @@ template assert*(cond: bool, msg = "") =
## that ``AssertionError`` is hidden from the effect system, so it doesn't
## produce ``{.raises: [AssertionError].}``. This exception is only supposed
## to be caught by unit testing frameworks.
##
## The compiler may not generate any code at all for ``assert`` if it is
## advised to do so through the ``-d:release`` or ``--assertions:off``
## `command line switches <nimc.html#command-line-switches>`_.
@ -3992,3 +4000,38 @@ when defined(windows) and appType == "console" and defined(nimSetUtf8CodePage):
proc setConsoleOutputCP(codepage: cint): cint {.stdcall, dynlib: "kernel32",
importc: "SetConsoleOutputCP".}
discard setConsoleOutputCP(65001) # 65001 - utf-8 codepage
when defined(nimHasRunnableExamples):
proc runnableExamples*(body: untyped) {.magic: "RunnableExamples".}
## A section you should use to mark `runnable example`:idx: code with.
##
## - In normal debug and release builds code within
## a ``runnableExamples`` section is ignored.
## - The documentation generator is aware of these examples and considers them
## part of the ``##`` doc comment. As the last step of documentation
## generation the examples are put into an ``$file_example.nim`` file,
## compiled and tested. The collected examples are
## put into their own module to ensure the examples do not refer to
## non-exported symbols.
else:
template runnableExamples*(body: untyped) =
discard
template doAssertRaises*(exception, code: untyped): typed =
## Raises ``AssertionError`` if specified ``code`` does not raise the
## specified exception.
runnableExamples:
doAssertRaises(ValueError):
raise newException(ValueError, "Hello World")
try:
block:
code
raiseAssert(astToStr(exception) & " wasn't raised by:\n" & astToStr(code))
except exception:
discard
except Exception as exc:
raiseAssert(astToStr(exception) &
" wasn't raised, another error was raised instead by:\n"&
astToStr(code))

171
lib/system/alloc.nim
View file

@ -8,8 +8,6 @@
#
# Low level allocator for Nim. Has been designed to support the GC.
# TODO:
# - make searching for block O(1)
{.push profiler:off.}
include osalloc
@ -19,14 +17,17 @@ template track(op, address, size) =
memTrackerOp(op, address, size)
# We manage *chunks* of memory. Each chunk is a multiple of the page size.
# Each chunk starts at an address that is divisible by the page size. Chunks
# that are bigger than ``ChunkOsReturn`` are returned back to the operating
# system immediately.
# Each chunk starts at an address that is divisible by the page size.
const
ChunkOsReturn = 256 * PageSize # 1 MB
InitialMemoryRequest = ChunkOsReturn div 2 # < ChunkOsReturn!
InitialMemoryRequest = 128 * PageSize # 0.5 MB
SmallChunkSize = PageSize
MaxFli = 30
MaxLog2Sli = 5 # 32, this cannot be increased without changing 'uint32'
# everywhere!
MaxSli = 1 shl MaxLog2Sli
FliOffset = 6
RealFli = MaxFli - FliOffset
type
PTrunk = ptr Trunk
@ -99,10 +100,12 @@ type
MemRegion = object
minLargeObj, maxLargeObj: int
freeSmallChunks: array[0..SmallChunkSize div MemAlign-1, PSmallChunk]
flBitmap: uint32
slBitmap: array[RealFli, uint32]
matrix: array[RealFli, array[MaxSli, PBigChunk]]
llmem: PLLChunk
currMem, maxMem, freeMem: int # memory sizes (allocated from OS)
lastSize: int # needed for the case that OS gives us pages linearly
freeChunksList: PBigChunk # XXX make this a datastructure with O(1) access
chunkStarts: IntSet
root, deleted, last, freeAvlNodes: PAvlNode
locked, blockChunkSizeIncrease: bool # if locked, we cannot free pages.
@ -110,7 +113,109 @@ type
bottomData: AvlNode
heapLinks: HeapLinks
{.deprecated: [TMemRegion: MemRegion].}
const
fsLookupTable: array[byte, int8] = [
-1'i8, 0, 1, 1, 2, 2, 2, 2, 3, 3, 3, 3, 3, 3, 3, 3,
4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5, 5,
5, 5, 5, 5, 5, 5, 5, 5,
6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,
6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,
6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6, 6,
7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,
7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,
7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,
7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,
7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7,
7, 7, 7, 7, 7, 7, 7, 7
]
proc msbit(x: uint32): int {.inline.} =
let a = if x <= 0xff_ff:
(if x <= 0xff: 0 else: 8)
else:
(if x <= 0xff_ff_ff: 16 else: 24)
result = int(fsLookupTable[byte(x shr a)]) + a
proc lsbit(x: uint32): int {.inline.} =
msbit(x and ((not x) + 1))
proc setBit(nr: int; dest: var uint32) {.inline.} =
dest = dest or (1u32 shl (nr and 0x1f))
proc clearBit(nr: int; dest: var uint32) {.inline.} =
dest = dest and not (1u32 shl (nr and 0x1f))
proc mappingSearch(r, fl, sl: var int) {.inline.} =
#let t = (1 shl (msbit(uint32 r) - MaxLog2Sli)) - 1
# This diverges from the standard TLSF algorithm because we need to ensure
# PageSize alignment:
let t = roundup((1 shl (msbit(uint32 r) - MaxLog2Sli)), PageSize) - 1
r = r + t
fl = msbit(uint32 r)
sl = (r shr (fl - MaxLog2Sli)) - MaxSli
dec fl, FliOffset
r = r and not t
sysAssert((r and PageMask) == 0, "mappingSearch: still not aligned")
# See http://www.gii.upv.es/tlsf/files/papers/tlsf_desc.pdf for details of
# this algorithm.
proc mappingInsert(r: int): tuple[fl, sl: int] {.inline.} =
sysAssert((r and PageMask) == 0, "mappingInsert: still not aligned")
result.fl = msbit(uint32 r)
result.sl = (r shr (result.fl - MaxLog2Sli)) - MaxSli
dec result.fl, FliOffset
template mat(): untyped = a.matrix[fl][sl]
proc findSuitableBlock(a: MemRegion; fl, sl: var int): PBigChunk {.inline.} =
let tmp = a.slBitmap[fl] and (not 0u32 shl sl)
result = nil
if tmp != 0:
sl = lsbit(tmp)
result = mat()
else:
fl = lsbit(a.flBitmap and (not 0u32 shl (fl + 1)))
if fl > 0:
sl = lsbit(a.slBitmap[fl])
result = mat()
template clearBits(sl, fl) =
clearBit(sl, a.slBitmap[fl])
if a.slBitmap[fl] == 0u32:
# do not forget to cascade:
clearBit(fl, a.flBitmap)
proc removeChunkFromMatrix(a: var MemRegion; b: PBigChunk) =
let (fl, sl) = mappingInsert(b.size)
if b.next != nil: b.next.prev = b.prev
if b.prev != nil: b.prev.next = b.next
if mat() == b:
mat() = b.next
if mat() == nil:
clearBits(sl, fl)
b.prev = nil
b.next = nil
proc removeChunkFromMatrix2(a: var MemRegion; b: PBigChunk; fl, sl: int) =
mat() = b.next
if mat() != nil:
mat().prev = nil
else:
clearBits(sl, fl)
b.prev = nil
b.next = nil
proc addChunkToMatrix(a: var MemRegion; b: PBigChunk) =
let (fl, sl) = mappingInsert(b.size)
b.prev = nil
b.next = mat()
if mat() != nil:
mat().prev = b
mat() = b
setBit(sl, a.slBitmap[fl])
setBit(fl, a.flBitmap)
{.push stack_trace: off.}
proc initAllocator() = discard "nothing to do anymore"
@ -203,6 +308,7 @@ proc llDeallocAll(a: var MemRegion) =
var next = it.next
osDeallocPages(it, PageSize)
it = next
a.llmem = nil
proc intSetGet(t: IntSet, key: int): PTrunk =
var it = t.data[key and high(t.data)]
@ -369,6 +475,7 @@ proc requestOsChunks(a: var MemRegion, size: int): PBigChunk =
result.prevSize = 0 or (result.prevSize and 1) # unknown
# but do not overwrite 'used' field
a.lastSize = size # for next request
sysAssert((cast[int](result) and PageMask) == 0, "requestOschunks: unaligned chunk")
proc isAccessible(a: MemRegion, p: pointer): bool {.inline.} =
result = contains(a.chunkStarts, pageIndex(p))
@ -419,7 +526,7 @@ proc freeBigChunk(a: var MemRegion, c: PBigChunk) =
if isAccessible(a, ri) and chunkUnused(ri):
sysAssert(not isSmallChunk(ri), "freeBigChunk 3")
if not isSmallChunk(ri):
listRemove(a.freeChunksList, cast[PBigChunk](ri))
removeChunkFromMatrix(a, cast[PBigChunk](ri))
inc(c.size, ri.size)
excl(a.chunkStarts, pageIndex(ri))
when coalescLeft:
@ -430,49 +537,44 @@ proc freeBigChunk(a: var MemRegion, c: PBigChunk) =
if isAccessible(a, le) and chunkUnused(le):
sysAssert(not isSmallChunk(le), "freeBigChunk 5")
if not isSmallChunk(le):
listRemove(a.freeChunksList, cast[PBigChunk](le))
removeChunkFromMatrix(a, cast[PBigChunk](le))
inc(le.size, c.size)
excl(a.chunkStarts, pageIndex(c))
c = cast[PBigChunk](le)
incl(a, a.chunkStarts, pageIndex(c))
updatePrevSize(a, c, c.size)
listAdd(a.freeChunksList, c)
addChunkToMatrix(a, c)
# set 'used' to false:
c.prevSize = c.prevSize and not 1
proc splitChunk(a: var MemRegion, c: PBigChunk, size: int) =
var rest = cast[PBigChunk](cast[ByteAddress](c) +% size)
sysAssert(rest notin a.freeChunksList, "splitChunk")
rest.size = c.size - size
track("rest.origSize", addr rest.origSize, sizeof(int))
# XXX check if these two nil assignments are dead code given
# addChunkToMatrix's implementation:
rest.next = nil
rest.prev = nil
# size and not used
# size and not used:
rest.prevSize = size
sysAssert((size and 1) == 0, "splitChunk 2")
sysAssert((size and PageMask) == 0,
"splitChunk: size is not a multiple of the PageSize")
updatePrevSize(a, c, rest.size)
c.size = size
incl(a, a.chunkStarts, pageIndex(rest))
listAdd(a.freeChunksList, rest)
addChunkToMatrix(a, rest)
proc getBigChunk(a: var MemRegion, size: int): PBigChunk =
# use first fit for now:
sysAssert((size and PageMask) == 0, "getBigChunk 1")
sysAssert(size > 0, "getBigChunk 2")
result = a.freeChunksList
block search:
while result != nil:
sysAssert chunkUnused(result), "getBigChunk 3"
if result.size == size:
listRemove(a.freeChunksList, result)
break search
elif result.size > size:
listRemove(a.freeChunksList, result)
splitChunk(a, result, size)
break search
result = result.next
sysAssert result != a.freeChunksList, "getBigChunk 4"
var size = size # roundup(size, PageSize)
var fl, sl: int
mappingSearch(size, fl, sl)
sysAssert((size and PageMask) == 0, "getBigChunk: unaligned chunk")
result = findSuitableBlock(a, fl, sl)
if result == nil:
if size < InitialMemoryRequest:
result = requestOsChunks(a, InitialMemoryRequest)
splitChunk(a, result, size)
@ -481,7 +583,10 @@ proc getBigChunk(a: var MemRegion, size: int): PBigChunk =
# if we over allocated split the chunk:
if result.size > size:
splitChunk(a, result, size)
else:
removeChunkFromMatrix2(a, result, fl, sl)
if result.size >= size + PageSize:
splitChunk(a, result, size)
# set 'used' to to true:
result.prevSize = 1
track("setUsedToFalse", addr result.origSize, sizeof(int))
@ -572,14 +677,14 @@ proc rawAlloc(a: var MemRegion, requestedSize: int): pointer =
size == 0, "rawAlloc 21")
sysAssert(allocInv(a), "rawAlloc: end small size")
else:
size = roundup(requestedSize+bigChunkOverhead(), PageSize)
size = requestedSize + bigChunkOverhead() # roundup(requestedSize+bigChunkOverhead(), PageSize)
# allocate a large block
var c = getBigChunk(a, size)
sysAssert c.prev == nil, "rawAlloc 10"
sysAssert c.next == nil, "rawAlloc 11"
sysAssert c.size == size, "rawAlloc 12"
result = addr(c.data)
sysAssert((cast[ByteAddress](result) and (MemAlign-1)) == 0, "rawAlloc 13")
sysAssert((cast[ByteAddress](c) and (MemAlign-1)) == 0, "rawAlloc 13")
sysAssert((cast[ByteAddress](c) and PageMask) == 0, "rawAlloc: Not aligned on a page boundary")
if a.root == nil: a.root = getBottom(a)
add(a, a.root, cast[ByteAddress](result), cast[ByteAddress](result)+%size)
sysAssert(isAccessible(a, result), "rawAlloc 14")

13
lib/system/mmdisp.nim
View file

@ -42,7 +42,8 @@ type
# Page size of the system; in most cases 4096 bytes. For exotic OS or
# CPU this needs to be changed:
const
PageShift = when defined(cpu16): 8 else: 12
PageShift = when defined(cpu16): 8 else: 12 # \
# my tests showed no improvments for using larger page sizes.
PageSize = 1 shl PageShift
PageMask = PageSize-1
@ -343,7 +344,6 @@ elif defined(gogc):
const goFlagNoZero: uint32 = 1 shl 3
proc goRuntimeMallocGC(size: uint, typ: uint, flag: uint32): pointer {.importc: "runtime_mallocgc", dynlib: goLib.}
proc goFree(v: pointer) {.importc: "__go_free", dynlib: goLib.}
proc goSetFinalizer(obj: pointer, f: pointer) {.importc: "set_finalizer", codegenDecl:"$1 $2$3 __asm__ (\"main.Set_finalizer\");\n$1 $2$3", dynlib: goLib.}
@ -376,7 +376,6 @@ elif defined(gogc):
result = goRuntimeMallocGC(roundup(newsize, sizeof(pointer)).uint, 0.uint, goFlagNoZero)
copyMem(result, old, oldsize)
zeroMem(cast[pointer](cast[ByteAddress](result) +% oldsize), newsize - oldsize)
goFree(old)
proc nimGCref(p: pointer) {.compilerproc, inline.} = discard
proc nimGCunref(p: pointer) {.compilerproc, inline.} = discard
@ -573,3 +572,11 @@ when not declared(nimNewSeqOfCap):
cast[PGenericSeq](result).reserved = cap
{.pop.}
when not declared(ForeignCell):
type ForeignCell* = object
data*: pointer
proc protect*(x: pointer): ForeignCell = ForeignCell(data: x)
proc dispose*(x: ForeignCell) = discard
proc isNotForeign*(x: ForeignCell): bool = false

2
lib/system/osalloc.nim
View file

@ -166,7 +166,7 @@ elif defined(windows):
# space heavily, so we now treat Windows as a strange unmap target.
when reallyOsDealloc:
if virtualFree(p, 0, MEM_RELEASE) == 0:
cprintf "yes, failing!"
cprintf "virtualFree failing!"
quit 1
#VirtualFree(p, size, MEM_DECOMMIT)

11
lib/system/sysstr.nim
View file

@ -259,7 +259,7 @@ proc incrSeqV2(seq: PGenericSeq, elemSize: int): PGenericSeq {.compilerProc.} =
result.reserved = r
proc setLengthSeq(seq: PGenericSeq, elemSize, newLen: int): PGenericSeq {.
compilerRtl.} =
compilerRtl, inl.} =
result = seq
if result.space < newLen:
let r = max(resize(result.space), newLen)
@ -282,10 +282,11 @@ proc setLengthSeq(seq: PGenericSeq, elemSize, newLen: int): PGenericSeq {.
doDecRef(gch.tempStack.d[i], LocalHeap, MaybeCyclic)
gch.tempStack.len = len0
else:
for i in newLen..result.len-1:
forAllChildrenAux(cast[pointer](cast[ByteAddress](result) +%
GenericSeqSize +% (i*%elemSize)),
extGetCellType(result).base, waZctDecRef)
if ntfNoRefs notin extGetCellType(result).base.flags:
for i in newLen..result.len-1:
forAllChildrenAux(cast[pointer](cast[ByteAddress](result) +%
GenericSeqSize +% (i*%elemSize)),
extGetCellType(result).base, waZctDecRef)
# XXX: zeroing out the memory can still result in crashes if a wiped-out
# cell is aliased by another pointer (ie proc parameter or a let variable).

4
lib/system/threads.nim
View file

@ -255,9 +255,9 @@ when emulatedThreadVars:
proc nimThreadVarsSize(): int {.noconv, importc: "NimThreadVarsSize".}
# we preallocate a fixed size for thread local storage, so that no heap
# allocations are needed. Currently less than 7K are used on a 64bit machine.
# allocations are needed. Currently less than 16K are used on a 64bit machine.
# We use ``float`` for proper alignment:
const nimTlsSize {.intdefine.} = 8000
const nimTlsSize {.intdefine.} = 16000
type
ThreadLocalStorage = array[0..(nimTlsSize div sizeof(float)), float]

1
lib/windows/winlean.nim
View file

@ -541,6 +541,7 @@ var
SO_DONTLINGER* {.importc, header: "winsock2.h".}: cint
SO_EXCLUSIVEADDRUSE* {.importc, header: "winsock2.h".}: cint # disallow local address reuse
SO_ERROR* {.importc, header: "winsock2.h".}: cint
TCP_NODELAY* {.importc, header: "winsock2.h".}: cint
proc `==`*(x, y: SocketHandle): bool {.borrow.}