From 0d19de18cce2ba62e3c753e7474c29bbe4c47ad7 Mon Sep 17 00:00:00 2001 From: Araq Date: Wed, 28 Nov 2012 02:02:46 +0100 Subject: [PATCH 001/106] first steps for the 'export' feature --- compiler/ast.nim | 2 + compiler/parser.nim | 38 +++---- compiler/renderer.nim | 15 +++ lib/core/macros.nim | 10 +- todo.txt | 224 ++++++++++++++---------------------------- 5 files changed, 114 insertions(+), 175 deletions(-) diff --git a/compiler/ast.nim b/compiler/ast.nim index 027f4d003..77f8723e2 100755 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -173,7 +173,9 @@ type nkStmtList, # a list of statements nkImportStmt, # an import statement nkFromStmt, # a from * import statement + nkImportExceptStmt, # an import x except a statement nkIncludeStmt, # an include statement + nkExportStmt, # an export statement nkBindStmt, # a bind statement nkMixinStmt, # a mixin statement nkCommentStmt, # a comment statement diff --git a/compiler/parser.nim b/compiler/parser.nim index d5457fcd8..7ab87804e 100755 --- a/compiler/parser.nim +++ b/compiler/parser.nim @@ -901,27 +901,30 @@ proc parseExprStmt(p: var TParser): PNode = addSon(result, b) if b.kind == nkElse: break -proc parseImportOrIncludeStmt(p: var TParser, kind: TNodeKind): PNode = +proc parseModuleName(p: var TParser): PNode {.inline.} = + result = parseExpr(p) + +proc parseImportOrIncludeStmt(p: var TParser, kind: TNodeKind): PNode = var a: PNode result = newNodeP(kind, p) getTok(p) # skip `import` or `include` optInd(p, result) - while true: + while true: case p.tok.tokType - of tkEof, tkSad, tkDed: - break - of tkSymbol, tkAccent: + of tkEof, tkSad, tkDed: + break + of tkSymbol, tkAccent: a = parseSymbol(p) - of tkRStrLit: + of tkRStrLit: a = newStrNodeP(nkRStrLit, p.tok.literal, p) getTok(p) - of tkStrLit: + of tkStrLit: a = newStrNodeP(nkStrLit, p.tok.literal, p) getTok(p) - of tkTripleStrLit: + of tkTripleStrLit: a = newStrNodeP(nkTripleStrLit, p.tok.literal, p) getTok(p) - else: + else: parMessage(p, errIdentifierExpected, p.tok) break addSon(result, a) @@ -931,25 +934,10 @@ proc parseImportOrIncludeStmt(p: var TParser, kind: TNodeKind): PNode = expectNl(p) proc parseFromStmt(p: var TParser): PNode = - var a: PNode result = newNodeP(nkFromStmt, p) getTok(p) # skip `from` optInd(p, result) - case p.tok.tokType - of tkSymbol, tkAccent: - a = parseSymbol(p) - of tkRStrLit: - a = newStrNodeP(nkRStrLit, p.tok.literal, p) - getTok(p) - of tkStrLit: - a = newStrNodeP(nkStrLit, p.tok.literal, p) - getTok(p) - of tkTripleStrLit: - a = newStrNodeP(nkTripleStrLit, p.tok.literal, p) - getTok(p) - else: - parMessage(p, errIdentifierExpected, p.tok) - return + var a = parseModuleName(p) addSon(result, a) #optInd(p, a); eat(p, tkImport) optInd(p, result) diff --git a/compiler/renderer.nim b/compiler/renderer.nim index b9d522694..4444f2ab0 100755 --- a/compiler/renderer.nim +++ b/compiler/renderer.nim @@ -1148,6 +1148,14 @@ proc gsub(g: var TSrcGen, n: PNode, c: TContext) = putWithSpace(g, tkImport, "import") gcomma(g, n, emptyContext, 1) putNL(g) + of nkImportExceptStmt: + putWithSpace(g, tkImport, "import") + gsub(g, n.sons[0]) + put(g, tkSpaces, Space) + putWithSpace(g, tkExcept, "except") + gcommaAux(g, n, g.indent, 1) + gcoms(g) + putNL(g) of nkIncludeStmt: putWithSpace(g, tkInclude, "include") gcoms(g) @@ -1155,6 +1163,13 @@ proc gsub(g: var TSrcGen, n: PNode, c: TContext) = gcommaAux(g, n, g.indent) dedent(g) putNL(g) + of nkExportStmt: + putWithSpace(g, tkExport, "export") + gcoms(g) + indentNL(g) + gcommaAux(g, n, g.indent) + dedent(g) + putNL(g) of nkCommentStmt: gcoms(g) optNL(g) diff --git a/lib/core/macros.nim b/lib/core/macros.nim index 966a21a1b..d2d842025 100755 --- a/lib/core/macros.nim +++ b/lib/core/macros.nim @@ -45,8 +45,14 @@ type nnkConstDef, nnkTypeDef, nnkYieldStmt, nnkTryStmt, nnkFinally, nnkRaiseStmt, nnkReturnStmt, nnkBreakStmt, nnkContinueStmt, nnkBlockStmt, nnkStaticStmt, - nnkDiscardStmt, nnkStmtList, nnkImportStmt, nnkFromStmt, - nnkIncludeStmt, nnkBindStmt, nnkMixinStmt, + nnkDiscardStmt, nnkStmtList, + + nnkImportStmt, nnkFromStmt, + nkImportExceptStmt, + nnkIncludeStmt, + nnkExportStmt, + + nnkBindStmt, nnkMixinStmt, nnkCommentStmt, nnkStmtListExpr, nnkBlockExpr, nnkStmtListType, nnkBlockType, nnkTypeOfExpr, nnkObjectTy, nnkTupleTy, nnkRecList, nnkRecCase, nnkRecWhen, diff --git a/todo.txt b/todo.txt index 7e940c3e4..10b044a6d 100755 --- a/todo.txt +++ b/todo.txt @@ -1,9 +1,10 @@ version 0.9.2 ============= - + +- 'export' feature; from buggymodule import * except optBroken, optBroken2 - test&finish first class iterators: * nested iterators - * arglist as a type? + * test generic iterators - fix closure bug finally - overloading based on ASTs: 'constraint' should not be in PType but for the @@ -14,130 +15,31 @@ version 0.9.2 - ``hoist`` pragma for loop hoisting: can be easily done with AST overloading + global +- implement ``system.unsafeNew``. + + +version 0.9.X +============= + +- implement the missing features wrt inheritance +- implement generic methods - improve the compiler as a service - ``=`` should be overloadable; requires specialization for ``=`` -- implement constructors +- implement constructors + full 'not nil' checking - make 'bind' default for templates and introduce 'mixin'; special rule for ``[]=`` - implicit deref for parameter matching; overloading based on 'var T' - optimize genericAssign in the code generator - - -Bugs ----- - -- sneaking with qualifiedLookup() is really broken! -- bug: aporia.nim(968, 5) Error: ambiguous identifier: 'DELETE' -- - use a qualifier -- bug: the parser is not strict enough with newlines: 'echo "a" echo "b"' - compiles -- bug: blocks can "export" an identifier but the CCG generates {} for them ... - - -version 0.9.XX -============== - -- improve not-nil types -- make: - p(a, b): - echo a - echo b - - the same as: - - p(a, b, proc() = - echo a - echo b) - -- implement read/write tracking in the effect system -- implement the "snoopResult" pragma; no, make a strutils with string append - semantics instead ... -- implement "closure tuple consists of a single 'ref'" optimization -- JS gen: - - fix exception handling -- object branch transitions can't work with the current 'reset'; add a 'reset' - with an additional parameter --> re-evaluate this issue after constructors - have been added -- fix remaining closure bugs: - - test evals.nim with closures - - what about macros with closures? - -- allow implicit forward declarations of procs via a pragma (so that the - wrappers can deactivate it) -- rethink the syntax: distinction between expr and stmt is unfortunate; - indentation handling is quite complex too; problem with exception handling - is that often the scope of ``try`` is wrong and apart from that ``try`` is - a full blown statement; a ``try`` expression might be a good idea to make - error handling more light-weight -- fix destructors; don't work yet when used as expression -- make use of ``tyIter`` to fix the implicit items/pairs issue - better support for macros that rewrite procs - macros need access to types and symbols (partially implemented) -- document nimdoc properly finally -- make 'clamp' a magic for the range stuff -- we need to support iteration of 2 different data structures in parallel -- proc specialization in the code gen for write barrier specialization -- tlastmod returns wrong results on BSD (Linux, MacOS X: works) -- nested tuple unpacking; tuple unpacking in non-var-context -- make pegs support a compile-time option and make c2nim use regexes instead - per default? -- 'const' objects including case objects -- 'export' feature -- from buggymodule import * except optBroken, optBroken2 -- think about ``{:}.toTable[int, string]()`` -- mocking support with ``tyProxy`` that does: - o.p(x) --> p(o, x) --> myMacro(p, o, x) - - This is really the opposite of ``tyExpr``: - * For parameter ``tyExpr`` any argument matches. - * Argument ``tyProxy`` matches any parameter. - -Library -------- - -- suffix trees -- locale support; i18n module -- bignums +- rethink the syntax/grammar: + * parser is not strict enough with newlines + * change comment handling in the AST -Low priority ------------- - -- change how comments are part of the AST -- ``with proc `+`(x, y: T): T`` for generic code -- new feature: ``distinct T with operations`` -- implement the "easy" constructors idea -- resizing of strings/sequences could take into account the memory that - is allocated -- timeout for locks -- compilation cache: - - adapt thread var emulation to care about the new merge operation - - check for interface changes; if only the implementation changes, no - need to recompile clients; er ... what about templates, macros or anything - that has inlining semantics? -- codegen should use "NIM_CAST" macro and respect aliasing rules for GCC -- GC: precise stack marking; - escape analysis for string/seq seems to be easy to do too; - even further write barrier specialization -- GC: marker procs Boehm GC -- implement marker procs for message passing -- optimize method dispatchers -- activate more thread tests -- implement ``--script:sh|bat`` command line option; think about script - generation -- implement closures that support nesting of *procs* > 1 - - -Further optimization ideas -========================== - -- To optimize further copies away, you want to gather the additional - information inlining would provide, but don't inline for code size reasons. - - -Version 2 and beyond -==================== +Concurrency +----------- - shared memory heap: ``shared ref`` etc. The only hard part in the GC is to "stop the world". However, it may be worthwhile to generate explicit @@ -147,40 +49,66 @@ Version 2 and beyond calls syncGC() so that's pointless.) Hm instead of an heuristic simply provide a ``syncgc`` pragma to trigger compiler injection --> more general: an ``injectLoop`` pragma +- 'writes: []' effect; track reads/writes for shared types +- use the effect system to for static deadlock prevention -- const ptr/ref --> pointless because of aliasing; - much better: 'writes: []' effect -- language change: inheritance should only work with reference types, so that - the ``type`` field is not needed for objects! --> zero overhead aggregation - BETTER: ``of`` and safe object conversions only work with ref objects. Same - for multi methods. - -- explicit nil types? - * nil seq[int] - * nil string - * nil ref int - * nil ptr THallo - * nil proc +GC +== -- better for backwards compatibility: default nilable, but ``not nil`` - notation: - - type - PWindow = ref TWindow not nil - - The problem with ``nil`` is that the language currently relies on it for - implicit initialization. Initialization is different from assignment. The - issues can "easily" dealt with by ensuring: - - var x = myProc() # checks myProc() initializes every pointer explicitely +- precise stack marking; embrace C++ code generation for that +- marker procs for Boehm GC +- implement 'mixed' GC mode -- guards for the 'case' statement; generalized case statement; - a guard looks like: - - case x - of nkStmtList if x.value == 0: - - a generalized case statement looks like: - - case x with `=~` + +version 0.9.XX +============== + +- implement the "snoopResult" pragma; no, make a strutils with string append + semantics instead ... +- implement "closure tuple consists of a single 'ref'" optimization +- object branch transitions can't work with the current 'reset'; add a 'reset' + with an additional parameter --> re-evaluate this issue after constructors + have been added +- allow implicit forward declarations of procs via a pragma (so that the + wrappers can deactivate it) +- fix destructors; don't work yet when used as expression +- document nimdoc properly finally +- make 'clamp' a magic for the range stuff +- 'const' objects including case objects +- mocking support with ``tyProxy`` that does: fallback for ``.`` operator + + +Not essential for 1.0.0 +======================= + +- investigate the implicit items/pairs issue +- optimize method dispatchers +- ``with proc `+`(x, y: T): T`` for generic code +- new feature: ``distinct T with operations`` +- arglist as a type (iterator chaining); variable length type lists for generics +- resizing of strings/sequences could take into account the memory that + is allocated +- codegen should use "NIM_CAST" macro and respect aliasing rules for GCC +- implement marker procs for message passing +- activate more thread tests +- implement closures that support nesting of *procs* > 1 + + +Optimizations +============= + +- escape analysis for string/seq seems to be easy to do too; + even further write barrier specialization +- inlining of first class functions +- proc specialization in the code gen for write barrier specialization + + +Bugs +==== + +- sneaking with qualifiedLookup() is really broken! +- aporia.nim(968, 5) Error: ambiguous identifier: 'DELETE' -- + use a qualifier +- blocks can "export" an identifier but the CCG generates {} for them ... +- JS gen: fix exception handling From 9483606d5bc074b5c820c3ca8b73c9507139c48a Mon Sep 17 00:00:00 2001 From: Araq Date: Wed, 28 Nov 2012 08:03:25 +0100 Subject: [PATCH 002/106] bugfix: implicit items/pairs work better --- compiler/semexprs.nim | 7 ++++--- tests/compile/tcolonisproc.nim | 12 ++++++++++++ todo.txt | 1 - 3 files changed, 16 insertions(+), 4 deletions(-) create mode 100644 tests/compile/tcolonisproc.nim diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index e1d69c0bc..cf0e8f4dd 100755 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -589,9 +589,10 @@ proc semStaticExpr(c: PContext, n: PNode): PNode = proc semOverloadedCallAnalyseEffects(c: PContext, n: PNode, nOrig: PNode, flags: TExprFlags): PNode = - if efWantIterator in flags: - result = semOverloadedCall(c, n, nOrig, {skIterator}) - elif efInTypeOf in flags: + if flags*{efInTypeOf, efWantIterator} != {}: + # consider: 'for x in pReturningArray()' --> we don't want the restriction + # to 'skIterator' anymore; skIterator is preferred in sigmatch already for + # typeof support. # for ``type(countup(1,3))``, see ``tests/ttoseq``. result = semOverloadedCall(c, n, nOrig, {skProc, skMethod, skConverter, skMacro, skTemplate, skIterator}) diff --git a/tests/compile/tcolonisproc.nim b/tests/compile/tcolonisproc.nim new file mode 100644 index 000000000..e55587dfc --- /dev/null +++ b/tests/compile/tcolonisproc.nim @@ -0,0 +1,12 @@ + +proc p(a, b: int, c: proc ()) = + c() + + +p(1, 3): + echo 1 + echo 3 + +p(1, 1, proc() = + echo 1 + echo 2) diff --git a/todo.txt b/todo.txt index 10b044a6d..79b3cb529 100755 --- a/todo.txt +++ b/todo.txt @@ -82,7 +82,6 @@ version 0.9.XX Not essential for 1.0.0 ======================= -- investigate the implicit items/pairs issue - optimize method dispatchers - ``with proc `+`(x, y: T): T`` for generic code - new feature: ``distinct T with operations`` From 0ef08e49e74ca075f3dd7348791a77f00e5b3cab Mon Sep 17 00:00:00 2001 From: Araq Date: Wed, 28 Nov 2012 19:57:41 +0100 Subject: [PATCH 003/106] bugfix: better implict 'items' support --- compiler/importer.nim | 5 +++-- compiler/semexprs.nim | 7 +++---- tests/compile/titerovl.nim | 21 +++++++++++++++++++++ todo.txt | 31 +++++++++++++++---------------- 4 files changed, 42 insertions(+), 22 deletions(-) create mode 100644 tests/compile/titerovl.nim diff --git a/compiler/importer.nim b/compiler/importer.nim index f96377915..c928e8f3c 100755 --- a/compiler/importer.nim +++ b/compiler/importer.nim @@ -11,7 +11,7 @@ import intsets, strutils, os, ast, astalgo, msgs, options, idents, rodread, lookups, - semdata, passes + semdata, passes, renderer proc evalImport*(c: PContext, n: PNode): PNode proc evalFrom*(c: PContext, n: PNode): PNode @@ -29,7 +29,8 @@ proc getModuleName*(n: PNode): string = of nkSym: result = n.sym.name.s else: - internalError(n.info, "getModuleName") + localError(n.info, errGenerated, + "invalide module name: '$1'" % renderTree(n)) result = "" proc checkModuleName*(n: PNode): string = diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index e1d69c0bc..482489a65 100755 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -589,11 +589,10 @@ proc semStaticExpr(c: PContext, n: PNode): PNode = proc semOverloadedCallAnalyseEffects(c: PContext, n: PNode, nOrig: PNode, flags: TExprFlags): PNode = - if efWantIterator in flags: - result = semOverloadedCall(c, n, nOrig, {skIterator}) - elif efInTypeOf in flags: + if {efInTypeOf, efWantIterator} * flags != {}: + # consider 'proc p(): seq[int]; for x in p()' here and # for ``type(countup(1,3))``, see ``tests/ttoseq``. - result = semOverloadedCall(c, n, nOrig, + result = semOverloadedCall(c, n, nOrig, {skProc, skMethod, skConverter, skMacro, skTemplate, skIterator}) else: result = semOverloadedCall(c, n, nOrig, diff --git a/tests/compile/titerovl.nim b/tests/compile/titerovl.nim new file mode 100644 index 000000000..be665b2b7 --- /dev/null +++ b/tests/compile/titerovl.nim @@ -0,0 +1,21 @@ +discard """ + output: '''9 +1 +2 +3 +''' +""" + +# Test the new overloading rules for iterators: + +# test that iterator 'p' is preferred: +proc p(): seq[int] = @[1, 2, 3] +iterator p(): int = yield 9 + +for x in p(): echo x + +# test that 'q' works in this position: +proc q(): seq[int] = @[1, 2, 3] + +for x in q(): echo x + diff --git a/todo.txt b/todo.txt index 10b044a6d..567437c7f 100755 --- a/todo.txt +++ b/todo.txt @@ -53,36 +53,27 @@ Concurrency - use the effect system to for static deadlock prevention -GC -== - -- precise stack marking; embrace C++ code generation for that -- marker procs for Boehm GC -- implement 'mixed' GC mode - - version 0.9.XX ============== -- implement the "snoopResult" pragma; no, make a strutils with string append - semantics instead ... -- implement "closure tuple consists of a single 'ref'" optimization - object branch transitions can't work with the current 'reset'; add a 'reset' with an additional parameter --> re-evaluate this issue after constructors have been added -- allow implicit forward declarations of procs via a pragma (so that the - wrappers can deactivate it) - fix destructors; don't work yet when used as expression - document nimdoc properly finally - make 'clamp' a magic for the range stuff -- 'const' objects including case objects -- mocking support with ``tyProxy`` that does: fallback for ``.`` operator Not essential for 1.0.0 ======================= -- investigate the implicit items/pairs issue +- 'const' objects including case objects +- mocking support with ``tyProxy`` that does: fallback for ``.`` operator +- allow implicit forward declarations of procs via a pragma (so that the + wrappers can deactivate it) +- implement the "snoopResult" pragma; no, make a strutils with string append + semantics instead ... +- implement "closure tuple consists of a single 'ref'" optimization - optimize method dispatchers - ``with proc `+`(x, y: T): T`` for generic code - new feature: ``distinct T with operations`` @@ -95,6 +86,14 @@ Not essential for 1.0.0 - implement closures that support nesting of *procs* > 1 +GC +== + +- precise stack marking; embrace C++ code generation for that +- marker procs for Boehm GC +- implement 'mixed' GC mode + + Optimizations ============= From 92e10e4b182f4b106db79f4be228e19b2720d043 Mon Sep 17 00:00:00 2001 From: Araq Date: Wed, 28 Nov 2012 23:54:23 +0100 Subject: [PATCH 004/106] implemented 'import except' --- compiler/ast.nim | 5 +- compiler/ccgstmts.nim | 3 +- compiler/depends.nim | 2 +- compiler/docgen.nim | 2 +- compiler/ecmasgen.nim | 3 +- compiler/importer.nim | 28 ++++++++- compiler/parser.nim | 102 +++++++++++---------------------- compiler/renderer.nim | 33 ++++++----- compiler/semexprs.nim | 3 + compiler/semtypes.nim | 1 + doc/manual.txt | 13 +++++ lib/core/macros.nim | 8 ++- tests/reject/timportexcept.nim | 10 ++++ todo.txt | 2 +- 14 files changed, 120 insertions(+), 95 deletions(-) create mode 100644 tests/reject/timportexcept.nim diff --git a/compiler/ast.nim b/compiler/ast.nim index 77f8723e2..f4ba1ef70 100755 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -172,10 +172,11 @@ type nkDiscardStmt, # a discard statement nkStmtList, # a list of statements nkImportStmt, # an import statement - nkFromStmt, # a from * import statement nkImportExceptStmt, # an import x except a statement - nkIncludeStmt, # an include statement nkExportStmt, # an export statement + nkExportExceptStmt, # an 'export except' statement + nkFromStmt, # a from * import statement + nkIncludeStmt, # an include statement nkBindStmt, # a bind statement nkMixinStmt, # a mixin statement nkCommentStmt, # a comment statement diff --git a/compiler/ccgstmts.nim b/compiler/ccgstmts.nim index a950c62df..089f7fb83 100755 --- a/compiler/ccgstmts.nim +++ b/compiler/ccgstmts.nim @@ -880,7 +880,8 @@ proc genStmts(p: BProc, t: PNode) = # we have to emit the type information for object types here to support # separate compilation: genTypeSection(p.module, t) - of nkCommentStmt, nkNilLit, nkIteratorDef, nkIncludeStmt, nkImportStmt, + of nkCommentStmt, nkNilLit, nkIteratorDef, nkIncludeStmt, + nkImportStmt, nkImportExceptStmt, nkExportStmt, nkExportExceptStmt, nkFromStmt, nkTemplateDef, nkMacroDef: nil of nkPragma: genPragma(p, t) diff --git a/compiler/depends.nim b/compiler/depends.nim index 4fde24ab4..b9d38236b 100755 --- a/compiler/depends.nim +++ b/compiler/depends.nim @@ -35,7 +35,7 @@ proc addDotDependency(c: PPassContext, n: PNode): PNode = for i in countup(0, sonsLen(n) - 1): var imported = getModuleName(n.sons[i]) addDependencyAux(g.module.name.s, imported) - of nkFromStmt: + of nkFromStmt, nkImportExceptStmt: var imported = getModuleName(n.sons[0]) addDependencyAux(g.module.name.s, imported) of nkStmtList, nkBlockStmt, nkStmtListExpr, nkBlockExpr: diff --git a/compiler/docgen.nim b/compiler/docgen.nim index 8bbc53b74..5d46be874 100755 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -273,7 +273,7 @@ proc generateDoc*(d: PDoc, n: PNode) = generateDoc(d, lastSon(n.sons[0])) of nkImportStmt: for i in 0 .. sonsLen(n)-1: traceDeps(d, n.sons[i]) - of nkFromStmt: traceDeps(d, n.sons[0]) + of nkFromStmt, nkImportExceptStmt: traceDeps(d, n.sons[0]) else: nil proc genSection(d: PDoc, kind: TSymKind) = diff --git a/compiler/ecmasgen.nim b/compiler/ecmasgen.nim index cff4cc1dd..770e6a4a7 100755 --- a/compiler/ecmasgen.nim +++ b/compiler/ecmasgen.nim @@ -1465,7 +1465,8 @@ proc genStmt(p: var TProc, n: PNode, r: var TCompRes) = of nkAsmStmt: genAsmStmt(p, n, r) of nkTryStmt: genTryStmt(p, n, r) of nkRaiseStmt: genRaiseStmt(p, n, r) - of nkTypeSection, nkCommentStmt, nkIteratorDef, nkIncludeStmt, nkImportStmt, + of nkTypeSection, nkCommentStmt, nkIteratorDef, nkIncludeStmt, + nkImportStmt, nkImportExceptStmt, nkExportStmt, nkExportExceptStmt, nkFromStmt, nkTemplateDef, nkMacroDef, nkPragma: nil of nkProcDef, nkMethodDef, nkConverterDef: var s = n.sons[namePos].sym diff --git a/compiler/importer.nim b/compiler/importer.nim index c928e8f3c..b3821746c 100755 --- a/compiler/importer.nim +++ b/compiler/importer.nim @@ -103,11 +103,23 @@ proc importAllSymbols(c: PContext, fromMod: PSym) = while s != nil: if s.kind != skModule: if s.kind != skEnumField: - if not (s.Kind in ExportableSymKinds): + if s.Kind notin ExportableSymKinds: InternalError(s.info, "importAllSymbols: " & $s.kind) rawImportSymbol(c, s) # this is correct! s = NextIter(i, fromMod.tab) +proc importAllSymbolsExcept(c: PContext, fromMod: PSym, exceptSet: TIntSet) = + var i: TTabIter + var s = InitTabIter(i, fromMod.tab) + while s != nil: + if s.kind != skModule: + if s.kind != skEnumField: + if s.Kind notin ExportableSymKinds: + InternalError(s.info, "importAllSymbols: " & $s.kind) + if s.name.id notin exceptSet: + rawImportSymbol(c, s) + s = NextIter(i, fromMod.tab) + proc evalImport(c: PContext, n: PNode): PNode = result = n for i in countup(0, sonsLen(n) - 1): @@ -129,3 +141,17 @@ proc evalFrom(c: PContext, n: PNode): PNode = n.sons[0] = newSymNode(m) addDecl(c, m) # add symbol to symbol table of module for i in countup(1, sonsLen(n) - 1): importSymbol(c, n.sons[i], m) + +proc evalImportExcept*(c: PContext, n: PNode): PNode = + result = n + checkMinSonsLen(n, 2) + var f = checkModuleName(n.sons[0]) + if f.len > 0: + var m = gImportModule(f) + n.sons[0] = newSymNode(m) + addDecl(c, m) # add symbol to symbol table of module + var exceptSet = initIntSet() + for i in countup(1, sonsLen(n) - 1): + let ident = lookups.considerAcc(n.sons[i]) + exceptSet.incl(ident.id) + importAllSymbolsExcept(c, m, exceptSet) diff --git a/compiler/parser.nim b/compiler/parser.nim index 7ab87804e..bfc6b3b65 100755 --- a/compiler/parser.nim +++ b/compiler/parser.nim @@ -872,10 +872,10 @@ proc parseExprStmt(p: var TParser): PNode = getTok(p) skipComment(p, result) if p.tok.tokType == tkSad: getTok(p) - if not (p.tok.TokType in {tkOf, tkElif, tkElse, tkExcept}): + if p.tok.TokType notin {tkOf, tkElif, tkElse, tkExcept}: let body = parseStmt(p) addSon(result, newProcNode(nkDo, body.info, body)) - while true: + while true: if p.tok.tokType == tkSad: getTok(p) var b: PNode case p.tok.tokType @@ -900,33 +900,34 @@ proc parseExprStmt(p: var TParser): PNode = addSon(b, parseStmt(p)) addSon(result, b) if b.kind == nkElse: break - -proc parseModuleName(p: var TParser): PNode {.inline.} = - result = parseExpr(p) -proc parseImportOrIncludeStmt(p: var TParser, kind: TNodeKind): PNode = - var a: PNode +proc parseImport(p: var TParser, kind: TNodeKind): PNode = result = newNodeP(kind, p) + getTok(p) # skip `import` or `export` + optInd(p, result) + var a = parseExpr(p) + addSon(result, a) + if p.tok.tokType in {tkComma, tkExcept}: + if p.tok.tokType == tkExcept: + result.kind = succ(kind) + getTok(p) + optInd(p, result) + while p.tok.tokType notin {tkEof, tkSad, tkDed}: + a = parseExpr(p) + if a.kind == nkEmpty: break + addSon(result, a) + if p.tok.tokType != tkComma: break + getTok(p) + optInd(p, a) + expectNl(p) + +proc parseIncludeStmt(p: var TParser): PNode = + result = newNodeP(nkIncludeStmt, p) getTok(p) # skip `import` or `include` optInd(p, result) - while true: - case p.tok.tokType - of tkEof, tkSad, tkDed: - break - of tkSymbol, tkAccent: - a = parseSymbol(p) - of tkRStrLit: - a = newStrNodeP(nkRStrLit, p.tok.literal, p) - getTok(p) - of tkStrLit: - a = newStrNodeP(nkStrLit, p.tok.literal, p) - getTok(p) - of tkTripleStrLit: - a = newStrNodeP(nkTripleStrLit, p.tok.literal, p) - getTok(p) - else: - parMessage(p, errIdentifierExpected, p.tok) - break + while p.tok.tokType notin {tkEof, tkSad, tkDed}: + var a = parseExpr(p) + if a.kind == nkEmpty: break addSon(result, a) if p.tok.tokType != tkComma: break getTok(p) @@ -937,19 +938,13 @@ proc parseFromStmt(p: var TParser): PNode = result = newNodeP(nkFromStmt, p) getTok(p) # skip `from` optInd(p, result) - var a = parseModuleName(p) + var a = parseExpr(p) addSon(result, a) #optInd(p, a); eat(p, tkImport) optInd(p, result) - while true: - case p.tok.tokType #optInd(p, a); - of tkEof, tkSad, tkDed: - break - of tkSymbol, tkAccent: - a = parseSymbol(p) - else: - parMessage(p, errIdentifierExpected, p.tok) - break + while p.tok.tokType notin {tkEof, tkSad, tkDed}: + a = parseExpr(p) + if a.kind == nkEmpty: break addSon(result, a) if p.tok.tokType != tkComma: break getTok(p) @@ -1264,14 +1259,7 @@ proc parseEnum(p: var TParser): PNode = result = newNodeP(nkEnumTy, p) a = nil getTok(p) - if false and p.tok.tokType == tkOf: - a = newNodeP(nkOfInherit, p) - getTok(p) - optInd(p, a) - addSon(a, parseTypeDesc(p)) - addSon(result, a) - else: - addSon(result, ast.emptyNode) + addSon(result, ast.emptyNode) optInd(p, result) while true: case p.tok.tokType @@ -1410,18 +1398,6 @@ proc parseDistinct(p: var TParser): PNode = optInd(p, result) addSon(result, parseTypeDesc(p)) -proc parsePointerInTypeSection(p: var TParser, kind: TNodeKind): PNode = - result = newNodeP(kind, p) - getTok(p) - optInd(p, result) - if not isOperator(p.tok): - case p.tok.tokType - of tkObject: addSon(result, parseObject(p)) - of tkTuple: addSon(result, parseTuple(p, true)) - else: - if isExprStart(p): - addSon(result, parseTypeDesc(p)) - proc parseTypeDef(p: var TParser): PNode = result = newNodeP(nkTypeDef, p) addSon(result, identWithPragma(p)) @@ -1430,15 +1406,6 @@ proc parseTypeDef(p: var TParser): PNode = if p.tok.tokType == tkEquals: getTok(p) optInd(p, result) - #var a: PNode - #case p.tok.tokType - #of tkObject: a = parseObject(p) - #of tkEnum: a = parseEnum(p) - #of tkDistinct: a = parseDistinct(p) - #of tkTuple: a = parseTuple(p, true) - #of tkRef: a = parsePointerInTypeSection(p, nkRefTy) - #of tkPtr: a = parsePointerInTypeSection(p, nkPtrTy) - #else: a = parseTypeDesc(p) addSon(result, parseTypeDefAux(p)) else: addSon(result, ast.emptyNode) @@ -1496,11 +1463,12 @@ proc simpleStmt(p: var TParser): PNode = of tkBreak: result = parseBreakOrContinue(p, nkBreakStmt) of tkContinue: result = parseBreakOrContinue(p, nkContinueStmt) of tkCurlyDotLe: result = parseStmtPragma(p) - of tkImport: result = parseImportOrIncludeStmt(p, nkImportStmt) + of tkImport: result = parseImport(p, nkImportStmt) + of tkExport: result = parseImport(p, nkExportStmt) of tkFrom: result = parseFromStmt(p) - of tkInclude: result = parseImportOrIncludeStmt(p, nkIncludeStmt) + of tkInclude: result = parseIncludeStmt(p) of tkComment: result = newCommentStmt(p) - else: + else: if isExprStart(p): result = parseExprStmt(p) else: result = ast.emptyNode if result.kind != nkEmpty: skipComment(p, result) diff --git a/compiler/renderer.nim b/compiler/renderer.nim index 4444f2ab0..c8ee66e03 100755 --- a/compiler/renderer.nim +++ b/compiler/renderer.nim @@ -1134,13 +1134,27 @@ proc gsub(g: var TSrcGen, n: PNode, c: TContext) = put(g, tkCurlyDotLe, "{.") gcomma(g, n, emptyContext) put(g, tkCurlyDotRi, ".}") - of nkImportStmt: - putWithSpace(g, tkImport, "import") + of nkImportStmt, nkExportStmt: + if n.kind == nkImportStmt: + putWithSpace(g, tkImport, "import") + else: + putWithSpace(g, tkExport, "export") gcoms(g) indentNL(g) gcommaAux(g, n, g.indent) dedent(g) putNL(g) + of nkImportExceptStmt, nkExportExceptStmt: + if n.kind == nkImportExceptStmt: + putWithSpace(g, tkImport, "import") + else: + putWithSpace(g, tkExport, "export") + gsub(g, n.sons[0]) + put(g, tkSpaces, Space) + putWithSpace(g, tkExcept, "except") + gcommaAux(g, n, g.indent, 1) + gcoms(g) + putNL(g) of nkFromStmt: putWithSpace(g, tkFrom, "from") gsub(g, n.sons[0]) @@ -1148,14 +1162,6 @@ proc gsub(g: var TSrcGen, n: PNode, c: TContext) = putWithSpace(g, tkImport, "import") gcomma(g, n, emptyContext, 1) putNL(g) - of nkImportExceptStmt: - putWithSpace(g, tkImport, "import") - gsub(g, n.sons[0]) - put(g, tkSpaces, Space) - putWithSpace(g, tkExcept, "except") - gcommaAux(g, n, g.indent, 1) - gcoms(g) - putNL(g) of nkIncludeStmt: putWithSpace(g, tkInclude, "include") gcoms(g) @@ -1163,13 +1169,6 @@ proc gsub(g: var TSrcGen, n: PNode, c: TContext) = gcommaAux(g, n, g.indent) dedent(g) putNL(g) - of nkExportStmt: - putWithSpace(g, tkExport, "export") - gcoms(g) - indentNL(g) - gcommaAux(g, n, g.indent) - dedent(g) - putNL(g) of nkCommentStmt: gcoms(g) optNL(g) diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index 482489a65..9f2b431f6 100755 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -1850,6 +1850,9 @@ proc semExpr(c: PContext, n: PNode, flags: TExprFlags = {}): PNode = of nkImportStmt: if not isTopLevel(c): LocalError(n.info, errXOnlyAtModuleScope, "import") result = evalImport(c, n) + of nkImportExceptStmt: + if not isTopLevel(c): LocalError(n.info, errXOnlyAtModuleScope, "import") + result = evalImportExcept(c, n) of nkFromStmt: if not isTopLevel(c): LocalError(n.info, errXOnlyAtModuleScope, "from") result = evalFrom(c, n) diff --git a/compiler/semtypes.nim b/compiler/semtypes.nim index f644683f5..9efeba5b7 100755 --- a/compiler/semtypes.nim +++ b/compiler/semtypes.nim @@ -825,6 +825,7 @@ proc semTypeNode(c: PContext, n: PNode, prev: PType): PType = checkSonsLen(n, 3) result = semTypeNode(c, n.sons[1], prev) if result.kind in NilableTypes and n.sons[2].kind == nkNilLit: + # XXX this is wrong for tyString at least result.flags.incl(tfNotNil) else: LocalError(n.info, errGenerated, "invalid type") diff --git a/doc/manual.txt b/doc/manual.txt index f6ff576bf..a5bce775b 100755 --- a/doc/manual.txt +++ b/doc/manual.txt @@ -3741,6 +3741,19 @@ This is best illustrated by an example: return x + 1 +Import statement +~~~~~~~~~~~~~~~~ + +After the import statement a list of module names can follow or a single +module name followed by an ``except`` to prevent some symbols to be imported: + +.. code-block:: nimrod + import strutils except `%` + + # doesn't work then: + echo "$1" % "abc" + + Scope rules ----------- Identifiers are valid from the point of their declaration until the end of diff --git a/lib/core/macros.nim b/lib/core/macros.nim index d2d842025..bf7510a92 100755 --- a/lib/core/macros.nim +++ b/lib/core/macros.nim @@ -47,10 +47,12 @@ type nnkReturnStmt, nnkBreakStmt, nnkContinueStmt, nnkBlockStmt, nnkStaticStmt, nnkDiscardStmt, nnkStmtList, - nnkImportStmt, nnkFromStmt, - nkImportExceptStmt, - nnkIncludeStmt, + nnkImportStmt, + nnkImportExceptStmt, nnkExportStmt, + nnkExportExceptStmt, + nnkFromStmt, + nnkIncludeStmt, nnkBindStmt, nnkMixinStmt, nnkCommentStmt, nnkStmtListExpr, nnkBlockExpr, diff --git a/tests/reject/timportexcept.nim b/tests/reject/timportexcept.nim new file mode 100644 index 000000000..93a7fd642 --- /dev/null +++ b/tests/reject/timportexcept.nim @@ -0,0 +1,10 @@ +discard """ + line: 9 + errormsg: "undeclared identifier: '%'" +""" + +import strutils except `%` + +# doesn't work +echo "$1" % "abc" + diff --git a/todo.txt b/todo.txt index 567437c7f..804f6e5f6 100755 --- a/todo.txt +++ b/todo.txt @@ -1,7 +1,7 @@ version 0.9.2 ============= -- 'export' feature; from buggymodule import * except optBroken, optBroken2 +- 'export' feature - test&finish first class iterators: * nested iterators * test generic iterators From f503439e811d822f19daa9591cb3bd9e90edabe7 Mon Sep 17 00:00:00 2001 From: Araq Date: Fri, 30 Nov 2012 19:57:23 +0100 Subject: [PATCH 005/106] implements 'import dir/module' without quotes --- compiler/importer.nim | 8 +++++--- tests/compile/twalker.nim | 2 +- todo.txt | 2 +- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/compiler/importer.nim b/compiler/importer.nim index b3821746c..11fc45601 100755 --- a/compiler/importer.nim +++ b/compiler/importer.nim @@ -29,9 +29,11 @@ proc getModuleName*(n: PNode): string = of nkSym: result = n.sym.name.s else: - localError(n.info, errGenerated, - "invalide module name: '$1'" % renderTree(n)) - result = "" + # hacky way to implement 'x / y /../ z': + result = renderTree(n, {renderNoComments}).replace(" ") + #localError(n.info, errGenerated, + # "invalide module name: '$1'" % renderTree(n)) + #result = "" proc checkModuleName*(n: PNode): string = # This returns the full canonical path for a given module import diff --git a/tests/compile/twalker.nim b/tests/compile/twalker.nim index 3fdd8769b..89e6c2b9d 100755 --- a/tests/compile/twalker.nim +++ b/tests/compile/twalker.nim @@ -1,7 +1,7 @@ # iterate over all files with a given filter: import - os, times + "../../lib/pure/os.nim", ../../ lib / pure / times proc main(filter: string) = for filename in walkFiles(filter): diff --git a/todo.txt b/todo.txt index 804f6e5f6..e6731582f 100755 --- a/todo.txt +++ b/todo.txt @@ -50,7 +50,7 @@ Concurrency provide a ``syncgc`` pragma to trigger compiler injection --> more general: an ``injectLoop`` pragma - 'writes: []' effect; track reads/writes for shared types -- use the effect system to for static deadlock prevention +- use the effect system for static deadlock prevention version 0.9.XX From 336da8f44e28da0cc4f51933f52b836f7e2e9c59 Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Sat, 1 Dec 2012 16:04:59 +0000 Subject: [PATCH 006/106] IRC module is now saner at the expense of a little API breakage. Added warnings for unstable APIs to modules that use AsyncIO. --- lib/pure/asyncio.nim | 3 + lib/pure/ftpclient.nim | 4 + lib/pure/httpserver.nim | 2 + lib/pure/irc.nim | 193 +++++++++++++++++++++------------------- lib/pure/scgi.nim | 2 + 5 files changed, 111 insertions(+), 93 deletions(-) diff --git a/lib/pure/asyncio.nim b/lib/pure/asyncio.nim index 5ecc16de7..0f5197792 100644 --- a/lib/pure/asyncio.nim +++ b/lib/pure/asyncio.nim @@ -35,6 +35,9 @@ import sockets, os ## that in the future this type's fields will not be exported therefore breaking ## your code. ## +## **Warning:** The API of this module is unstable, and therefore is subject +## to change. +## ## Asynchronous sockets ## ==================== ## diff --git a/lib/pure/ftpclient.nim b/lib/pure/ftpclient.nim index 410c5e8bc..78c96c3f1 100644 --- a/lib/pure/ftpclient.nim +++ b/lib/pure/ftpclient.nim @@ -26,6 +26,10 @@ import sockets, strutils, parseutils, times, os, asyncio ## var ftp = FTPClient("example.org", user = "user", pass = "pass") ## ftp.connect() ## ftp.retrFile("file.ext", "file.ext") +## +## **Warning:** The API of this module is unstable, and therefore is subject +## to change. + type TFTPClient* = object of TObject diff --git a/lib/pure/httpserver.nim b/lib/pure/httpserver.nim index 9d9ea2012..ce816b7d4 100755 --- a/lib/pure/httpserver.nim +++ b/lib/pure/httpserver.nim @@ -22,6 +22,8 @@ ## ## run(handleRequest, TPort(80)) ## +## **Warning:** The API of this module is unstable, and therefore is subject +## to change. import parseutils, strutils, os, osproc, strtabs, streams, sockets, asyncio diff --git a/lib/pure/irc.nim b/lib/pure/irc.nim index 81a1befb5..11808ca33 100644 --- a/lib/pure/irc.nim +++ b/lib/pure/irc.nim @@ -16,15 +16,21 @@ ## the amount of lag. ## ## .. code-block:: Nimrod -## var client = irc("irc.server.net", joinChans = @["#channel"]) +## +## var client = irc("picheta.me", joinChans = @["#bots"]) ## client.connect() ## while True: ## var event: TIRCEvent ## if client.poll(event): ## case event.typ -## of EvDisconnected: break +## of EvConnected: nil +## of EvDisconnected: +## client.reconnect() ## of EvMsg: -## # Where all the magic happens. +## # Write your message reading code here. +## +## **Warning:** The API of this module is unstable, and therefore is subject +## to change. import sockets, strutils, parseutils, times, asyncio @@ -34,11 +40,15 @@ type port: TPort nick, user, realname, serverPass: string case isAsync: bool - of false: - sock: TSocket of true: - handleEvent: proc (irc: var TAsyncIRC, ev: TIRCEvent) {.closure.} + handleEvent: proc (irc: PAsyncIRC, ev: TIRCEvent) {.closure.} asyncSock: PAsyncSocket + myDispatcher: PDispatcher + of false: + dummyA: pointer + dummyB: pointer # workaround a Nimrod API issue + dummyC: pointer + sock: TSocket status: TInfo lastPing: float lastPong: float @@ -47,6 +57,8 @@ type msgLimit: bool messageBuffer: seq[tuple[timeToSend: float, m: string]] + PIRC* = ref TIRC + PAsyncIRC* = ref TAsyncIRC TAsyncIRC* = object of TIRC @@ -68,9 +80,13 @@ type MError TIRCEventType* = enum - EvMsg, EvDisconnected + EvMsg, EvConnected, EvDisconnected TIRCEvent* = object ## IRC Event case typ*: TIRCEventType + of EvConnected: + ## Connected to server. + ## Only occurs with AsyncIRC. + nil of EvDisconnected: ## Disconnected from the server nil @@ -82,7 +98,7 @@ type origin*: string ## The channel/user that this msg originated from raw*: string ## Raw IRC message -proc send*(irc: var TIRC, message: string, sendImmediately = false) = +proc send*(irc: PIRC, message: string, sendImmediately = false) = ## Sends ``message`` as a raw command. It adds ``\c\L`` for you. var sendMsg = true if irc.msgLimit and not sendImmediately: @@ -104,15 +120,15 @@ proc send*(irc: var TIRC, message: string, sendImmediately = false) = # but I can't exactly check for EBrokenPipe. irc.status = SockClosed -proc privmsg*(irc: var TIRC, target, message: string) = +proc privmsg*(irc: PIRC, target, message: string) = ## Sends ``message`` to ``target``. ``Target`` can be a channel, or a user. irc.send("PRIVMSG $1 :$2" % [target, message]) -proc notice*(irc: var TIRC, target, message: string) = +proc notice*(irc: PIRC, target, message: string) = ## Sends ``notice`` to ``target``. ``Target`` can be a channel, or a user. irc.send("NOTICE $1 :$2" % [target, message]) -proc join*(irc: var TIRC, channel: string, key = "") = +proc join*(irc: PIRC, channel: string, key = "") = ## Joins ``channel``. ## ## If key is not ``""``, then channel is assumed to be key protected and this @@ -122,16 +138,19 @@ proc join*(irc: var TIRC, channel: string, key = "") = else: irc.send("JOIN " & channel & " " & key) -proc part*(irc: var TIRC, channel, message: string) = +proc part*(irc: PIRC, channel, message: string) = ## Leaves ``channel`` with ``message``. irc.send("PART " & channel & " :" & message) -proc close*(irc: var TIRC) = +proc close*(irc: PIRC) = ## Closes connection to an IRC server. ## ## **Warning:** This procedure does not send a ``QUIT`` message to the server. irc.status = SockClosed - irc.sock.close() + if irc.isAsync: + irc.asyncSock.close() + else: + irc.sock.close() proc isNumber(s: string): bool = ## Checks if `s` contains only numbers. @@ -202,12 +221,11 @@ proc parseMessage(msg: string): TIRCEvent = inc(i) # Skip `:`. result.params.add(msg[i..msg.len-1]) -proc connect*(irc: var TIRC) = +proc connect*(irc: PIRC) = ## Connects to an IRC server as specified by ``irc``. assert(irc.address != "") assert(irc.port != TPort(0)) - irc.sock = socket() irc.sock.connect(irc.address, irc.port) irc.status = SockConnected @@ -217,13 +235,21 @@ proc connect*(irc: var TIRC) = irc.send("NICK " & irc.nick, true) irc.send("USER $1 * 0 :$2" % [irc.user, irc.realname], true) +proc reconnect*(irc: PIRC) = + ## Reconnects to an IRC server. + ## + ## This should be used when an ``EvDisconnected`` event occurs. + irc.sock = socket() + irc.connect() + proc irc*(address: string, port: TPort = 6667.TPort, nick = "NimrodBot", user = "NimrodBot", realname = "NimrodBot", serverPass = "", joinChans: seq[string] = @[], - msgLimit: bool = true): TIRC = + msgLimit: bool = true): PIRC = ## Creates a ``TIRC`` object. + new(result) result.address = address result.port = port result.nick = nick @@ -237,8 +263,9 @@ proc irc*(address: string, port: TPort = 6667.TPort, result.msgLimit = msgLimit result.messageBuffer = @[] result.status = SockIdle + result.sock = socket() -proc processLine(irc: var TIRC, line: string): TIRCEvent = +proc processLine(irc: PIRC, line: string): TIRCEvent = if line.len == 0: irc.close() result.typ = EvDisconnected @@ -254,8 +281,8 @@ proc processLine(irc: var TIRC, line: string): TIRCEvent = result.typ = EvDisconnected return - if result.cmd == MPing: - irc.send("PONG " & result.params[0]) + #if result.cmd == MPing: + # irc.send("PONG " & result.params[0]) if result.cmd == MPong: irc.lag = epochTime() - parseFloat(result.params[result.params.high]) irc.lastPong = epochTime() @@ -271,11 +298,11 @@ proc processLine(irc: var TIRC, line: string): TIRCEvent = if result.nick == irc.nick: irc.nick = result.params[0] -proc processOther(irc: var TIRC, ev: var TIRCEvent): bool = +proc processOther(irc: PIRC, ev: var TIRCEvent): bool = result = false - if epochTime() - irc.lastPing >= 20.0: - irc.lastPing = epochTime() - irc.send("PING :" & formatFloat(irc.lastPing), true) + #if epochTime() - irc.lastPing >= 20.0: + # irc.lastPing = epochTime() + # irc.send("PING :" & formatFloat(irc.lastPing), true) if epochTime() - irc.lastPong >= 120.0 and irc.lastPong != -1.0: irc.close() @@ -290,7 +317,7 @@ proc processOther(irc: var TIRC, ev: var TIRCEvent): bool = break # messageBuffer is guaranteed to be from the quickest to the # later-est. -proc poll*(irc: var TIRC, ev: var TIRCEvent, +proc poll*(irc: PIRC, ev: var TIRCEvent, timeout: int = 500): bool = ## This function parses a single message from the IRC server and returns ## a TIRCEvent. @@ -316,46 +343,32 @@ proc poll*(irc: var TIRC, ev: var TIRCEvent, if processOther(irc, ev): result = true -proc getLag*(irc: var TIRC): float = +proc getLag*(irc: PIRC): float = ## Returns the latency between this client and the IRC server in seconds. ## ## If latency is unknown, returns -1.0. return irc.lag -proc isConnected*(irc: var TIRC): bool = +proc isConnected*(irc: PIRC): bool = ## Returns whether this IRC client is connected to an IRC server. return irc.status == SockConnected -proc getNick*(irc: var TIRC): string = +proc getNick*(irc: PIRC): string = ## Returns the current nickname of the client. return irc.nick # -- Asyncio dispatcher -proc connect*(irc: PAsyncIRC) = - ## Equivalent of connect for ``TIRC`` but specifically created for asyncio. - assert(irc.address != "") - assert(irc.port != TPort(0)) - - irc.asyncSock = AsyncSocket() - irc.asyncSock.connect(irc.address, irc.port) - proc handleConnect(s: PAsyncSocket, irc: PAsyncIRC) = # Greet the server :) - if irc.serverPass != "": irc[].send("PASS " & irc.serverPass, true) - irc[].send("NICK " & irc.nick, true) - irc[].send("USER $1 * 0 :$2" % [irc.user, irc.realname], true) - -discard """proc handleConnect(h: PObject) = - var irc = PAsyncIRC(h) - - # Greet the server :) - if irc.serverPass != "": irc[].send("PASS " & irc.serverPass, true) - irc[].send("NICK " & irc.nick, true) - irc[].send("USER $1 * 0 :$2" % [irc.user, irc.realname], true) - + if irc.serverPass != "": irc.send("PASS " & irc.serverPass, true) + irc.send("NICK " & irc.nick, true) + irc.send("USER $1 * 0 :$2" % [irc.user, irc.realname], true) irc.status = SockConnected -""" + + var ev: TIRCEvent + ev.typ = EvConnected + irc.handleEvent(irc, ev) proc handleRead(s: PAsyncSocket, irc: PAsyncIRC) = var line = "".TaintedString @@ -363,42 +376,48 @@ proc handleRead(s: PAsyncSocket, irc: PAsyncIRC) = if ret: if line == "": var ev: TIRCEvent - irc[].close() + irc.close() ev.typ = EvDisconnected - irc[].handleEvent(irc[], ev) + irc.handleEvent(irc, ev) else: - var ev = irc[].processLine(line.string) - irc[].handleEvent(irc[], ev) - -discard """proc handleRead(h: PObject) = - var irc = PAsyncIRC(h) - var line = "".TaintedString - var ret = irc.sock.recvLineAsync(line) - case ret - of RecvFullLine: - var ev = irc[].processLine(irc.lineBuffer.string & line.string) - irc.handleEvent(irc[], ev, irc.userArg) - irc.lineBuffer = "".TaintedString - of RecvPartialLine: - if line.string != "": - string(irc.lineBuffer).add(line.string) - of RecvDisconnected: - var ev: TIRCEvent - irc[].close() - ev.typ = EvDisconnected - irc.handleEvent(irc[], ev, irc.userArg) - of RecvFail: nil""" + var ev = irc.processLine(line.string) + irc.handleEvent(irc, ev) proc handleTask(s: PAsyncSocket, irc: PAsyncIRC) = var ev: TIRCEvent - if irc[].processOther(ev): - irc.handleEvent(irc[], ev) + if irc.processOther(ev): + irc.handleEvent(irc, ev) + +proc register*(d: PDispatcher, irc: PAsyncIRC) = + ## Registers ``irc`` with dispatcher ``d``. + irc.asyncSock.handleConnect = + proc (s: PAsyncSocket) = + handleConnect(s, irc) + irc.asyncSock.handleRead = + proc (s: PAsyncSocket) = + handleRead(s, irc) + irc.asyncSock.handleTask = + proc (s: PAsyncSocket) = + handleTask(s, irc) + d.register(irc.asyncSock) + irc.myDispatcher = d + +proc connect*(irc: PAsyncIRC) = + ## Equivalent of connect for ``TIRC`` but specifically created for asyncio. + assert(irc.address != "") + assert(irc.port != TPort(0)) -discard """proc handleTask(h: PObject) = - var irc = PAsyncIRC(h) - var ev: TIRCEvent - if PAsyncIRC(h)[].processOther(ev): - irc.handleEvent(irc[], ev, irc.userArg)""" + irc.asyncSock.connect(irc.address, irc.port) + +proc reconnect*(irc: PAsyncIRC) = + ## Reconnects to an IRC server. + ## + ## This should be used when an ``EvDisconnected`` event occurs. + ## + ## When successfully reconnected an ``EvConnected`` event will occur. + irc.asyncSock = AsyncSocket() + irc.myDispatcher.register(irc) + irc.connect() proc asyncIRC*(address: string, port: TPort = 6667.TPort, nick = "NimrodBot", @@ -406,7 +425,7 @@ proc asyncIRC*(address: string, port: TPort = 6667.TPort, realname = "NimrodBot", serverPass = "", joinChans: seq[string] = @[], msgLimit: bool = true, - ircEvent: proc (irc: var TAsyncIRC, ev: TIRCEvent) {.closure.} + ircEvent: proc (irc: PAsyncIRC, ev: TIRCEvent) {.closure.} ): PAsyncIRC = ## Use this function if you want to use asyncio's dispatcher. ## @@ -429,19 +448,7 @@ proc asyncIRC*(address: string, port: TPort = 6667.TPort, result.msgLimit = msgLimit result.messageBuffer = @[] result.handleEvent = ircEvent - -proc register*(d: PDispatcher, irc: PAsyncIRC) = - ## Registers ``irc`` with dispatcher ``d``. - irc.asyncSock.handleConnect = - proc (s: PAsyncSocket) = - handleConnect(s, irc) - irc.asyncSock.handleRead = - proc (s: PAsyncSocket) = - handleRead(s, irc) - irc.asyncSock.handleTask = - proc (s: PAsyncSocket) = - handleTask(s, irc) - d.register(irc.asyncSock) + result.asyncSock = AsyncSocket() when isMainModule: #var m = parseMessage("ERROR :Closing Link: dom96.co.cc (Ping timeout: 252 seconds)") diff --git a/lib/pure/scgi.nim b/lib/pure/scgi.nim index 44a579a7d..0f3b44e00 100755 --- a/lib/pure/scgi.nim +++ b/lib/pure/scgi.nim @@ -23,6 +23,8 @@ ## ## run(handleRequest) ## +## **Warning:** The API of this module is unstable, and therefore is subject +## to change. import sockets, strutils, os, strtabs, asyncio From 10d13d5032bd1109207a9feb9f487ad5d7e102de Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Sat, 1 Dec 2012 16:38:33 +0000 Subject: [PATCH 007/106] Fixed IRC test. --- lib/pure/irc.nim | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/lib/pure/irc.nim b/lib/pure/irc.nim index 11808ca33..8916681f7 100644 --- a/lib/pure/irc.nim +++ b/lib/pure/irc.nim @@ -281,8 +281,8 @@ proc processLine(irc: PIRC, line: string): TIRCEvent = result.typ = EvDisconnected return - #if result.cmd == MPing: - # irc.send("PONG " & result.params[0]) + if result.cmd == MPing: + irc.send("PONG " & result.params[0]) if result.cmd == MPong: irc.lag = epochTime() - parseFloat(result.params[result.params.high]) irc.lastPong = epochTime() @@ -300,9 +300,9 @@ proc processLine(irc: PIRC, line: string): TIRCEvent = proc processOther(irc: PIRC, ev: var TIRCEvent): bool = result = false - #if epochTime() - irc.lastPing >= 20.0: - # irc.lastPing = epochTime() - # irc.send("PING :" & formatFloat(irc.lastPing), true) + if epochTime() - irc.lastPing >= 20.0: + irc.lastPing = epochTime() + irc.send("PING :" & formatFloat(irc.lastPing), true) if epochTime() - irc.lastPong >= 120.0 and irc.lastPong != -1.0: irc.close() @@ -463,6 +463,8 @@ when isMainModule: var event: TIRCEvent if client.poll(event): case event.typ + of EvConnected: + nil of EvDisconnected: break of EvMsg: From c98e3d2c274ac4bd4227d4e3f7e0d2636827f88c Mon Sep 17 00:00:00 2001 From: Araq Date: Sat, 1 Dec 2012 19:10:47 +0100 Subject: [PATCH 008/106] implements 'export' feature --- compiler/astalgo.nim | 17 ++++---- compiler/importer.nim | 71 +++++++++++++++++++------------- compiler/semexprs.nim | 23 +++++++++++ doc/manual.txt | 31 +++++++++++++- lib/pure/collections/intsets.nim | 5 +++ tests/compile/mexporta.nim | 8 ++++ tests/compile/mexportb.nim | 7 ++++ tests/compile/texport.nim | 10 +++++ todo.txt | 2 +- web/news.txt | 3 ++ 10 files changed, 137 insertions(+), 40 deletions(-) create mode 100644 tests/compile/mexporta.nim create mode 100644 tests/compile/mexportb.nim create mode 100644 tests/compile/texport.nim diff --git a/compiler/astalgo.nim b/compiler/astalgo.nim index da0de3e94..9a2bebab4 100755 --- a/compiler/astalgo.nim +++ b/compiler/astalgo.nim @@ -585,8 +585,9 @@ proc StrTableContains(t: TStrTable, n: PSym): bool = proc StrTableRawInsert(data: var TSymSeq, n: PSym) = var h: THash = n.name.h and high(data) while data[h] != nil: - if data[h] == n: - InternalError(n.info, "StrTableRawInsert: " & n.name.s) + if data[h] == n: + # allowed for 'export' feature: + #InternalError(n.info, "StrTableRawInsert: " & n.name.s) return h = nextTry(h, high(data)) assert(data[h] == nil) @@ -617,23 +618,23 @@ proc StrTableAdd(t: var TStrTable, n: PSym) = StrTableRawInsert(t.data, n) inc(t.counter) -proc StrTableIncl*(t: var TStrTable, n: PSym): bool = +proc StrTableIncl*(t: var TStrTable, n: PSym): bool {.discardable.} = # returns true if n is already in the string table: # It is essential that `n` is written nevertheless! # This way the newest redefinition is picked by the semantic analyses! assert n.name != nil var h: THash = n.name.h and high(t.data) - while true: + while true: var it = t.data[h] - if it == nil: break - if it.name.id == n.name.id: + if it == nil: break + if it.name.id == n.name.id: t.data[h] = n # overwrite it with newer definition! return true # found it h = nextTry(h, high(t.data)) - if mustRehash(len(t.data), t.counter): + if mustRehash(len(t.data), t.counter): StrTableEnlarge(t) StrTableRawInsert(t.data, n) - else: + else: assert(t.data[h] == nil) t.data[h] = n inc(t.counter) diff --git a/compiler/importer.nim b/compiler/importer.nim index 11fc45601..774bcea6a 100755 --- a/compiler/importer.nim +++ b/compiler/importer.nim @@ -15,7 +15,6 @@ import proc evalImport*(c: PContext, n: PNode): PNode proc evalFrom*(c: PContext, n: PNode): PNode -proc importAllSymbols*(c: PContext, fromMod: PSym) proc getModuleName*(n: PNode): string = # This returns a short relative module name without the nim extension @@ -42,42 +41,43 @@ proc checkModuleName*(n: PNode): string = if result.len == 0: LocalError(n.info, errCannotOpenFile, modulename) -proc rawImportSymbol(c: PContext, s: PSym) = +proc rawImportSymbol(c: PContext, s: PSym) = # This does not handle stubs, because otherwise loading on demand would be # pointless in practice. So importing stubs is fine here! - var copy = s # do not copy symbols when importing! # check if we have already a symbol of the same name: var check = StrTableGet(c.tab.stack[importTablePos], s.name) - if check != nil and check.id != copy.id: - if s.kind notin OverloadableSyms: + if check != nil and check.id != s.id: + if s.kind notin OverloadableSyms: # s and check need to be qualified: - Incl(c.AmbiguousSymbols, copy.id) + Incl(c.AmbiguousSymbols, s.id) Incl(c.AmbiguousSymbols, check.id) - StrTableAdd(c.tab.stack[importTablePos], copy) - if s.kind == skType: + # thanks to 'export' feature, it could be we import the same symbol from + # multiple sources, so we need to call 'StrTableAdd' here: + StrTableAdd(c.tab.stack[importTablePos], s) + if s.kind == skType: var etyp = s.typ - if etyp.kind in {tyBool, tyEnum} and sfPure notin s.flags: - for j in countup(0, sonsLen(etyp.n) - 1): + if etyp.kind in {tyBool, tyEnum} and sfPure notin s.flags: + for j in countup(0, sonsLen(etyp.n) - 1): var e = etyp.n.sons[j].sym - if (e.Kind != skEnumField): + if e.Kind != skEnumField: InternalError(s.info, "rawImportSymbol") # BUGFIX: because of aliases for enums the symbol may already # have been put into the symbol table # BUGFIX: but only iff they are the same symbols! var it: TIdentIter check = InitIdentIter(it, c.tab.stack[importTablePos], e.name) - while check != nil: - if check.id == e.id: + while check != nil: + if check.id == e.id: e = nil - break + break check = NextIdentIter(it, c.tab.stack[importTablePos]) - if e != nil: + if e != nil: rawImportSymbol(c, e) else: # rodgen assures that converters and patterns are no stubs if s.kind == skConverter: addConverter(c, s) if hasPattern(s): addPattern(c, s) - + proc importSymbol(c: PContext, n: PNode, fromMod: PSym) = let ident = lookups.considerAcc(n) let s = StrTableGet(fromMod.tab, ident) @@ -98,17 +98,6 @@ proc importSymbol(c: PContext, n: PNode, fromMod: PSym) = rawImportSymbol(c, e) e = NextIdentIter(it, fromMod.tab) else: rawImportSymbol(c, s) - -proc importAllSymbols(c: PContext, fromMod: PSym) = - var i: TTabIter - var s = InitTabIter(i, fromMod.tab) - while s != nil: - if s.kind != skModule: - if s.kind != skEnumField: - if s.Kind notin ExportableSymKinds: - InternalError(s.info, "importAllSymbols: " & $s.kind) - rawImportSymbol(c, s) # this is correct! - s = NextIter(i, fromMod.tab) proc importAllSymbolsExcept(c: PContext, fromMod: PSym, exceptSet: TIntSet) = var i: TTabIter @@ -118,12 +107,34 @@ proc importAllSymbolsExcept(c: PContext, fromMod: PSym, exceptSet: TIntSet) = if s.kind != skEnumField: if s.Kind notin ExportableSymKinds: InternalError(s.info, "importAllSymbols: " & $s.kind) - if s.name.id notin exceptSet: + if exceptSet.empty or s.name.id notin exceptSet: rawImportSymbol(c, s) s = NextIter(i, fromMod.tab) +proc importAllSymbols*(c: PContext, fromMod: PSym) = + var exceptSet: TIntSet + importAllSymbolsExcept(c, fromMod, exceptSet) + +proc importForwarded(c: PContext, n: PNode, exceptSet: TIntSet) = + if n.isNil: return + case n.kind + of nkExportStmt: + for a in n: + assert a.kind == nkSym + let s = a.sym + if s.kind == skModule: + importAllSymbolsExcept(c, s, exceptSet) + elif exceptSet.empty or s.name.id notin exceptSet: + rawImportSymbol(c, s) + of nkExportExceptStmt: + localError(n.info, errGenerated, "'export except' not implemented") + else: + for i in 0 ..safeLen(n)-1: + importForwarded(c, n.sons[i], exceptSet) + proc evalImport(c: PContext, n: PNode): PNode = result = n + var emptySet: TIntSet for i in countup(0, sonsLen(n) - 1): var f = checkModuleName(n.sons[i]) if f.len > 0: @@ -132,7 +143,8 @@ proc evalImport(c: PContext, n: PNode): PNode = Message(n.sons[i].info, warnDeprecated, m.name.s) # ``addDecl`` needs to be done before ``importAllSymbols``! addDecl(c, m) # add symbol to symbol table of module - importAllSymbols(c, m) + importAllSymbolsExcept(c, m, emptySet) + importForwarded(c, m.ast, emptySet) proc evalFrom(c: PContext, n: PNode): PNode = result = n @@ -157,3 +169,4 @@ proc evalImportExcept*(c: PContext, n: PNode): PNode = let ident = lookups.considerAcc(n.sons[i]) exceptSet.incl(ident.id) importAllSymbolsExcept(c, m, exceptSet) + importForwarded(c, m.ast, exceptSet) diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index 9f2b431f6..7acfb830f 100755 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -1657,6 +1657,26 @@ proc fixImmediateParams(n: PNode): PNode = result = n +proc semExport(c: PContext, n: PNode): PNode = + var x = newNodeI(n.kind, n.info) + #let L = if n.kind == nkExportExceptStmt: L = 1 else: n.len + for i in 0.. Date: Sun, 2 Dec 2012 00:44:29 +0100 Subject: [PATCH 009/106] first version of ropes.nim with unsafeNew (broken) --- compiler/ccgexprs.nim | 15 +++++-- compiler/ccgmerge.nim | 37 +++++++++++++--- compiler/docgen.nim | 8 ++-- compiler/ropes.nim | 101 ++++++++++++++++++++++-------------------- lib/system.nim | 10 ++++- todo.txt | 2 +- 6 files changed, 110 insertions(+), 63 deletions(-) diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index 57d06a988..cb0dcfe00 100755 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -933,22 +933,31 @@ proc genNew(p: BProc, e: PNode) = var a, b: TLoc reftype, bt: PType + sizeExpr: PRope refType = skipTypes(e.sons[1].typ, abstractVarRange) InitLocExpr(p, e.sons[1], a) initLoc(b, locExpr, a.t, OnHeap) + # 'genNew' also handles 'unsafeNew': + if e.len == 3: + var se: TLoc + InitLocExpr(p, e.sons[2], se) + sizeExpr = se.rdLoc + else: + sizeExpr = ropef("sizeof($1)", + getTypeDesc(p.module, skipTypes(reftype.sons[0], abstractRange))) let args = [getTypeDesc(p.module, reftype), genTypeInfo(p.module, refType), - getTypeDesc(p.module, skipTypes(reftype.sons[0], abstractRange))] + sizeExpr] if a.s == OnHeap and optRefcGc in gGlobalOptions: # use newObjRC1 as an optimization; and we don't need 'keepAlive' either if canFormAcycle(a.t): lineCg(p, cpsStmts, "if ($1) #nimGCunref($1);$n", a.rdLoc) else: lineCg(p, cpsStmts, "if ($1) #nimGCunrefNoCycle($1);$n", a.rdLoc) - b.r = ropecg(p.module, "($1) #newObjRC1($2, sizeof($3))", args) + b.r = ropecg(p.module, "($1) #newObjRC1($2, $3)", args) lineCg(p, cpsStmts, "$1 = $2;$n", a.rdLoc, b.rdLoc) else: - b.r = ropecg(p.module, "($1) #newObj($2, sizeof($3))", args) + b.r = ropecg(p.module, "($1) #newObj($2, $3)", args) genAssignment(p, a, b, {needToKeepAlive}) # set the object type: bt = skipTypes(refType.sons[0], abstractRange) genObjectInit(p, cpsStmts, bt, a, false) diff --git a/compiler/ccgmerge.nim b/compiler/ccgmerge.nim index df7e7f68b..027cd5780 100644 --- a/compiler/ccgmerge.nim +++ b/compiler/ccgmerge.nim @@ -145,20 +145,47 @@ proc atEndMark(buf: cstring, pos: int): bool = while s < NimMergeEndMark.len and buf[pos+s] == NimMergeEndMark[s]: inc s result = s == NimMergeEndMark.len +when false: + proc readVerbatimSection(L: var TBaseLexer): PRope = + var pos = L.bufpos + var buf = L.buf + result = newMutableRope(30_000) + while true: + case buf[pos] + of CR: + pos = lexbase.HandleCR(L, pos) + buf = L.buf + result.data.add(tnl) + of LF: + pos = lexbase.HandleLF(L, pos) + buf = L.buf + result.data.add(tnl) + of '\0': + InternalError("ccgmerge: expected: " & NimMergeEndMark) + break + else: + if atEndMark(buf, pos): + inc pos, NimMergeEndMark.len + break + result.data.add(buf[pos]) + inc pos + L.bufpos = pos + freezeMutableRope(result) + proc readVerbatimSection(L: var TBaseLexer): PRope = var pos = L.bufpos var buf = L.buf - result = newMutableRope(30_000) + var r = newStringOfCap(30_000) while true: case buf[pos] of CR: pos = lexbase.HandleCR(L, pos) buf = L.buf - result.data.add(tnl) + r.add(tnl) of LF: pos = lexbase.HandleLF(L, pos) buf = L.buf - result.data.add(tnl) + r.add(tnl) of '\0': InternalError("ccgmerge: expected: " & NimMergeEndMark) break @@ -166,10 +193,10 @@ proc readVerbatimSection(L: var TBaseLexer): PRope = if atEndMark(buf, pos): inc pos, NimMergeEndMark.len break - result.data.add(buf[pos]) + r.add(buf[pos]) inc pos L.bufpos = pos - freezeMutableRope(result) + result = r.toRope proc readKey(L: var TBaseLexer, result: var string) = var pos = L.bufpos diff --git a/compiler/docgen.nim b/compiler/docgen.nim index 5d46be874..e06277f91 100755 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -350,9 +350,11 @@ proc CommandRstAux(filename, outExt: string) = var d = newDocumentor(filen, options.gConfigVars) var rst = parseRst(readFile(filen), filen, 0, 1, d.hasToc, {roSupportRawDirective}) - d.modDesc = newMutableRope(30_000) - renderRstToOut(d[], rst, d.modDesc.data) - freezeMutableRope(d.modDesc) + var modDesc = newStringOfCap(30_000) + #d.modDesc = newMutableRope(30_000) + renderRstToOut(d[], rst, modDesc) + #freezeMutableRope(d.modDesc) + d.modDesc = toRope(modDesc) writeOutput(d, filename, outExt) generateIndex(d) diff --git a/compiler/ropes.nim b/compiler/ropes.nim index 50c89e4d9..af05c68a7 100755 --- a/compiler/ropes.nim +++ b/compiler/ropes.nim @@ -58,6 +58,9 @@ import msgs, strutils, platform, hashes, crc, options +const + PayloadSize = 64_000 # dummy size for range checking + type TFormatStr* = string # later we may change it to CString for better # performance of the code generator (assignments @@ -66,9 +69,9 @@ type PRope* = ref TRope TRope*{.acyclic.} = object of TObject # the empty rope is represented # by nil to safe space - left*, right*: PRope - length*: int - data*: string # != nil if a leaf + left, right: PRope + L: int # < 0 if a leaf + d: array [0..PayloadSize, char] # != nil if a leaf TRopeSeq* = seq[PRope] @@ -94,22 +97,31 @@ proc RopeInvariant*(r: PRope): bool proc ropeLen(a: PRope): int = if a == nil: result = 0 - else: result = a.length + else: result = a.L.abs -proc newRope(data: string = nil): PRope = - new(result) - if data != nil: - result.length = len(data) - result.data = data +proc newRope(data: string = nil): PRope = + if data != nil: + unsafeNew(result, sizeof(TRope)-PayloadSize+len(data)) + result.L = -len(data) + # copy including '\0': + copyMem(addr result.d, cstring(data), len(data)) + else: + unsafeNew(result, sizeof(TRope)-PayloadSize) -proc newMutableRope*(capacity = 30): PRope = - ## creates a new rope that supports direct modifications of the rope's - ## 'data' and 'length' fields. - new(result) - result.data = newStringOfCap(capacity) +proc eqContent(r: PRope, s: string): bool = + assert r.L < 0 + if -r.L == s.len: + result = equalMem(addr(r.d), cstring(s), s.len) -proc freezeMutableRope*(r: PRope) {.inline.} = - r.length = r.data.len +when false: + proc newMutableRope*(capacity = 30): PRope = + ## creates a new rope that supports direct modifications of the rope's + ## 'data' and 'length' fields. + new(result) + result.data = newStringOfCap(capacity) + + proc freezeMutableRope*(r: PRope) {.inline.} = + r.length = r.data.len var cache: array[0..2048*2 -1, PRope] @@ -130,7 +142,7 @@ proc RopeInvariant(r: PRope): bool = proc insertInCache(s: string): PRope = var h = hash(s) and high(cache) result = cache[h] - if isNil(result) or result.data != s: + if isNil(result) or not eqContent(result, s): result = newRope(s) cache[h] = result @@ -155,19 +167,19 @@ proc newRecRopeToStr(result: var string, resultLen: var int, r: PRope) = var stack = @[r] while len(stack) > 0: var it = pop(stack) - while it.data == nil: + while it.L >= 0: add(stack, it.right) it = it.left - assert(it.data != nil) - CopyMem(addr(result[resultLen]), addr(it.data[0]), it.length) - Inc(resultLen, it.length) + assert(it.L < 0) + CopyMem(addr(result[resultLen]), addr(it.d[0]), -it.L) + Inc(resultLen, -it.L) assert(resultLen <= len(result)) proc ropeToStr(p: PRope): string = if p == nil: result = "" - else: - result = newString(p.length) + else: + result = newString(p.L.abs) var resultLen = 0 newRecRopeToStr(result, resultLen, p) @@ -176,7 +188,7 @@ proc con(a, b: PRope): PRope = elif b == nil: result = a else: result = newRope() - result.length = a.length + b.length + result.L = a.L.abs + b.L.abs result.left = a result.right = b @@ -192,16 +204,16 @@ proc app(a: var PRope, b: PRope) = a = con(a, b) proc app(a: var PRope, b: string) = a = con(a, b) proc prepend(a: var PRope, b: PRope) = a = con(b, a) -proc writeRope*(f: TFile, c: PRope) = +proc writeRope*(f: TFile, c: PRope) = var stack = @[c] - while len(stack) > 0: + while len(stack) > 0: var it = pop(stack) - while it.data == nil: + while it.L >= 0: add(stack, it.right) it = it.left assert(it != nil) - assert(it.data != nil) - write(f, it.data) + assert(it.L < 0) + write(f, cstring(it.d), -it.L) proc WriteRope*(head: PRope, filename: string, useWarning = false) = var f: tfile @@ -261,14 +273,14 @@ const bufSize = 1024 # 1 KB is reasonable proc auxRopeEqualsFile(r: PRope, bin: var tfile, buf: Pointer): bool = - if r.data != nil: - if r.length > bufSize: + if r.L < 0: + if -r.L > bufSize: internalError("ropes: token too long") return - var readBytes = readBuffer(bin, buf, r.length) - result = readBytes == r.length and - equalMem(buf, addr(r.data[0]), r.length) # BUGFIX - else: + var readBytes = readBuffer(bin, buf, -r.L) + result = readBytes == -r.L and + equalMem(buf, addr(r.d[0]), readBytes) + else: result = auxRopeEqualsFile(r.left, bin, buf) if result: result = auxRopeEqualsFile(r.right, bin, buf) @@ -284,29 +296,20 @@ proc RopeEqualsFile(r: PRope, f: string): bool = dealloc(buf) close(bin) -proc crcFromRopeAux(r: PRope, startVal: TCrc32): TCrc32 = - if r.data != nil: - result = startVal - for i in countup(0, len(r.data) - 1): - result = updateCrc32(r.data[i], result) - else: - result = crcFromRopeAux(r.left, startVal) - result = crcFromRopeAux(r.right, result) - proc newCrcFromRopeAux(r: PRope, startVal: TCrc32): TCrc32 = # XXX profiling shows this is actually expensive var stack: TRopeSeq = @[r] result = startVal while len(stack) > 0: var it = pop(stack) - while it.data == nil: + while it.L >= 0: add(stack, it.right) it = it.left - assert(it.data != nil) + assert(it.L < 0) var i = 0 - var L = len(it.data) - while i < L: - result = updateCrc32(it.data[i], result) + var L = -it.L + while i < L: + result = updateCrc32(it.d[i], result) inc(i) proc crcFromRope(r: PRope): TCrc32 = diff --git a/lib/system.nim b/lib/system.nim index 9ad99fb79..e199d7611 100755 --- a/lib/system.nim +++ b/lib/system.nim @@ -112,11 +112,17 @@ proc new*[T](a: var ref T) {.magic: "New", noSideEffect.} ## creates a new object of type ``T`` and returns a safe (traced) ## reference to it in ``a``. -proc new(T: typedesc): ref T = +proc new*(T: typedesc): ref T = ## creates a new object of type ``T`` and returns a safe (traced) ## reference to it as result value new(result) - + +proc unsafeNew*[T](a: var ref T, size: int) {.magic: "New", noSideEffect.} + ## creates a new object of type ``T`` and returns a safe (traced) + ## reference to it in ``a``. This is **unsafe** as it allocates an object + ## of the passed ``size``. This should only be used for optimization + ## purposes when you know what you're doing! + proc internalNew*[T](a: var ref T) {.magic: "New", noSideEffect.} ## leaked implementation detail. Do not use. diff --git a/todo.txt b/todo.txt index 61b178a23..5c2bea08f 100755 --- a/todo.txt +++ b/todo.txt @@ -50,7 +50,7 @@ Concurrency provide a ``syncgc`` pragma to trigger compiler injection --> more general: an ``injectLoop`` pragma - 'writes: []' effect; track reads/writes for shared types -- use the effect system for static deadlock prevention +- use the effect system for static deadlock prevention and race detection version 0.9.XX From d7adc7c3289feae7de617953452eb71fb75f40da Mon Sep 17 00:00:00 2001 From: Araq Date: Sun, 2 Dec 2012 11:11:54 +0100 Subject: [PATCH 010/106] dont use unsafeNew in ropes.nim for now --- compiler/ropes.nim | 725 +++++++++++++++++++++++++++++++-------------- todo.txt | 2 - web/news.txt | 1 + 3 files changed, 497 insertions(+), 231 deletions(-) diff --git a/compiler/ropes.nim b/compiler/ropes.nim index af05c68a7..f27bf70f7 100755 --- a/compiler/ropes.nim +++ b/compiler/ropes.nim @@ -66,54 +66,49 @@ type # performance of the code generator (assignments # copy the format strings # though it is not necessary) - PRope* = ref TRope - TRope*{.acyclic.} = object of TObject # the empty rope is represented - # by nil to safe space - left, right: PRope - L: int # < 0 if a leaf - d: array [0..PayloadSize, char] # != nil if a leaf - - TRopeSeq* = seq[PRope] -proc con*(a, b: PRope): PRope -proc con*(a: PRope, b: string): PRope -proc con*(a: string, b: PRope): PRope -proc con*(a: varargs[PRope]): PRope -proc app*(a: var PRope, b: PRope) -proc app*(a: var PRope, b: string) -proc prepend*(a: var PRope, b: PRope) -proc toRope*(s: string): PRope -proc toRope*(i: BiggestInt): PRope -proc ropeLen*(a: PRope): int -proc writeRopeIfNotEqual*(r: PRope, filename: string): bool -proc ropeToStr*(p: PRope): string -proc ropef*(frmt: TFormatStr, args: varargs[PRope]): PRope -proc appf*(c: var PRope, frmt: TFormatStr, args: varargs[PRope]) -proc RopeEqualsFile*(r: PRope, f: string): bool - # returns true if the rope r is the same as the contents of file f -proc RopeInvariant*(r: PRope): bool - # exported for debugging -# implementation +when true: + # working version: + type + PRope* = ref TRope + TRope*{.acyclic.} = object of TObject # the empty rope is represented + # by nil to safe space + left*, right*: PRope + length*: int + data*: string # != nil if a leaf + + TRopeSeq* = seq[PRope] -proc ropeLen(a: PRope): int = - if a == nil: result = 0 - else: result = a.L.abs - -proc newRope(data: string = nil): PRope = - if data != nil: - unsafeNew(result, sizeof(TRope)-PayloadSize+len(data)) - result.L = -len(data) - # copy including '\0': - copyMem(addr result.d, cstring(data), len(data)) - else: - unsafeNew(result, sizeof(TRope)-PayloadSize) + proc con*(a, b: PRope): PRope + proc con*(a: PRope, b: string): PRope + proc con*(a: string, b: PRope): PRope + proc con*(a: varargs[PRope]): PRope + proc app*(a: var PRope, b: PRope) + proc app*(a: var PRope, b: string) + proc prepend*(a: var PRope, b: PRope) + proc toRope*(s: string): PRope + proc toRope*(i: BiggestInt): PRope + proc ropeLen*(a: PRope): int + proc writeRopeIfNotEqual*(r: PRope, filename: string): bool + proc ropeToStr*(p: PRope): string + proc ropef*(frmt: TFormatStr, args: varargs[PRope]): PRope + proc appf*(c: var PRope, frmt: TFormatStr, args: varargs[PRope]) + proc RopeEqualsFile*(r: PRope, f: string): bool + # returns true if the rope r is the same as the contents of file f + proc RopeInvariant*(r: PRope): bool + # exported for debugging + # implementation -proc eqContent(r: PRope, s: string): bool = - assert r.L < 0 - if -r.L == s.len: - result = equalMem(addr(r.d), cstring(s), s.len) + proc ropeLen(a: PRope): int = + if a == nil: result = 0 + else: result = a.length + + proc newRope(data: string = nil): PRope = + new(result) + if data != nil: + result.length = len(data) + result.data = data -when false: proc newMutableRope*(capacity = 30): PRope = ## creates a new rope that supports direct modifications of the rope's ## 'data' and 'length' fields. @@ -123,204 +118,476 @@ when false: proc freezeMutableRope*(r: PRope) {.inline.} = r.length = r.data.len -var - cache: array[0..2048*2 -1, PRope] + var + cache: array[0..2048*2 -1, PRope] -proc RopeInvariant(r: PRope): bool = - if r == nil: - result = true - else: - result = true # - # if r.data <> snil then - # result := true - # else begin - # result := (r.left <> nil) and (r.right <> nil); - # if result then result := ropeInvariant(r.left); - # if result then result := ropeInvariant(r.right); - # end + proc RopeInvariant(r: PRope): bool = + if r == nil: + result = true + else: + result = true # + # if r.data <> snil then + # result := true + # else begin + # result := (r.left <> nil) and (r.right <> nil); + # if result then result := ropeInvariant(r.left); + # if result then result := ropeInvariant(r.right); + # end -proc insertInCache(s: string): PRope = - var h = hash(s) and high(cache) - result = cache[h] - if isNil(result) or not eqContent(result, s): - result = newRope(s) - cache[h] = result - -proc toRope(s: string): PRope = - if s.len == 0: - result = nil - else: - result = insertInCache(s) - assert(RopeInvariant(result)) + proc insertInCache(s: string): PRope = + var h = hash(s) and high(cache) + result = cache[h] + if isNil(result) or result.data != s: + result = newRope(s) + cache[h] = result + + proc toRope(s: string): PRope = + if s.len == 0: + result = nil + else: + result = insertInCache(s) + assert(RopeInvariant(result)) -proc RopeSeqInsert(rs: var TRopeSeq, r: PRope, at: Natural) = - var length = len(rs) - if at > length: - setlen(rs, at + 1) - else: - setlen(rs, length + 1) # move old rope elements: - for i in countdown(length, at + 1): - rs[i] = rs[i - 1] # this is correct, I used pen and paper to validate it - rs[at] = r + proc RopeSeqInsert(rs: var TRopeSeq, r: PRope, at: Natural) = + var length = len(rs) + if at > length: + setlen(rs, at + 1) + else: + setlen(rs, length + 1) # move old rope elements: + for i in countdown(length, at + 1): + rs[i] = rs[i - 1] # this is correct, I used pen and paper to validate it + rs[at] = r -proc newRecRopeToStr(result: var string, resultLen: var int, r: PRope) = - var stack = @[r] - while len(stack) > 0: - var it = pop(stack) - while it.L >= 0: - add(stack, it.right) - it = it.left - assert(it.L < 0) - CopyMem(addr(result[resultLen]), addr(it.d[0]), -it.L) - Inc(resultLen, -it.L) - assert(resultLen <= len(result)) + proc newRecRopeToStr(result: var string, resultLen: var int, r: PRope) = + var stack = @[r] + while len(stack) > 0: + var it = pop(stack) + while it.data == nil: + add(stack, it.right) + it = it.left + assert(it.data != nil) + CopyMem(addr(result[resultLen]), addr(it.data[0]), it.length) + Inc(resultLen, it.length) + assert(resultLen <= len(result)) -proc ropeToStr(p: PRope): string = - if p == nil: - result = "" - else: - result = newString(p.L.abs) - var resultLen = 0 - newRecRopeToStr(result, resultLen, p) + proc ropeToStr(p: PRope): string = + if p == nil: + result = "" + else: + result = newString(p.length) + var resultLen = 0 + newRecRopeToStr(result, resultLen, p) -proc con(a, b: PRope): PRope = - if a == nil: result = b - elif b == nil: result = a - else: - result = newRope() - result.L = a.L.abs + b.L.abs - result.left = a - result.right = b + proc con(a, b: PRope): PRope = + if a == nil: result = b + elif b == nil: result = a + else: + result = newRope() + result.length = a.length + b.length + result.left = a + result.right = b -proc con(a: PRope, b: string): PRope = result = con(a, toRope(b)) -proc con(a: string, b: PRope): PRope = result = con(toRope(a), b) + proc con(a: PRope, b: string): PRope = result = con(a, toRope(b)) + proc con(a: string, b: PRope): PRope = result = con(toRope(a), b) -proc con(a: varargs[PRope]): PRope = - for i in countup(0, high(a)): result = con(result, a[i]) + proc con(a: varargs[PRope]): PRope = + for i in countup(0, high(a)): result = con(result, a[i]) -proc toRope(i: BiggestInt): PRope = result = toRope($i) + proc toRope(i: BiggestInt): PRope = result = toRope($i) -proc app(a: var PRope, b: PRope) = a = con(a, b) -proc app(a: var PRope, b: string) = a = con(a, b) -proc prepend(a: var PRope, b: PRope) = a = con(b, a) + proc app(a: var PRope, b: PRope) = a = con(a, b) + proc app(a: var PRope, b: string) = a = con(a, b) + proc prepend(a: var PRope, b: PRope) = a = con(b, a) -proc writeRope*(f: TFile, c: PRope) = - var stack = @[c] - while len(stack) > 0: - var it = pop(stack) - while it.L >= 0: - add(stack, it.right) - it = it.left - assert(it != nil) - assert(it.L < 0) - write(f, cstring(it.d), -it.L) + proc writeRope*(f: TFile, c: PRope) = + var stack = @[c] + while len(stack) > 0: + var it = pop(stack) + while it.data == nil: + add(stack, it.right) + it = it.left + assert(it != nil) + assert(it.data != nil) + write(f, it.data) -proc WriteRope*(head: PRope, filename: string, useWarning = false) = - var f: tfile - if open(f, filename, fmWrite): - if head != nil: WriteRope(f, head) - close(f) - else: - rawMessage(if useWarning: warnCannotOpenFile else: errCannotOpenFile, - filename) + proc WriteRope*(head: PRope, filename: string, useWarning = false) = + var f: tfile + if open(f, filename, fmWrite): + if head != nil: WriteRope(f, head) + close(f) + else: + rawMessage(if useWarning: warnCannotOpenFile else: errCannotOpenFile, + filename) -proc ropef(frmt: TFormatStr, args: varargs[PRope]): PRope = - var i = 0 - var length = len(frmt) - result = nil - var num = 0 - while i <= length - 1: - if frmt[i] == '$': - inc(i) # skip '$' - case frmt[i] - of '$': - app(result, "$") - inc(i) - of '#': - inc(i) - app(result, args[num]) - inc(num) - of '0'..'9': - var j = 0 - while true: - j = (j * 10) + Ord(frmt[i]) - ord('0') - inc(i) - if (i > length + 0 - 1) or not (frmt[i] in {'0'..'9'}): break - num = j - if j > high(args) + 1: - internalError("ropes: invalid format string $" & $(j)) - else: - app(result, args[j - 1]) - of 'n': - if optLineDir notin gOptions: app(result, tnl) - inc i - of 'N': - app(result, tnl) - inc(i) - else: InternalError("ropes: invalid format string $" & frmt[i]) - var start = i - while i < length: - if frmt[i] != '$': inc(i) - else: break - if i - 1 >= start: - app(result, substr(frmt, start, i - 1)) - assert(RopeInvariant(result)) - -proc appf(c: var PRope, frmt: TFormatStr, args: varargs[PRope]) = - app(c, ropef(frmt, args)) - -const - bufSize = 1024 # 1 KB is reasonable - -proc auxRopeEqualsFile(r: PRope, bin: var tfile, buf: Pointer): bool = - if r.L < 0: - if -r.L > bufSize: - internalError("ropes: token too long") - return - var readBytes = readBuffer(bin, buf, -r.L) - result = readBytes == -r.L and - equalMem(buf, addr(r.d[0]), readBytes) - else: - result = auxRopeEqualsFile(r.left, bin, buf) - if result: result = auxRopeEqualsFile(r.right, bin, buf) - -proc RopeEqualsFile(r: PRope, f: string): bool = - var bin: tfile - result = open(bin, f) - if not result: - return # not equal if file does not exist - var buf = alloc(BufSize) - result = auxRopeEqualsFile(r, bin, buf) - if result: - result = readBuffer(bin, buf, bufSize) == 0 # really at the end of file? - dealloc(buf) - close(bin) - -proc newCrcFromRopeAux(r: PRope, startVal: TCrc32): TCrc32 = - # XXX profiling shows this is actually expensive - var stack: TRopeSeq = @[r] - result = startVal - while len(stack) > 0: - var it = pop(stack) - while it.L >= 0: - add(stack, it.right) - it = it.left - assert(it.L < 0) + proc ropef(frmt: TFormatStr, args: varargs[PRope]): PRope = var i = 0 - var L = -it.L - while i < L: - result = updateCrc32(it.d[i], result) - inc(i) + var length = len(frmt) + result = nil + var num = 0 + while i <= length - 1: + if frmt[i] == '$': + inc(i) # skip '$' + case frmt[i] + of '$': + app(result, "$") + inc(i) + of '#': + inc(i) + app(result, args[num]) + inc(num) + of '0'..'9': + var j = 0 + while true: + j = (j * 10) + Ord(frmt[i]) - ord('0') + inc(i) + if (i > length + 0 - 1) or not (frmt[i] in {'0'..'9'}): break + num = j + if j > high(args) + 1: + internalError("ropes: invalid format string $" & $(j)) + else: + app(result, args[j - 1]) + of 'n': + if optLineDir notin gOptions: app(result, tnl) + inc i + of 'N': + app(result, tnl) + inc(i) + else: InternalError("ropes: invalid format string $" & frmt[i]) + var start = i + while i < length: + if frmt[i] != '$': inc(i) + else: break + if i - 1 >= start: + app(result, substr(frmt, start, i - 1)) + assert(RopeInvariant(result)) -proc crcFromRope(r: PRope): TCrc32 = - result = newCrcFromRopeAux(r, initCrc32) + proc appf(c: var PRope, frmt: TFormatStr, args: varargs[PRope]) = + app(c, ropef(frmt, args)) -proc writeRopeIfNotEqual(r: PRope, filename: string): bool = - # returns true if overwritten - var c: TCrc32 - c = crcFromFile(filename) - if c != crcFromRope(r): - writeRope(r, filename) - result = true - else: - result = false + const + bufSize = 1024 # 1 KB is reasonable + + proc auxRopeEqualsFile(r: PRope, bin: var tfile, buf: Pointer): bool = + if r.data != nil: + if r.length > bufSize: + internalError("ropes: token too long") + return + var readBytes = readBuffer(bin, buf, r.length) + result = readBytes == r.length and + equalMem(buf, addr(r.data[0]), r.length) # BUGFIX + else: + result = auxRopeEqualsFile(r.left, bin, buf) + if result: result = auxRopeEqualsFile(r.right, bin, buf) + + proc RopeEqualsFile(r: PRope, f: string): bool = + var bin: tfile + result = open(bin, f) + if not result: + return # not equal if file does not exist + var buf = alloc(BufSize) + result = auxRopeEqualsFile(r, bin, buf) + if result: + result = readBuffer(bin, buf, bufSize) == 0 # really at the end of file? + dealloc(buf) + close(bin) + + proc crcFromRopeAux(r: PRope, startVal: TCrc32): TCrc32 = + if r.data != nil: + result = startVal + for i in countup(0, len(r.data) - 1): + result = updateCrc32(r.data[i], result) + else: + result = crcFromRopeAux(r.left, startVal) + result = crcFromRopeAux(r.right, result) + + proc newCrcFromRopeAux(r: PRope, startVal: TCrc32): TCrc32 = + # XXX profiling shows this is actually expensive + var stack: TRopeSeq = @[r] + result = startVal + while len(stack) > 0: + var it = pop(stack) + while it.data == nil: + add(stack, it.right) + it = it.left + assert(it.data != nil) + var i = 0 + var L = len(it.data) + while i < L: + result = updateCrc32(it.data[i], result) + inc(i) + + proc crcFromRope(r: PRope): TCrc32 = + result = newCrcFromRopeAux(r, initCrc32) + + proc writeRopeIfNotEqual(r: PRope, filename: string): bool = + # returns true if overwritten + var c: TCrc32 + c = crcFromFile(filename) + if c != crcFromRope(r): + writeRope(r, filename) + result = true + else: + result = false + +else: + # optimized but broken version: + + type + PRope* = ref TRope + TRope*{.acyclic.} = object of TObject # the empty rope is represented + # by nil to safe space + left, right: PRope + L: int # < 0 if a leaf + d: array [0..PayloadSize, char] # != nil if a leaf + + TRopeSeq* = seq[PRope] + + proc con*(a, b: PRope): PRope + proc con*(a: PRope, b: string): PRope + proc con*(a: string, b: PRope): PRope + proc con*(a: varargs[PRope]): PRope + proc app*(a: var PRope, b: PRope) + proc app*(a: var PRope, b: string) + proc prepend*(a: var PRope, b: PRope) + proc toRope*(s: string): PRope + proc toRope*(i: BiggestInt): PRope + proc ropeLen*(a: PRope): int + proc writeRopeIfNotEqual*(r: PRope, filename: string): bool + proc ropeToStr*(p: PRope): string + proc ropef*(frmt: TFormatStr, args: varargs[PRope]): PRope + proc appf*(c: var PRope, frmt: TFormatStr, args: varargs[PRope]) + proc RopeEqualsFile*(r: PRope, f: string): bool + # returns true if the rope r is the same as the contents of file f + proc RopeInvariant*(r: PRope): bool + # exported for debugging + # implementation + + proc ropeLen(a: PRope): int = + if a == nil: result = 0 + else: result = a.L.abs + + proc newRope(data: string = nil): PRope = + if data != nil: + unsafeNew(result, sizeof(TRope)-PayloadSize+len(data)+1) + result.L = -len(data) + # copy including '\0': + copyMem(addr result.d, cstring(data), len(data)) + else: + unsafeNew(result, sizeof(TRope)-PayloadSize+1) + + proc eqContent(r: PRope, s: string): bool = + assert r.L < 0 + if -r.L == s.len: + result = equalMem(addr(r.d), cstring(s), s.len) + + when false: + proc newMutableRope*(capacity = 30): PRope = + ## creates a new rope that supports direct modifications of the rope's + ## 'data' and 'length' fields. + new(result) + result.data = newStringOfCap(capacity) + + proc freezeMutableRope*(r: PRope) {.inline.} = + r.length = r.data.len + + var + cache: array[0..2048*2 -1, PRope] + + proc RopeInvariant(r: PRope): bool = + if r == nil: + result = true + else: + result = true # + # if r.data <> snil then + # result := true + # else begin + # result := (r.left <> nil) and (r.right <> nil); + # if result then result := ropeInvariant(r.left); + # if result then result := ropeInvariant(r.right); + # end + + proc insertInCache(s: string): PRope = + var h = hash(s) and high(cache) + result = cache[h] + if isNil(result) or not eqContent(result, s): + result = newRope(s) + cache[h] = result + + proc toRope(s: string): PRope = + if s.len == 0: + result = nil + else: + result = insertInCache(s) + assert(RopeInvariant(result)) + + proc RopeSeqInsert(rs: var TRopeSeq, r: PRope, at: Natural) = + var length = len(rs) + if at > length: + setlen(rs, at + 1) + else: + setlen(rs, length + 1) # move old rope elements: + for i in countdown(length, at + 1): + rs[i] = rs[i - 1] # this is correct, I used pen and paper to validate it + rs[at] = r + + proc newRecRopeToStr(result: var string, resultLen: var int, r: PRope) = + var stack = @[r] + while len(stack) > 0: + var it = pop(stack) + while it.L >= 0: + add(stack, it.right) + it = it.left + assert(it.L < 0) + CopyMem(addr(result[resultLen]), addr(it.d[0]), -it.L) + Inc(resultLen, -it.L) + assert(resultLen <= len(result)) + + proc ropeToStr(p: PRope): string = + if p == nil: + result = "" + else: + result = newString(p.L.abs) + var resultLen = 0 + newRecRopeToStr(result, resultLen, p) + + proc con(a, b: PRope): PRope = + if a == nil: result = b + elif b == nil: result = a + else: + result = newRope() + result.L = a.L.abs + b.L.abs + result.left = a + result.right = b + + proc con(a: PRope, b: string): PRope = result = con(a, toRope(b)) + proc con(a: string, b: PRope): PRope = result = con(toRope(a), b) + + proc con(a: varargs[PRope]): PRope = + for i in countup(0, high(a)): result = con(result, a[i]) + + proc toRope(i: BiggestInt): PRope = result = toRope($i) + + proc app(a: var PRope, b: PRope) = a = con(a, b) + proc app(a: var PRope, b: string) = a = con(a, b) + proc prepend(a: var PRope, b: PRope) = a = con(b, a) + + proc writeRope*(f: TFile, c: PRope) = + var stack = @[c] + while len(stack) > 0: + var it = pop(stack) + while it.L >= 0: + add(stack, it.right) + it = it.left + assert(it != nil) + assert(it.L < 0) + write(f, cstring(it.d), -it.L) + + proc WriteRope*(head: PRope, filename: string, useWarning = false) = + var f: tfile + if open(f, filename, fmWrite): + if head != nil: WriteRope(f, head) + close(f) + else: + rawMessage(if useWarning: warnCannotOpenFile else: errCannotOpenFile, + filename) + + proc ropef(frmt: TFormatStr, args: varargs[PRope]): PRope = + var i = 0 + var length = len(frmt) + result = nil + var num = 0 + while i <= length - 1: + if frmt[i] == '$': + inc(i) # skip '$' + case frmt[i] + of '$': + app(result, "$") + inc(i) + of '#': + inc(i) + app(result, args[num]) + inc(num) + of '0'..'9': + var j = 0 + while true: + j = (j * 10) + Ord(frmt[i]) - ord('0') + inc(i) + if (i > length + 0 - 1) or not (frmt[i] in {'0'..'9'}): break + num = j + if j > high(args) + 1: + internalError("ropes: invalid format string $" & $(j)) + else: + app(result, args[j - 1]) + of 'n': + if optLineDir notin gOptions: app(result, tnl) + inc i + of 'N': + app(result, tnl) + inc(i) + else: InternalError("ropes: invalid format string $" & frmt[i]) + var start = i + while i < length: + if frmt[i] != '$': inc(i) + else: break + if i - 1 >= start: + app(result, substr(frmt, start, i - 1)) + assert(RopeInvariant(result)) + + proc appf(c: var PRope, frmt: TFormatStr, args: varargs[PRope]) = + app(c, ropef(frmt, args)) + + const + bufSize = 1024 # 1 KB is reasonable + + proc auxRopeEqualsFile(r: PRope, bin: var tfile, buf: Pointer): bool = + if r.L < 0: + if -r.L > bufSize: + internalError("ropes: token too long") + return + var readBytes = readBuffer(bin, buf, -r.L) + result = readBytes == -r.L and + equalMem(buf, addr(r.d[0]), readBytes) + else: + result = auxRopeEqualsFile(r.left, bin, buf) + if result: result = auxRopeEqualsFile(r.right, bin, buf) + + proc RopeEqualsFile(r: PRope, f: string): bool = + var bin: tfile + result = open(bin, f) + if not result: + return # not equal if file does not exist + var buf = alloc(BufSize) + result = auxRopeEqualsFile(r, bin, buf) + if result: + result = readBuffer(bin, buf, bufSize) == 0 # really at the end of file? + dealloc(buf) + close(bin) + + proc newCrcFromRopeAux(r: PRope, startVal: TCrc32): TCrc32 = + # XXX profiling shows this is actually expensive + var stack: TRopeSeq = @[r] + result = startVal + while len(stack) > 0: + var it = pop(stack) + while it.L >= 0: + add(stack, it.right) + it = it.left + assert(it.L < 0) + var i = 0 + var L = -it.L + while i < L: + result = updateCrc32(it.d[i], result) + inc(i) + + proc crcFromRope(r: PRope): TCrc32 = + result = newCrcFromRopeAux(r, initCrc32) + + proc writeRopeIfNotEqual(r: PRope, filename: string): bool = + # returns true if overwritten + var c: TCrc32 + c = crcFromFile(filename) + if c != crcFromRope(r): + writeRope(r, filename) + result = true + else: + result = false diff --git a/todo.txt b/todo.txt index 5c2bea08f..ca2def6aa 100755 --- a/todo.txt +++ b/todo.txt @@ -15,8 +15,6 @@ version 0.9.2 - ``hoist`` pragma for loop hoisting: can be easily done with AST overloading + global -- implement ``system.unsafeNew``. - version 0.9.X ============= diff --git a/web/news.txt b/web/news.txt index fc54bbce2..1650bd562 100755 --- a/web/news.txt +++ b/web/news.txt @@ -17,6 +17,7 @@ Library Additions - Added ``system.onRaise`` to support a condition system. - Added ``macros.quote`` for AST quasi-quoting. +- Added ``system.unsafeNew`` to support hacky variable length objects. Changes affecting backwards compatibility From fa198f5be1a2867a0a2fd6972e0465cf9f8a66ff Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Sun, 2 Dec 2012 13:21:21 +0000 Subject: [PATCH 011/106] Fixes SSL httpclient and problems with hidden path splitting in OS. Added babel's libs dir to Nimrod's path. --- config/nimrod.cfg | 2 +- lib/pure/httpclient.nim | 31 ++++++++++++++++++++----------- lib/pure/os.nim | 3 ++- 3 files changed, 23 insertions(+), 13 deletions(-) diff --git a/config/nimrod.cfg b/config/nimrod.cfg index 564a0d4a6..3ae38133d 100755 --- a/config/nimrod.cfg +++ b/config/nimrod.cfg @@ -36,7 +36,7 @@ path="$lib/windows" path="$lib/posix" path="$lib/ecmas" path="$lib/pure/unidecode" -#recursivePath:"$home/.babel/lib" +path="$home/.babel/libs/" @if release or quick: obj_checks:off diff --git a/lib/pure/httpclient.nim b/lib/pure/httpclient.nim index 184bca867..462db91b1 100755 --- a/lib/pure/httpclient.nim +++ b/lib/pure/httpclient.nim @@ -238,8 +238,15 @@ type httpCONNECT ## Converts the request connection to a transparent ## TCP/IP tunnel, usually used for proxies. +when not defined(ssl): + type PSSLContext = ref object + let defaultSSLContext = nil +else: + let defaultSSLContext = newContext(verifyMode = CVerifyNone) + proc request*(url: string, httpMethod = httpGET, extraHeaders = "", - body = ""): TResponse = + body = "", + sslContext: PSSLContext = defaultSSLContext): TResponse = ## | Requests ``url`` with the specified ``httpMethod``. ## | Extra headers can be specified and must be seperated by ``\c\L`` var r = parseUrl(url) @@ -257,7 +264,7 @@ proc request*(url: string, httpMethod = httpGET, extraHeaders = "", var port = TPort(80) if r.scheme == "https": when defined(ssl): - s.wrapSocket(verifyMode = CVerifyNone) + sslContext.wrapSocket(s) else: raise newException(EHttpRequestErr, "SSL support was not compiled in. Cannot connect over SSL.") port = TPort(443) @@ -277,7 +284,7 @@ proc redirection(status: string): bool = if status.startsWith(i): return True -proc get*(url: string, maxRedirects = 5): TResponse = +proc get*(url: string, maxRedirects = 5, sslContext: PSSLContext = defaultSSLContext): TResponse = ## | GET's the ``url`` and returns a ``TResponse`` object ## | This proc also handles redirection result = request(url) @@ -285,24 +292,24 @@ proc get*(url: string, maxRedirects = 5): TResponse = if result.status.redirection(): var locationHeader = result.headers["Location"] if locationHeader == "": httpError("location header expected") - result = request(locationHeader) + result = request(locationHeader, sslContext = sslContext) -proc getContent*(url: string): string = +proc getContent*(url: string, sslContext: PSSLContext = defaultSSLContext): string = ## | GET's the body and returns it as a string. ## | Raises exceptions for the status codes ``4xx`` and ``5xx`` - var r = get(url) + var r = get(url, sslContext = sslContext) if r.status[0] in {'4','5'}: raise newException(EHTTPRequestErr, r.status) else: return r.body proc post*(url: string, extraHeaders = "", body = "", - maxRedirects = 5): TResponse = + maxRedirects = 5, sslContext: PSSLContext = defaultSSLContext): TResponse = ## | POST's ``body`` to the ``url`` and returns a ``TResponse`` object. ## | This proc adds the necessary Content-Length header. ## | This proc also handles redirection. var xh = extraHeaders & "Content-Length: " & $len(body) & "\c\L" - result = request(url, httpPOST, xh, body) + result = request(url, httpPOST, xh, body, sslContext) for i in 1..maxRedirects: if result.status.redirection(): var locationHeader = result.headers["Location"] @@ -310,7 +317,8 @@ proc post*(url: string, extraHeaders = "", body = "", var meth = if result.status != "307": httpGet else: httpPost result = request(locationHeader, meth, xh, body) -proc postContent*(url: string, extraHeaders = "", body = ""): string = +proc postContent*(url: string, extraHeaders = "", body = "", + sslContext: PSSLContext = defaultSSLContext): string = ## | POST's ``body`` to ``url`` and returns the response's body as a string ## | Raises exceptions for the status codes ``4xx`` and ``5xx`` var r = post(url, extraHeaders, body) @@ -319,11 +327,12 @@ proc postContent*(url: string, extraHeaders = "", body = ""): string = else: return r.body -proc downloadFile*(url: string, outputFilename: string) = +proc downloadFile*(url: string, outputFilename: string, + sslContext: PSSLContext = defaultSSLContext) = ## Downloads ``url`` and saves it to ``outputFilename`` var f: TFile if open(f, outputFilename, fmWrite): - f.write(getContent(url)) + f.write(getContent(url, sslContext)) f.close() else: fileError("Unable to open file") diff --git a/lib/pure/os.nim b/lib/pure/os.nim index a220c1da4..9513dbffb 100755 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -554,7 +554,8 @@ proc splitFile*(path: string): tuple[dir, name, ext: string] {. var dotPos = path.len for i in countdown(len(path)-1, 0): if path[i] == ExtSep: - if dotPos == path.len and i > 0: dotPos = i + if dotPos == path.len and i > 0 and + path[i-1] notin {dirsep, altsep}: dotPos = i elif path[i] in {dirsep, altsep}: sepPos = i break From 0916137287b1d1468810f0ad4cf638eed5436f5e Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Sun, 2 Dec 2012 13:45:28 +0000 Subject: [PATCH 012/106] Fixes error in httpclient. --- lib/pure/httpclient.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pure/httpclient.nim b/lib/pure/httpclient.nim index 462db91b1..20b448123 100755 --- a/lib/pure/httpclient.nim +++ b/lib/pure/httpclient.nim @@ -240,7 +240,7 @@ type when not defined(ssl): type PSSLContext = ref object - let defaultSSLContext = nil + let defaultSSLContext: PSSLContext = nil else: let defaultSSLContext = newContext(verifyMode = CVerifyNone) From 8948a97151fc000414aee7cd4064881ef646a26c Mon Sep 17 00:00:00 2001 From: Araq Date: Sun, 2 Dec 2012 20:02:42 +0100 Subject: [PATCH 013/106] bugfix: 'not nil' and 'shared' types --- compiler/semtypes.nim | 9 ++++++++- lib/pure/sockets.nim | 2 ++ lib/wrappers/openssl.nim | 2 ++ tests/reject/tnotnil.nim | 12 ++++++++++-- todo.txt | 3 +-- 5 files changed, 23 insertions(+), 5 deletions(-) diff --git a/compiler/semtypes.nim b/compiler/semtypes.nim index 9efeba5b7..6478c1ec6 100755 --- a/compiler/semtypes.nim +++ b/compiler/semtypes.nim @@ -787,6 +787,12 @@ proc semTypeExpr(c: PContext, n: PNode): PType = else: LocalError(n.info, errTypeExpected, n.renderTree) +proc freshType(res, prev: PType): PType {.inline.} = + if prev.isNil: + result = copyType(result, result.owner, keepId=false) + else: + result = res + proc semTypeNode(c: PContext, n: PNode, prev: PType): PType = result = nil if gCmd == cmdIdeTools: suggestExpr(c, n) @@ -825,7 +831,7 @@ proc semTypeNode(c: PContext, n: PNode, prev: PType): PType = checkSonsLen(n, 3) result = semTypeNode(c, n.sons[1], prev) if result.kind in NilableTypes and n.sons[2].kind == nkNilLit: - # XXX this is wrong for tyString at least + result = freshType(result, prev) result.flags.incl(tfNotNil) else: LocalError(n.info, errGenerated, "invalid type") @@ -920,6 +926,7 @@ proc semTypeNode(c: PContext, n: PNode, prev: PType): PType = of nkSharedTy: checkSonsLen(n, 1) result = semTypeNode(c, n.sons[0], prev) + result = freshType(result, prev) result.flags.incl(tfShared) else: LocalError(n.info, errTypeExpected) diff --git a/lib/pure/sockets.nim b/lib/pure/sockets.nim index 371641b06..6a3c644bd 100755 --- a/lib/pure/sockets.nim +++ b/lib/pure/sockets.nim @@ -14,6 +14,8 @@ ## For OpenSSL support compile with ``-d:ssl``. When using SSL be aware that ## most functions will then raise ``ESSL`` on SSL errors. +{.deadCodeElim: on.} + when hostos == "solaris": {.passl: "-lsocket -lnsl".} diff --git a/lib/wrappers/openssl.nim b/lib/wrappers/openssl.nim index 1063f401d..752e594d1 100755 --- a/lib/wrappers/openssl.nim +++ b/lib/wrappers/openssl.nim @@ -39,6 +39,8 @@ ## OpenSSL support +{.deadCodeElim: on.} + when defined(WINDOWS): const DLLSSLName = "(ssleay32|libssl32).dll" diff --git a/tests/reject/tnotnil.nim b/tests/reject/tnotnil.nim index 8676aedf8..b02e33713 100644 --- a/tests/reject/tnotnil.nim +++ b/tests/reject/tnotnil.nim @@ -1,5 +1,5 @@ discard """ - line: 11 + line: 22 errormgs: "type mismatch" """ @@ -7,9 +7,17 @@ type PObj = ref TObj not nil TObj = object x: int + + MyString = string not nil -var x: PObj = nil +#var x: PObj = nil proc p(x: string not nil): int = result = 45 +proc q(x: MyString) = nil +proc q2(x: string) = nil + +q2(nil) +q(nil) + diff --git a/todo.txt b/todo.txt index ca2def6aa..f736ce8dd 100755 --- a/todo.txt +++ b/todo.txt @@ -1,7 +1,6 @@ version 0.9.2 ============= -- fix tfShared and tfNotNil - test&finish first class iterators: * nested iterators * test generic iterators @@ -20,7 +19,7 @@ version 0.9.X ============= - implement the missing features wrt inheritance -- implement generic methods +- implement generic methods and generic converters - improve the compiler as a service - ``=`` should be overloadable; requires specialization for ``=`` - implement constructors + full 'not nil' checking From 08342240709690f81286aea207a037348e46f635 Mon Sep 17 00:00:00 2001 From: Araq Date: Mon, 3 Dec 2012 21:48:12 +0100 Subject: [PATCH 014/106] disable SSLv2_method for ubuntu --- compiler/sigmatch.nim | 6 +++--- lib/pure/sockets.nim | 5 ++++- todo.txt | 11 ++++++----- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/compiler/sigmatch.nim b/compiler/sigmatch.nim index 799622355..8913b6d5d 100755 --- a/compiler/sigmatch.nim +++ b/compiler/sigmatch.nim @@ -837,10 +837,10 @@ proc matchesAux*(c: PContext, n, nOrig: PNode, m.baseTypeMatch = false var arg = ParamTypesMatch(c, m, formal.typ, n.sons[a].typ, n.sons[a], nOrig.sons[a]) - if arg == nil: + if arg == nil: m.state = csNoMatch - return - if m.baseTypeMatch: + return + if m.baseTypeMatch: assert(container == nil) container = newNodeI(nkBracket, n.sons[a].info) addSon(container, arg) diff --git a/lib/pure/sockets.nim b/lib/pure/sockets.nim index 6a3c644bd..01b97197e 100755 --- a/lib/pure/sockets.nim +++ b/lib/pure/sockets.nim @@ -258,7 +258,10 @@ when defined(ssl): of protSSLv23: newCTX = SSL_CTX_new(SSLv23_method()) # SSlv2,3 and TLS1 support. of protSSLv2: - newCTX = SSL_CTX_new(SSLv2_method()) + when not defined(linux): + newCTX = SSL_CTX_new(SSLv2_method()) + else: + SSLError() of protSSLv3: newCTX = SSL_CTX_new(SSLv3_method()) of protTLSv1: diff --git a/todo.txt b/todo.txt index f736ce8dd..b7147c235 100755 --- a/todo.txt +++ b/todo.txt @@ -1,11 +1,6 @@ version 0.9.2 ============= -- test&finish first class iterators: - * nested iterators - * test generic iterators - -- fix closure bug finally - overloading based on ASTs: 'constraint' should not be in PType but for the parameter *symbol* @@ -14,6 +9,12 @@ version 0.9.2 - ``hoist`` pragma for loop hoisting: can be easily done with AST overloading + global +- test&finish first class iterators: + * nested iterators + * test generic iterators +- fix closure bug finally +- fix marshal bug + version 0.9.X ============= From 7171ae62cb94bf5e7ed426ec0679e4247c993121 Mon Sep 17 00:00:00 2001 From: Araq Date: Wed, 5 Dec 2012 01:39:49 +0100 Subject: [PATCH 015/106] constraint now part of the parameter symbol and not of the type --- compiler/ast.nim | 2 +- compiler/parampatterns.nim | 4 ++-- compiler/patterns.nim | 4 ++-- compiler/rodread.nim | 6 +++--- compiler/rodwrite.nim | 6 +++--- compiler/semtypes.nim | 18 +++++++++++------- todo.txt | 6 +++--- 7 files changed, 25 insertions(+), 21 deletions(-) diff --git a/compiler/ast.nim b/compiler/ast.nim index f4ba1ef70..82a20c312 100755 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -624,6 +624,7 @@ type loc*: TLoc annex*: PLib # additional fields (seldom used, so we use a # reference to another object to safe space) + constraint*: PNode # additional constraints like 'lit|result' TTypeSeq* = seq[PType] TType* = object of TIdObj # types are identical iff they have the @@ -650,7 +651,6 @@ type align*: int # the type's alignment requirements containerID*: int # used for type checking of generics loc*: TLoc - constraint*: PNode # additional constraints like 'lit|result' TPair*{.final.} = object key*, val*: PObject diff --git a/compiler/parampatterns.nim b/compiler/parampatterns.nim index ee1f69818..21c7faf19 100644 --- a/compiler/parampatterns.nim +++ b/compiler/parampatterns.nim @@ -48,8 +48,8 @@ proc add(code: var TPatternCode, op: TOpcode) {.inline.} = add(code, chr(ord(op))) proc whichAlias*(p: PSym): TAliasRequest = - if p.typ.constraint != nil: - result = TAliasRequest(p.typ.constraint.strVal[0].ord) + if p.constraint != nil: + result = TAliasRequest(p.constraint.strVal[0].ord) proc compileConstraints(p: PNode, result: var TPatternCode) = case p.kind diff --git a/compiler/patterns.nim b/compiler/patterns.nim index af259c916..b7792100f 100644 --- a/compiler/patterns.nim +++ b/compiler/patterns.nim @@ -71,8 +71,8 @@ proc inSymChoice(sc, x: PNode): bool = proc checkTypes(c: PPatternContext, p: PSym, n: PNode): bool = # check param constraints first here as this is quite optimized: - if p.typ.constraint != nil: - result = matchNodeKinds(p.typ.constraint, n) + if p.constraint != nil: + result = matchNodeKinds(p.constraint, n) if not result: return if isNil(n.typ): result = p.typ.kind in {tyEmpty, tyStmt} diff --git a/compiler/rodread.nim b/compiler/rodread.nim index 4461641db..722887299 100755 --- a/compiler/rodread.nim +++ b/compiler/rodread.nim @@ -330,9 +330,6 @@ proc decodeType(r: PRodReader, info: TLineInfo): PType = if r.s[r.pos] == '@': inc(r.pos) result.containerID = decodeVInt(r.s, r.pos) - if r.s[r.pos] == '`': - inc(r.pos) - result.constraint = decodeNode(r, UnknownLineInfo()) decodeLoc(r, result.loc, info) while r.s[r.pos] == '^': inc(r.pos) @@ -423,6 +420,9 @@ proc decodeSym(r: PRodReader, info: TLineInfo): PSym = result.offset = - 1 decodeLoc(r, result.loc, result.info) result.annex = decodeLib(r, info) + if r.s[r.pos] == '#': + inc(r.pos) + result.constraint = decodeNode(r, UnknownLineInfo()) if r.s[r.pos] == '(': if result.kind in routineKinds: result.ast = decodeNodeLazyBody(r, result.info, result) diff --git a/compiler/rodwrite.nim b/compiler/rodwrite.nim index 5be9a2439..0a361d4dd 100755 --- a/compiler/rodwrite.nim +++ b/compiler/rodwrite.nim @@ -233,9 +233,6 @@ proc encodeType(w: PRodWriter, t: PType, result: var string) = if t.containerID != 0: add(result, '@') encodeVInt(t.containerID, result) - if t.constraint != nil: - add(result, '`') - encodeNode(w, UnknownLineInfo(), t.constraint, result) encodeLoc(w, t.loc, result) for i in countup(0, sonsLen(t) - 1): if t.sons[i] == nil: @@ -295,6 +292,9 @@ proc encodeSym(w: PRodWriter, s: PSym, result: var string) = encodeVInt(s.offset, result) encodeLoc(w, s.loc, result) if s.annex != nil: encodeLib(w, s.annex, s.info, result) + if s.constraint != nil: + add(result, '#') + encodeNode(w, UnknownLineInfo(), s.constraint, result) # lazy loading will soon reload the ast lazily, so the ast needs to be # the last entry of a symbol: if s.ast != nil: diff --git a/compiler/semtypes.nim b/compiler/semtypes.nim index 6478c1ec6..3da2a95b7 100755 --- a/compiler/semtypes.nim +++ b/compiler/semtypes.nim @@ -635,6 +635,13 @@ proc liftParamType(c: PContext, procKind: TSymKind, genericParams: PNode, genericParams.addSon(newSymNode(s)) result = typeClass +proc semParamType(c: PContext, n: PNode, constraint: var PNode): PType = + if n.kind == nkCurlyExpr: + result = semTypeNode(c, n.sons[0], nil) + constraint = semNodeKindConstraints(n) + else: + result = semTypeNode(c, n, nil) + proc semProcTypeNode(c: PContext, n, genericParams: PNode, prev: PType, kind: TSymKind): PType = var @@ -660,13 +667,14 @@ proc semProcTypeNode(c: PContext, n, genericParams: PNode, checkMinSonsLen(a, 3) var typ: PType = nil - def: PNode = nil + def: PNode = nil + constraint: PNode = nil length = sonsLen(a) hasType = a.sons[length-2].kind != nkEmpty hasDefault = a.sons[length-1].kind != nkEmpty if hasType: - typ = semTypeNode(c, a.sons[length-2], nil) + typ = semParamType(c, a.sons[length-2], constraint) if hasDefault: def = semExprWithType(c, a.sons[length-1]) @@ -689,6 +697,7 @@ proc semProcTypeNode(c: PContext, n, genericParams: PNode, arg.name.s, arg.info).skipIntLit arg.typ = finalType arg.position = counter + arg.constraint = constraint inc(counter) if def != nil and def.kind != nkEmpty: arg.ast = copyTree(def) if ContainsOrIncl(check, arg.name.id): @@ -839,11 +848,6 @@ proc semTypeNode(c: PContext, n: PNode, prev: PType): PType = result = semTypeExpr(c, n) else: result = semTypeExpr(c, n) - of nkCurlyExpr: - result = semTypeNode(c, n.sons[0], nil) - if result != nil: - result = copyType(result, getCurrOwner(), true) - result.constraint = semNodeKindConstraints(n) of nkWhenStmt: var whenResult = semWhen(c, n, false) if whenResult.kind == nkStmtList: whenResult.kind = nkStmtListType diff --git a/todo.txt b/todo.txt index b7147c235..3d3ba8346 100755 --- a/todo.txt +++ b/todo.txt @@ -1,8 +1,8 @@ version 0.9.2 ============= -- overloading based on ASTs: 'constraint' should not be in PType but for the - parameter *symbol* +- overloading based on ASTs +- implement generic converters - implement ``partial`` pragma for partial evaluation: easily done with AST overloading @@ -20,7 +20,7 @@ version 0.9.X ============= - implement the missing features wrt inheritance -- implement generic methods and generic converters +- implement generic methods - improve the compiler as a service - ``=`` should be overloadable; requires specialization for ``=`` - implement constructors + full 'not nil' checking From 38ab30d153f7ae3b1b2b74329092ffb7ca781ead Mon Sep 17 00:00:00 2001 From: Araq Date: Wed, 5 Dec 2012 22:03:36 +0100 Subject: [PATCH 016/106] implemented generic converters --- compiler/semcall.nim | 12 +++++++++++ compiler/semstmts.nim | 4 ++-- compiler/sigmatch.nim | 19 +++++++++++++--- tests/compile/tsecondarrayproperty.nim | 28 ++++++++++++++++++++++++ tests/run/tgenericconverter.nim | 30 ++++++++++++++++++++++++++ todo.txt | 2 -- web/news.txt | 1 + 7 files changed, 89 insertions(+), 7 deletions(-) create mode 100644 tests/compile/tsecondarrayproperty.nim create mode 100644 tests/run/tgenericconverter.nim diff --git a/compiler/semcall.nim b/compiler/semcall.nim index a5107bf64..962e4d3cc 100755 --- a/compiler/semcall.nim +++ b/compiler/semcall.nim @@ -84,6 +84,16 @@ proc resolveOverloads(c: PContext, n, orig: PNode, getProcHeader(best.calleeSym), getProcHeader(alt.calleeSym), args]) +proc instantiateGenericConverters(c: PContext, n: PNode, x: TCandidate) {. + noinline.}= + for i in 1 .. Date: Wed, 5 Dec 2012 22:58:52 +0100 Subject: [PATCH 017/106] fixes #266 --- compiler/semstmts.nim | 2 -- lib/pure/fsmonitor.nim | 4 +--- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index c5b226004..8b55058bb 100755 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -869,8 +869,6 @@ proc semMethod(c: PContext, n: PNode): PNode = proc semConverterDef(c: PContext, n: PNode): PNode = if not isTopLevel(c): LocalError(n.info, errXOnlyAtModuleScope, "converter") checkSonsLen(n, bodyPos + 1) - #if n.sons[genericParamsPos].kind != nkEmpty: - # LocalError(n.info, errNoGenericParamsAllowedForX, "converter") result = semProcAux(c, n, skConverter, converterPragmas) var s = result.sons[namePos].sym var t = s.typ diff --git a/lib/pure/fsmonitor.nim b/lib/pure/fsmonitor.nim index 92a80425a..a554cf963 100644 --- a/lib/pure/fsmonitor.nim +++ b/lib/pure/fsmonitor.nim @@ -17,9 +17,7 @@ ## module will therefore not work with any Linux kernel prior to that, unless ## it has been patched to support inotify. -when defined(windows): - {.error: "Windows is not yet supported by this module.".} -elif defined(linux): +when defined(linux) or defined(nimdoc): from posix import read else: {.error: "Your platform is not supported.".} From 6431e602160a5f2e1323a057e9c0166431e5a002 Mon Sep 17 00:00:00 2001 From: Araq Date: Wed, 5 Dec 2012 23:28:35 +0100 Subject: [PATCH 018/106] fixes #269 --- compiler/docgen.nim | 3 +++ compiler/semstmts.nim | 2 ++ 2 files changed, 5 insertions(+) diff --git a/compiler/docgen.nim b/compiler/docgen.nim index e06277f91..aff77cc1f 100755 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -145,6 +145,9 @@ proc isVisible(n: PNode): bool = var v = n.sons[0].ident result = v.id == ord(wStar) or v.id == ord(wMinus) elif n.kind == nkSym: + # we cannot generate code for forwarded symbols here as we have no + # exception tracking information here. Instead we copy over the comment + # from the proc header. result = {sfExported, sfFromGeneric, sfForward}*n.sym.flags == {sfExported} elif n.kind == nkPragmaExpr: result = isVisible(n.sons[0]) diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index 8b55058bb..68d485f48 100755 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -780,6 +780,8 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind, n.sons[pragmasPos] = proto.ast.sons[pragmasPos] if n.sons[namePos].kind != nkSym: InternalError(n.info, "semProcAux") n.sons[namePos].sym = proto + if gCmd == cmdDoc and not isNil(proto.ast.comment): + n.comment = proto.ast.comment proto.ast = n # needed for code generation popOwner() pushOwner(s) From a1f677980270d668b9b22e75a9892e100cc2a6d8 Mon Sep 17 00:00:00 2001 From: Araq Date: Thu, 6 Dec 2012 08:45:18 +0100 Subject: [PATCH 019/106] implemented AST based overloading --- compiler/procfind.nim | 4 ++-- compiler/sigmatch.nim | 19 +++++++++++++++---- compiler/types.nim | 3 ++- tests/run/tastoverload1.nim | 21 +++++++++++++++++++++ todo.txt | 1 + 5 files changed, 41 insertions(+), 7 deletions(-) create mode 100644 tests/run/tastoverload1.nim diff --git a/compiler/procfind.nim b/compiler/procfind.nim index fde4d22ea..4a1fb5ac8 100755 --- a/compiler/procfind.nim +++ b/compiler/procfind.nim @@ -29,7 +29,7 @@ proc equalGenericParams(procA, procB: PNode): bool = a = procA.sons[i].sym b = procB.sons[i].sym if (a.name.id != b.name.id) or - not sameTypeOrNil(a.typ, b.typ, {TypeDescExactMatch}): return + not sameTypeOrNil(a.typ, b.typ, {TypeDescExactMatch}): return if (a.ast != nil) and (b.ast != nil): if not ExprStructuralEquivalent(a.ast, b.ast): return result = true @@ -44,7 +44,7 @@ proc SearchForProc*(c: PContext, fn: PSym, tos: int): PSym = if equalGenericParams(result.ast.sons[genericParamsPos], fn.ast.sons[genericParamsPos]): case equalParams(result.typ.n, fn.typ.n) - of paramsEqual: + of paramsEqual: return of paramsIncompatible: LocalError(fn.info, errNotOverloadable, fn.name.s) diff --git a/compiler/sigmatch.nim b/compiler/sigmatch.nim index c48434eb7..1f2511694 100755 --- a/compiler/sigmatch.nim +++ b/compiler/sigmatch.nim @@ -12,18 +12,18 @@ import intsets, ast, astalgo, semdata, types, msgs, renderer, lookups, semtypinst, - magicsys, condsyms, idents, lexer, options + magicsys, condsyms, idents, lexer, options, parampatterns type TCandidateState* = enum csEmpty, csMatch, csNoMatch TCandidate* {.final.} = object - exactMatches*: int + exactMatches*: int # also misused to prefer iters over procs + genericMatches: int # also misused to prefer constraints subtypeMatches: int intConvMatches: int # conversions to int are not as expensive convMatches: int - genericMatches: int state*: TCandidateState callee*: PType # may not be nil! calleeSym*: PSym # may be nil @@ -773,6 +773,15 @@ proc setSon(father: PNode, at: int, son: PNode) = proc matchesAux*(c: PContext, n, nOrig: PNode, m: var TCandidate, marker: var TIntSet) = + template checkConstraint(n: expr) {.immediate, dirty.} = + if not formal.constraint.isNil: + if matchNodeKinds(formal.constraint, n): + # better match over other routines with no such restriction: + inc(m.genericMatches, 100) + else: + m.state = csNoMatch + return + var f = 1 # iterates over formal parameters var a = 1 # iterates over the actual given arguments m.state = csMatch # until proven otherwise @@ -805,7 +814,8 @@ proc matchesAux*(c: PContext, n, nOrig: PNode, n.sons[a].sons[1], nOrig.sons[a].sons[1]) if arg == nil: m.state = csNoMatch - return + return + checkConstraint(n.sons[a].sons[1]) if m.baseTypeMatch: assert(container == nil) container = newNodeI(nkBracket, n.sons[a].info) @@ -862,6 +872,7 @@ proc matchesAux*(c: PContext, n, nOrig: PNode, if f != formalLen - 1: container = nil else: setSon(m.call, formal.position + 1, arg) + checkConstraint(n.sons[a]) inc(a) inc(f) diff --git a/compiler/types.nim b/compiler/types.nim index 825b1027a..23d202fdf 100755 --- a/compiler/types.nim +++ b/compiler/types.nim @@ -632,7 +632,8 @@ proc SameTypeOrNil*(a, b: PType, flags: TTypeCmpFlags = {}): bool = result = SameTypeAux(a, b, c) proc equalParam(a, b: PSym): TParamsEquality = - if SameTypeOrNil(a.typ, b.typ, {TypeDescExactMatch}): + if SameTypeOrNil(a.typ, b.typ, {TypeDescExactMatch}) and + ExprStructuralEquivalent(a.constraint, b.constraint): if a.ast == b.ast: result = paramsEqual elif a.ast != nil and b.ast != nil: diff --git a/tests/run/tastoverload1.nim b/tests/run/tastoverload1.nim new file mode 100644 index 000000000..c8705547a --- /dev/null +++ b/tests/run/tastoverload1.nim @@ -0,0 +1,21 @@ +discard """ + output: '''string literal +no string literal +no string literal''' +""" + +proc optLit(a: string{lit}) = + echo "string literal" + +proc optLit(a: string) = + echo "no string literal" + +const + constant = "abc" + +var + variable = "xyz" + +optLit("literal") +optLit(constant) +optLit(variable) diff --git a/todo.txt b/todo.txt index 870265ac3..c5b5eb4e7 100755 --- a/todo.txt +++ b/todo.txt @@ -6,6 +6,7 @@ version 0.9.2 overloading - ``hoist`` pragma for loop hoisting: can be easily done with AST overloading + global + - document overloading based on ASTs - test&finish first class iterators: * nested iterators From a16e6bd22a4c238765bcf692ddfd3f86c39c4612 Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Thu, 6 Dec 2012 19:46:24 +0000 Subject: [PATCH 020/106] Updated readme. --- readme.md | 61 ++++++++++++++++++++++++++++++++++++++++ readme.txt | 82 ++++++++++++++++++++++++++++++++++++++++-------------- 2 files changed, 122 insertions(+), 21 deletions(-) create mode 100644 readme.md diff --git a/readme.md b/readme.md new file mode 100644 index 000000000..1391c530a --- /dev/null +++ b/readme.md @@ -0,0 +1,61 @@ +# Nimrod Compiler +This repo contains the Nimrod compiler, Nimrod's stdlib, tools and documentation. + +## Compiling +Compiling the Nimrod compiler is quite straightforward. Because +the Nimrod compiler itself is written in the Nimrod programming language +the C source of an older version of the compiler are needed to bootstrap the +latest version. The C sources are however included with this repository under +the build directory. + +Pre-compiled snapshots of the compiler are also available on +[Nimbuild](http://build.nimrod-code.org/). Your platform however may not currently +be built for. + +The compiler currently supports the following platform and architecture combinations: + + * Windows (Windows XP or greater) - x86 and x86_64 + * Linux (most, if not all, distributions) - x86, x86_64, ppc64 and armv6l + * Mac OS X 10.04 or higher - x86, x86_64 and ppc64 + +In reality a lot more are supported, however they are not tested regularly. + +To build from source you will need: + + * gcc 3.x or later recommended. Other alternatives which may work + are: clang, Visual C++, Intel's C++ compiler + * unzip + * git or wget + +If you are on a fairly modern *nix system, the following steps should work: + +``` +$ git clone git://github.com/Araq/Nimrod.git +$ cd Nimrod +$ unzip build/csources.zip +$ ./build.sh +$ bin/nimrod c koch +$ ./koch boot -d:release +``` + +The install script (``install.sh``) may then be used to install Nimrod, or you +can simply add it to your PATH. + +The above steps can be performed on Windows in a similar fashion, the +``build.bat`` and ``build64.bat`` (for x86_64 systems) are provided to be used +instead of ``build.sh``. + +## Getting help +A [forum](http://forum.nimrod-code.org/) is available if you have any questions, +and you can also get help in the IRC channel on [Freenode](irc://irc.freenode.net/nimrod) in #nimrod. + +## License +The compiler is licensed under the GPLv2 license, the standard library is +licensed under the LGPL license with a linking exception so that you can link +to it statically. This means that you can use any license for your own programs +developed with Nimrod, allowing you to create commercial applications. + +Read copying.txt for more details. + +Copyright (c) 2004-2012 Andreas Rumpf. +All rights reserved. \ No newline at end of file diff --git a/readme.txt b/readme.txt index c4d3b2ba7..1391c530a 100755 --- a/readme.txt +++ b/readme.txt @@ -1,21 +1,61 @@ -=========================================================== - Nimrod Compiler -=========================================================== - -This is the **Nimrod Compiler**. Nimrod is a new statically typed, imperative -programming language, that supports procedural, functional, object oriented and -generic programming styles while remaining simple and efficient. A special -feature that Nimrod inherited from Lisp is that Nimrod's abstract syntax tree -(AST) is part of the specification - this allows a powerful macro system which -can be used to create domain specific languages. - -*Nimrod* is a compiled, garbage-collected systems programming language -which has an excellent productivity/performance ratio. Nimrod's design -focuses on efficiency, expressiveness, elegance (in the order of -priority). - -See the file ``install.txt`` for installation instructions. See the file -``doc/intern.txt`` for the internal documentation for developers. - -Copyright (c) 2004-2012 Andreas Rumpf. -All rights reserved. +# Nimrod Compiler +This repo contains the Nimrod compiler, Nimrod's stdlib, tools and documentation. + +## Compiling +Compiling the Nimrod compiler is quite straightforward. Because +the Nimrod compiler itself is written in the Nimrod programming language +the C source of an older version of the compiler are needed to bootstrap the +latest version. The C sources are however included with this repository under +the build directory. + +Pre-compiled snapshots of the compiler are also available on +[Nimbuild](http://build.nimrod-code.org/). Your platform however may not currently +be built for. + +The compiler currently supports the following platform and architecture combinations: + + * Windows (Windows XP or greater) - x86 and x86_64 + * Linux (most, if not all, distributions) - x86, x86_64, ppc64 and armv6l + * Mac OS X 10.04 or higher - x86, x86_64 and ppc64 + +In reality a lot more are supported, however they are not tested regularly. + +To build from source you will need: + + * gcc 3.x or later recommended. Other alternatives which may work + are: clang, Visual C++, Intel's C++ compiler + * unzip + * git or wget + +If you are on a fairly modern *nix system, the following steps should work: + +``` +$ git clone git://github.com/Araq/Nimrod.git +$ cd Nimrod +$ unzip build/csources.zip +$ ./build.sh +$ bin/nimrod c koch +$ ./koch boot -d:release +``` + +The install script (``install.sh``) may then be used to install Nimrod, or you +can simply add it to your PATH. + +The above steps can be performed on Windows in a similar fashion, the +``build.bat`` and ``build64.bat`` (for x86_64 systems) are provided to be used +instead of ``build.sh``. + +## Getting help +A [forum](http://forum.nimrod-code.org/) is available if you have any questions, +and you can also get help in the IRC channel on [Freenode](irc://irc.freenode.net/nimrod) in #nimrod. + +## License +The compiler is licensed under the GPLv2 license, the standard library is +licensed under the LGPL license with a linking exception so that you can link +to it statically. This means that you can use any license for your own programs +developed with Nimrod, allowing you to create commercial applications. + +Read copying.txt for more details. + +Copyright (c) 2004-2012 Andreas Rumpf. +All rights reserved. \ No newline at end of file From 05b05be9f8554adc3081b71eb378940cb75428fd Mon Sep 17 00:00:00 2001 From: Araq Date: Thu, 6 Dec 2012 22:51:39 +0100 Subject: [PATCH 021/106] replaced recursivePath with shallowPath for Babel --- compiler/commands.nim | 5 ++--- doc/advopt.txt | 3 ++- todo.txt | 1 + 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/compiler/commands.nim b/compiler/commands.nim index 50b4a1e6f..ad5f2b7e7 100755 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -202,9 +202,8 @@ proc addPathRec(dir: string, info: TLineInfo) = if dir[pos] in {DirSep, AltSep}: inc(pos) for k,p in os.walkDir(dir): if k == pcDir and p[pos] != '.': - addPathRec(p, info) if not contains(options.searchPaths, p): - Message(info, hintPath, p) + #Message(info, hintPath, p) lists.PrependStr(options.searchPaths, p) proc track(arg: string, info: TLineInfo) = @@ -226,7 +225,7 @@ proc processSwitch(switch, arg: string, pass: TCmdlinePass, info: TLineInfo) = of "path", "p": expectArg(switch, arg, pass, info) addPath(processPath(arg), info) - of "recursivepath": + of "shallowpath": expectArg(switch, arg, pass, info) var path = processPath(arg) addPathRec(path, info) diff --git a/doc/advopt.txt b/doc/advopt.txt index a6718572e..2498df0bc 100755 --- a/doc/advopt.txt +++ b/doc/advopt.txt @@ -25,7 +25,8 @@ Advanced options: --warning[X]:on|off turn specific warning X on|off --hints:on|off turn all hints on|off --hint[X]:on|off turn specific hint X on|off - --recursivePath:PATH add a path and all of its subdirectories + --shallowPath:PATH add a path and all of its subdirectories (but + not recursively) --lib:PATH set the system library path --import:PATH add an automatically imported module --include:PATH add an automatically included module diff --git a/todo.txt b/todo.txt index c5b5eb4e7..66d76926e 100755 --- a/todo.txt +++ b/todo.txt @@ -108,3 +108,4 @@ Bugs use a qualifier - blocks can "export" an identifier but the CCG generates {} for them ... - JS gen: fix exception handling +- the better scoping for locals is the wrong default for endb From 1dc362dcd4e4752fce4d81800d965da8381790c8 Mon Sep 17 00:00:00 2001 From: Araq Date: Fri, 7 Dec 2012 17:20:51 +0100 Subject: [PATCH 022/106] compiler support for babel --- compiler/babelcmd.nim | 76 +++++++++++++++++++++++++++++++++++++++++++ compiler/commands.nim | 27 +++++---------- compiler/lists.nim | 9 ++++- doc/advopt.txt | 4 +-- lib/pure/strtabs.nim | 14 +++++++- 5 files changed, 108 insertions(+), 22 deletions(-) create mode 100644 compiler/babelcmd.nim diff --git a/compiler/babelcmd.nim b/compiler/babelcmd.nim new file mode 100644 index 000000000..99513e58b --- /dev/null +++ b/compiler/babelcmd.nim @@ -0,0 +1,76 @@ +# +# +# The Nimrod Compiler +# (c) Copyright 2012 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## Implements some helper procs for Babel (Nimrod's package manager) support. + +import parseutils, strtabs, os, options, msgs, lists + +proc addPath*(path: string, info: TLineInfo) = + if not contains(options.searchPaths, path): + lists.PrependStr(options.searchPaths, path) + +proc versionSplitPos(s: string): int = + result = s.len-2 + while result > 1 and s[result] in {'0'..'9', '.'}: dec result + if s[result] != '-': result = s.len + +const + latest = "head" + +proc `<.`(a, b: string): bool = + # wether a has a smaller version than b: + if a == latest: return false + var i = 0 + var j = 0 + var verA = 0 + var verB = 0 + while true: + let ii = parseInt(a, verA, i) + let jj = parseInt(b, verB, j) + # if A has no number left, but B has, B is prefered: 0.8 vs 0.8.3 + if ii <= 0 or jj <= 0: return jj > 0 + if verA < verB: return true + elif verA > verB: return false + # else: same version number; continue: + inc i, ii + inc j, jj + if a[i] == '.': inc i + if b[j] == '.': inc j + +proc addPackage(packages: PStringTable, p: string) = + let x = versionSplitPos(p) + let name = p.subStr(0, x-1) + if x < p.len: + let version = p.subStr(x+1) + if packages[name] <. version: + packages[name] = version + else: + packages[name] = latest + +iterator chosen(packages: PStringTable): string = + for key, val in pairs(packages): + let res = if val == latest: key else: key & '-' & val + yield res + +proc addPathRec(dir: string, info: TLineInfo) = + var packages = newStringTable(modeStyleInsensitive) + var pos = dir.len-1 + if dir[pos] in {DirSep, AltSep}: inc(pos) + for k,p in os.walkDir(dir): + if k == pcDir and p[pos] != '.': + addPackage(packages, p) + + for p in packages.chosen: + if not contains(options.searchPaths, p): + Message(info, hintPath, p) + lists.PrependStr(options.searchPaths, p) + +proc babelPath*(path: string, info: TLineInfo) = + addPathRec(path, info) + addPath(path, info) diff --git a/compiler/commands.nim b/compiler/commands.nim index ad5f2b7e7..4a72e25f3 100755 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -11,7 +11,7 @@ import os, msgs, options, nversion, condsyms, strutils, extccomp, platform, lists, - wordrecg, parseutils + wordrecg, parseutils, babelcmd proc writeCommandLineUsage*() @@ -193,19 +193,6 @@ proc processPath(path: string): string = "projectname", options.gProjectName, "projectpath", options.gProjectPath]) -proc addPath(path: string, info: TLineInfo) = - if not contains(options.searchPaths, path): - lists.PrependStr(options.searchPaths, path) - -proc addPathRec(dir: string, info: TLineInfo) = - var pos = dir.len-1 - if dir[pos] in {DirSep, AltSep}: inc(pos) - for k,p in os.walkDir(dir): - if k == pcDir and p[pos] != '.': - if not contains(options.searchPaths, p): - #Message(info, hintPath, p) - lists.PrependStr(options.searchPaths, p) - proc track(arg: string, info: TLineInfo) = var a = arg.split(',') if a.len != 3: LocalError(info, errTokenExpected, "FILE,LINE,COLUMN") @@ -225,11 +212,15 @@ proc processSwitch(switch, arg: string, pass: TCmdlinePass, info: TLineInfo) = of "path", "p": expectArg(switch, arg, pass, info) addPath(processPath(arg), info) - of "shallowpath": + of "babelpath": + if pass in {passCmd2, passPP}: + expectArg(switch, arg, pass, info) + let path = processPath(arg) + babelpath(path, info) + of "excludepath": expectArg(switch, arg, pass, info) - var path = processPath(arg) - addPathRec(path, info) - addPath(path, info) + let path = processPath(arg) + lists.ExcludeStr(options.searchPaths, path) of "nimcache": expectArg(switch, arg, pass, info) options.nimcacheDir = processPath(arg) diff --git a/compiler/lists.nim b/compiler/lists.nim index 1998581ce..9da4db177 100755 --- a/compiler/lists.nim +++ b/compiler/lists.nim @@ -89,7 +89,14 @@ proc Remove*(list: var TLinkedList, entry: PListEntry) = list.head = entry.next if entry.next != nil: entry.next.prev = entry.prev if entry.prev != nil: entry.prev.next = entry.next - + +proc ExcludeStr*(list: var TLinkedList, data: string) = + var it = list.head + while it != nil: + let nxt = it.next + if PStrEntry(it).data == data: remove(list, it) + it = nxt + proc Find*(list: TLinkedList, fn: TCompareProc, closure: Pointer): PListEntry = result = list.head while result != nil: diff --git a/doc/advopt.txt b/doc/advopt.txt index 2498df0bc..bd387e041 100755 --- a/doc/advopt.txt +++ b/doc/advopt.txt @@ -25,8 +25,6 @@ Advanced options: --warning[X]:on|off turn specific warning X on|off --hints:on|off turn all hints on|off --hint[X]:on|off turn specific hint X on|off - --shallowPath:PATH add a path and all of its subdirectories (but - not recursively) --lib:PATH set the system library path --import:PATH add an automatically imported module --include:PATH add an automatically included module @@ -65,6 +63,8 @@ Advanced options: --gc:refc|boehm|none use Nimrod's native GC|Boehm GC|no GC --index:on|off turn index file generation on|off --putenv:key=value set an environment variable + --babelPath:PATH add a path for Babel support + --excludePath:PATH exclude a path from the list of search paths --listCmd list the commands used to execute external programs --parallelBuild=0|1|... perform a parallel build value = number of processors (0 for auto-detect) diff --git a/lib/pure/strtabs.nim b/lib/pure/strtabs.nim index 09a243e97..77b463fc0 100755 --- a/lib/pure/strtabs.nim +++ b/lib/pure/strtabs.nim @@ -36,11 +36,23 @@ proc len*(t: PStringTable): int {.rtl, extern: "nst$1".} = result = t.counter iterator pairs*(t: PStringTable): tuple[key, value: string] = - ## iterates over any (key, value) pair in the table `t`. + ## iterates over every (key, value) pair in the table `t`. for h in 0..high(t.data): if not isNil(t.data[h].key): yield (t.data[h].key, t.data[h].val) +iterator keys*(t: PStringTable): string = + ## iterates over every key in the table `t`. + for h in 0..high(t.data): + if not isNil(t.data[h].key): + yield t.data[h].key + +iterator values*(t: PStringTable): string = + ## iterates over every value in the table `t`. + for h in 0..high(t.data): + if not isNil(t.data[h].key): + yield t.data[h].val + type TFormatFlag* = enum ## flags for the `%` operator useEnvironment, ## use environment variable if the ``$key`` From 40b611cc2f7d5cf4972dfb92fc63f0bb350adf89 Mon Sep 17 00:00:00 2001 From: Araq Date: Fri, 7 Dec 2012 17:43:15 +0100 Subject: [PATCH 023/106] documentation improvements --- doc/manual.txt | 296 ++++++++++++++++++++++++++++++++++++++++++++++++- readme.md | 13 ++- readme.txt | 13 ++- todo.txt | 4 +- 4 files changed, 309 insertions(+), 17 deletions(-) diff --git a/doc/manual.txt b/doc/manual.txt index 5ea3575ee..4d0c3b01b 100755 --- a/doc/manual.txt +++ b/doc/manual.txt @@ -593,7 +593,7 @@ widening type conversion are *implicit*: myInt16 + myInt # of type ``int`` myInt16 + 2i32 # of type ``int32`` -However, ``int`` literals are implicitely convertible to a smaller integer type +However, ``int`` literals are implicitly convertible to a smaller integer type if the literal's value fits this smaller type and such a conversion is less expensive than other implicit conversions, so ``myInt16 + 34`` produces an ``int16`` result. @@ -1847,7 +1847,7 @@ Syntax:: staticStmt ::= 'static' ':' stmt A `static`:idx: statement/expression can be used to enforce compile -time evaluation explicitely. Enforced compile time evaluation can even evaluate +time evaluation explicitly. Enforced compile time evaluation can even evaluate code that has side effects: .. code-block:: @@ -2479,7 +2479,7 @@ returned value is an l-value and can be modified by the caller: WriteAccessToG() = 6 assert g == 6 -It is a compile time error if the implicitely introduced pointer could be +It is a compile time error if the implicitly introduced pointer could be used to access a location beyond its lifetime: .. code-block:: nimrod @@ -3398,7 +3398,7 @@ template cannot be accessed in the instantiation context: template newException*(exceptn: typeDesc, message: string): expr = var - e: ref exceptn # e is implicitely gensym'ed here + e: ref exceptn # e is implicitly gensym'ed here new(e) e.msg = message e @@ -3699,8 +3699,294 @@ types that will match the typedesc param: The constraint can be a concrete type or a type class. +Term rewriting macros +===================== + +`Term rewriting macros`:idx: are macros or templates that have not only +a *name* but also a *pattern* that is searched for after the semantic checking +phase of the compiler: This means they provide an easy way to enhance the +compilation pipeline with user defined optimizations: + +.. code-block:: nimrod + template optMul{`*`(a, 2)}(a: int): int = a+a + + let x = 3 + echo x * 2 + +The compiler now rewrites ``x * 2`` as ``x + x``. The code inside the +curlies is the pattern to match against. The operators ``*``, ``**``, +``|``, ``~`` have a special meaning in patterns if they are written in infix +notation, so to match verbatim against ``*`` the ordinary function call syntax +needs to be used. + + +Unfortunately optimizations are hard to get right and even the tiny example +is **wrong**: + +.. code-block:: nimrod + template optMul{`*`(a, 2)}(a: int): int = a+a + + proc f(): int = + echo "side effect!" + result = 55 + + echo f() * 2 + +We cannot duplicate 'a' if it denotes an expression that has a side effect! +Fortunately Nimrod supports side effect analysis: + +.. code-block:: nimrod + template optMul{`*`(a, 2)}(a: int{noSideEffect}): int = a+a + + proc f(): int = + echo "side effect!" + result = 55 + + echo f() * 2 # not optimized ;-) + +So what about ``2 * a``? We should tell the compiler ``*`` is commutative. We +cannot really do that however as the following code only swaps arguments +blindly: + +.. code-block:: nimrod + template mulIsCommutative{`*`(a, b)}(a, b: int): int = b*a + +What optimizers really need to do is a *canonicalization*: + +.. code-block:: nimrod + template canonMul{`*`(a, b)}(a: int{lit}, b: int): int = b*a + +The ``int{lit}`` parameter pattern matches against an expression of +type ``int``, but only if it's a literal. + + + +Parameter constraints +--------------------- + +The `parameter constraint`:idx: expression can use the operators ``|`` (or), +``&`` (and) and ``~`` (not) and the following predicates: + +=================== ===================================================== +Predicate Meaning +=================== ===================================================== +``atom`` The matching node has no children. +``lit`` The matching node is a literal like "abc", 12. +``sym`` The matching node must be a symbol (a bound + identifier). +``ident`` The matching node must be an identifier (an unbound + identifier). +``call`` The matching AST must be a call/apply expression. +``lvalue`` The matching AST must be an lvalue. +``sideeffect`` The matching AST must have a side effect. +``nosideeffect`` The matching AST must have no side effect. +``param`` A symbol which is a parameter. +``genericparam`` A symbol which is a generic parameter. +``module`` A symbol which is a module. +``type`` A symbol which is a type. +``var`` A symbol which is a variable. +``let`` A symbol which is a ``let`` variable. +``const`` A symbol which is a constant. +``result`` The special ``result`` variable. +``proc`` A symbol which is a proc. +``method`` A symbol which is a method. +``iterator`` A symbol which is an iterator. +``converter`` A symbol which is a converter. +``macro`` A symbol which is a macro. +``template`` A symbol which is a template. +``field`` A symbol which is a field in a tuple or an object. +``enumfield`` A symbol which is a field in an enumeration. +``forvar`` A for loop variable. +``label`` A label (used in ``block`` statements). +``nk*`` The matching AST must have the specified kind. + (Example: ``nkIfStmt`` denotes an ``if`` statement.) +``alias`` States that the marked parameter needs to alias + with *some* other parameter. +``noalias`` States that *every* other parameter must not alias + with the marked parameter. +=================== ===================================================== + +The ``alias`` and ``noalias`` predicates refer not only to the matching AST, +but also to every other bound parameter; syntactially they need to occur after +the ordinary AST predicates: + +.. code-block:: nimrod + template ex{a = b + c}(a: int{noalias}, b, c: int) = + # this transformation is only valid if 'b' and 'c' do not alias 'a': + a = b + inc a, b + + +Pattern operators +----------------- + +The operators ``*``, ``**``, ``|``, ``~`` have a special meaning in patterns +if they are written in infix notation. + + +The ``|`` operator +~~~~~~~~~~~~~~~~~~ + +The ``|`` operator if used as infix operator creates an ordered choice: + +.. code-block:: nimrod + template t{0|1}(): expr = 3 + let a = 1 + # outputs 3: + echo a + +The matching is performed after the compiler performed some optimizations like +constant folding, so the following does not work: + +.. code-block:: nimrod + template t{0|1}(): expr = 3 + # outputs 1: + echo 1 + +The reason is that the compiler already transformed the 1 into "1" for +the ``echo`` statement. However, a term rewriting macro should not change the +semantics anyway. In fact they can be deactived with the ``--patterns:off`` +command line option or temporarily with the ``patterns`` pragma. + + +The ``{}`` operator +~~~~~~~~~~~~~~~~~~~ + +A pattern expression can be bound to a pattern parameter via the ``expr{param}`` +notation: + +.. code-block:: nimrod + template t{(0|1|2){x}}(x: expr): expr = x+1 + let a = 1 + # outputs 2: + echo a + + +The ``~`` operator +~~~~~~~~~~~~~~~~~~ + +The ``~`` operator is the **not** operator in patterns: + +.. code-block:: nimrod + template t{x = (~x){y} and (~x){z}}(x, y, z: bool): stmt = + x = y + if x: x = z + + var + a = false + b = true + c = false + a = b and c + echo a + + +The ``*`` operator +~~~~~~~~~~~~~~~~~~ + +The ``*`` operator can *flatten* a nested binary expression like ``a & b & c`` +to ``&(a, b, c)``: + +.. code-block:: nimrod + var + calls = 0 + + proc `&&`(s: varargs[string]): string = + result = s[0] + for i in 1..len(s)-1: result.add s[i] + inc calls + + template optConc{ `&&` * a }(a: string): expr = &&a + + let space = " " + echo "my" && (space & "awe" && "some " ) && "concat" + + # check that it's been optimized properly: + doAssert calls == 1 + + +The second operator of `*` must be a parameter; it is used to gather all the +arguments. The expression ``"my" && (space & "awe" && "some " ) && "concat"`` +is passed to ``optConc`` in ``a`` as a special list (of kind ``nkArgList``) +which is flattened into a call expression; thus the invocation of ``optConc`` +produces: + +.. code-block:: nimrod + `&&`("my", space & "awe", "some ", "concat") + + +The ``**`` operator +~~~~~~~~~~~~~~~~~~~ + +The ``**`` is much like the ``*`` operator, except that it gathers not only +all the arguments, but also the matched operators in reverse polish notation: + +.. code-block:: nimrod + import macros + + type + TMatrix = object + dummy: int + + proc `*`(a, b: TMatrix): TMatrix = nil + proc `+`(a, b: TMatrix): TMatrix = nil + proc `-`(a, b: TMatrix): TMatrix = nil + proc `$`(a: TMatrix): string = result = $a.dummy + proc mat21(): TMatrix = + result.dummy = 21 + + macro optM{ (`+`|`-`|`*`) ** a }(a: TMatrix): expr = + echo treeRepr(a) + result = newCall(bindSym"mat21") + + var x, y, z: TMatrix + + echo x + y * z - x + +This passes the expression ``x + y * z - x`` to the ``optM`` macro as +an ``nnkArgList`` node containing:: + + Arglist + Sym "x" + Sym "y" + Sym "z" + Sym "*" + Sym "+" + Sym "x" + Sym "-" + +(Which is the reverse polish notation of ``x + y * z - x``.) + + +Parameters +---------- + +Parameters in a pattern are type checked in the matching process. If a +parameter is of the type ``varargs`` it is treated specially and it can match +0 or more arguments in the AST to be matched against: + +.. code-block:: nimrod + template optWrite{ + write(f, x) + ((write|writeln){w})(f, y) + }(x, y: varargs[expr], f: TFile, w: expr) = + w(f, x, y) + + + +AST based overloading +===================== + +Parameter constraints can also be used for ordinary routine parameters; these +constraints affect ordinary overloading resolution then: + + + +However, the constraints ``alias`` and ``noalias`` are not available in +ordinary routines. + + Modules -------- +======= Nimrod supports splitting a program into pieces by a `module`:idx: concept. Each module needs to be in its own file and has its own `namespace`:idx:. Modules enable `information hiding`:idx: and `separate compilation`:idx:. diff --git a/readme.md b/readme.md index 1391c530a..fe4542b56 100644 --- a/readme.md +++ b/readme.md @@ -1,5 +1,6 @@ # Nimrod Compiler -This repo contains the Nimrod compiler, Nimrod's stdlib, tools and documentation. +This repo contains the Nimrod compiler, Nimrod's stdlib, tools and +documentation. ## Compiling Compiling the Nimrod compiler is quite straightforward. Because @@ -9,10 +10,11 @@ latest version. The C sources are however included with this repository under the build directory. Pre-compiled snapshots of the compiler are also available on -[Nimbuild](http://build.nimrod-code.org/). Your platform however may not currently -be built for. +[Nimbuild](http://build.nimrod-code.org/). Your platform however may not +currently be built for. -The compiler currently supports the following platform and architecture combinations: +The compiler currently supports the following platform and architecture +combinations: * Windows (Windows XP or greater) - x86 and x86_64 * Linux (most, if not all, distributions) - x86, x86_64, ppc64 and armv6l @@ -47,7 +49,8 @@ instead of ``build.sh``. ## Getting help A [forum](http://forum.nimrod-code.org/) is available if you have any questions, -and you can also get help in the IRC channel on [Freenode](irc://irc.freenode.net/nimrod) in #nimrod. +and you can also get help in the IRC channel +on [Freenode](irc://irc.freenode.net/nimrod) in #nimrod. ## License The compiler is licensed under the GPLv2 license, the standard library is diff --git a/readme.txt b/readme.txt index 1391c530a..fe4542b56 100755 --- a/readme.txt +++ b/readme.txt @@ -1,5 +1,6 @@ # Nimrod Compiler -This repo contains the Nimrod compiler, Nimrod's stdlib, tools and documentation. +This repo contains the Nimrod compiler, Nimrod's stdlib, tools and +documentation. ## Compiling Compiling the Nimrod compiler is quite straightforward. Because @@ -9,10 +10,11 @@ latest version. The C sources are however included with this repository under the build directory. Pre-compiled snapshots of the compiler are also available on -[Nimbuild](http://build.nimrod-code.org/). Your platform however may not currently -be built for. +[Nimbuild](http://build.nimrod-code.org/). Your platform however may not +currently be built for. -The compiler currently supports the following platform and architecture combinations: +The compiler currently supports the following platform and architecture +combinations: * Windows (Windows XP or greater) - x86 and x86_64 * Linux (most, if not all, distributions) - x86, x86_64, ppc64 and armv6l @@ -47,7 +49,8 @@ instead of ``build.sh``. ## Getting help A [forum](http://forum.nimrod-code.org/) is available if you have any questions, -and you can also get help in the IRC channel on [Freenode](irc://irc.freenode.net/nimrod) in #nimrod. +and you can also get help in the IRC channel +on [Freenode](irc://irc.freenode.net/nimrod) in #nimrod. ## License The compiler is licensed under the GPLv2 license, the standard library is diff --git a/todo.txt b/todo.txt index 66d76926e..3854102c2 100755 --- a/todo.txt +++ b/todo.txt @@ -1,12 +1,12 @@ version 0.9.2 ============= -- overloading based on ASTs +- document overloading based on ASTs - implement ``partial`` pragma for partial evaluation: easily done with AST overloading - ``hoist`` pragma for loop hoisting: can be easily done with AST overloading + global - - document overloading based on ASTs + - move semantics - test&finish first class iterators: * nested iterators From d6632ad973b69808263263f70ffe01704954a311 Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Sat, 8 Dec 2012 19:04:45 +0000 Subject: [PATCH 024/106] Fixes #268 --- lib/pure/asyncio.nim | 43 ++++++++++++++++++++++++++++++++++++++---- lib/pure/ftpclient.nim | 18 ++++++++++++------ lib/pure/sockets.nim | 22 +++++++++++++-------- 3 files changed, 65 insertions(+), 18 deletions(-) diff --git a/lib/pure/asyncio.nim b/lib/pure/asyncio.nim index 0f5197792..133d8c24e 100644 --- a/lib/pure/asyncio.nim +++ b/lib/pure/asyncio.nim @@ -126,6 +126,7 @@ type handleTask*: proc (s: PAsyncSocket) {.closure.} lineBuffer: TaintedString ## Temporary storage for ``recvLine`` + sendBuffer: string ## Temporary storage for ``send`` sslNeedAccept: bool proto: TProtocol deleg: PDelegate @@ -155,6 +156,7 @@ proc newAsyncSocket(): PAsyncSocket = result.handleTask = (proc (s: PAsyncSocket) = nil) result.lineBuffer = "".TaintedString + result.sendBuffer = "" proc AsyncSocket*(domain: TDomain = AF_INET, typ: TType = SOCK_STREAM, protocol: TProtocol = IPPROTO_TCP, @@ -225,10 +227,22 @@ proc asyncSockHandleWrite(h: PObject) = else: PAsyncSocket(h).deleg.mode = fmReadWrite else: - if PAsyncSocket(h).handleWrite != nil: - PAsyncSocket(h).handleWrite(PAsyncSocket(h)) + if PAsyncSocket(h).sendBuffer != "": + let sock = PAsyncSocket(h) + let bytesSent = sock.socket.sendAsync(sock.sendBuffer) + assert bytesSent > 0 + if bytesSent != sock.sendBuffer.len: + sock.sendBuffer = sock.sendBuffer[bytesSent .. -1] + elif bytesSent == sock.sendBuffer.len: + sock.sendBuffer = "" + + if PAsyncSocket(h).handleWrite != nil: + PAsyncSocket(h).handleWrite(PAsyncSocket(h)) else: - PAsyncSocket(h).deleg.mode = fmRead + if PAsyncSocket(h).handleWrite != nil: + PAsyncSocket(h).handleWrite(PAsyncSocket(h)) + else: + PAsyncSocket(h).deleg.mode = fmRead when defined(ssl): proc asyncSockDoHandshake(h: PObject) = @@ -340,7 +354,8 @@ proc acceptAddr*(server: PAsyncSocket, client: var PAsyncSocket, # deleg.open is set in ``toDelegate``. client.socket = c - client.lineBuffer = "" + client.lineBuffer = "".TaintedString + client.sendBuffer = "" client.info = SockConnected proc accept*(server: PAsyncSocket, client: var PAsyncSocket) = @@ -445,6 +460,26 @@ proc recvLine*(s: PAsyncSocket, line: var TaintedString): bool = of RecvFail: result = false +proc send*(sock: PAsyncSocket, data: string) = + ## Sends ``data`` to socket ``sock``. This is basically a nicer implementation + ## of ``sockets.sendAsync``. + ## + ## If ``data`` cannot be sent immediately it will be buffered and sent + ## when ``sock`` becomes writeable (during the ``handleWrite`` event). + ## It's possible that only a part of ``data`` will be sent immediately, while + ## the rest of it will be buffered and sent later. + if sock.sendBuffer.len != 0: + sock.sendBuffer.add(data) + return + let bytesSent = sock.socket.sendAsync(data) + assert bytesSent >= 0 + if bytesSent == 0: + sock.sendBuffer.add(data) + sock.deleg.mode = fmReadWrite + elif bytesSent != data.len: + sock.sendBuffer.add(data[bytesSent .. -1]) + sock.deleg.mode = fmReadWrite + proc timeValFromMilliseconds(timeout = 500): TTimeVal = if timeout != -1: var seconds = timeout div 1000 diff --git a/lib/pure/ftpclient.nim b/lib/pure/ftpclient.nim index 78c96c3f1..ad3d7a3bb 100644 --- a/lib/pure/ftpclient.nim +++ b/lib/pure/ftpclient.nim @@ -454,11 +454,13 @@ proc doUpload(ftp: PFTPClient, async = false): bool = if ftp.dsockConnected: if ftp.job.toStore.len() > 0: assert(async) - if ftp.asyncDSock.sendAsync(ftp.job.toStore): + let bytesSent = ftp.asyncDSock.sendAsync(ftp.job.toStore) + if bytesSent == ftp.job.toStore.len: ftp.job.toStore = "" - ftp.job.progress.inc(ftp.job.toStore.len) - ftp.job.oneSecond.inc(ftp.job.toStore.len) - + elif bytesSent != ftp.job.toStore.len and bytesSent != 0: + ftp.job.toStore = ftp.job.toStore[bytesSent .. -1] + ftp.job.progress.inc(bytesSent) + ftp.job.oneSecond.inc(bytesSent) else: var s = newStringOfCap(4000) var len = ftp.job.file.readBuffer(addr(s[0]), 4000) @@ -476,8 +478,12 @@ proc doUpload(ftp: PFTPClient, async = false): bool = if not async: getDSock(ftp).send(s) else: - if not ftp.asyncDSock.sendAsync(s): - ftp.job.toStore = s + let bytesSent = ftp.asyncDSock.sendAsync(s) + if bytesSent == 0: + ftp.job.toStore.add(s) + elif bytesSent != s.len: + ftp.job.toStore.add(s[bytesSent .. -1]) + len = bytesSent ftp.job.progress.inc(len) ftp.job.oneSecond.inc(len) diff --git a/lib/pure/sockets.nim b/lib/pure/sockets.nim index 01b97197e..e0eae196f 100755 --- a/lib/pure/sockets.nim +++ b/lib/pure/sockets.nim @@ -1317,13 +1317,18 @@ proc send*(socket: TSocket, data: string) {.tags: [FWriteIO].} = OSError() -proc sendAsync*(socket: TSocket, data: string): bool {.tags: [FWriteIO].} = - ## sends data to a non-blocking socket. Returns whether ``data`` was sent. - result = true - var bytesSent = send(socket, cstring(data), data.len) +proc sendAsync*(socket: TSocket, data: string): int {.tags: [FWriteIO].} = + ## sends data to a non-blocking socket. + ## Returns ``0`` if no data could be sent, if data has been sent + ## returns the amount of bytes of ``data`` that was successfully sent. This + ## number may not always be the length of ``data`` but typically is. + ## + ## An EOS (or ESSL if socket is an SSL socket) exception is raised if an error + ## occurs. + result = send(socket, cstring(data), data.len) when defined(ssl): if socket.isSSL: - if bytesSent <= 0: + if result <= 0: let ret = SSLGetError(socket.sslHandle, bytesSent.cint) case ret of SSL_ERROR_ZERO_RETURN: @@ -1339,17 +1344,18 @@ proc sendAsync*(socket: TSocket, data: string): bool {.tags: [FWriteIO].} = else: SSLError("Unknown Error") else: return - if bytesSent == -1: + if result == -1: when defined(windows): var err = WSAGetLastError() # TODO: Test on windows. if err == WSAEINPROGRESS: - return false + return 0 else: OSError() else: if errno == EAGAIN or errno == EWOULDBLOCK: - return false + return 0 else: OSError() + proc trySend*(socket: TSocket, data: string): bool {.tags: [FWriteIO].} = ## safe alternative to ``send``. Does not raise an EOS when an error occurs, From 63112696b081374d9c5ac30947d7cfe75dd3edd3 Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Sat, 8 Dec 2012 19:27:00 +0000 Subject: [PATCH 025/106] Fixes SSL implementation of sendAsync. --- lib/pure/sockets.nim | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/pure/sockets.nim b/lib/pure/sockets.nim index e0eae196f..5f7ba6ac2 100755 --- a/lib/pure/sockets.nim +++ b/lib/pure/sockets.nim @@ -1329,14 +1329,14 @@ proc sendAsync*(socket: TSocket, data: string): int {.tags: [FWriteIO].} = when defined(ssl): if socket.isSSL: if result <= 0: - let ret = SSLGetError(socket.sslHandle, bytesSent.cint) + let ret = SSLGetError(socket.sslHandle, result.cint) case ret of SSL_ERROR_ZERO_RETURN: SSLError("TLS/SSL connection failed to initiate, socket closed prematurely.") of SSL_ERROR_WANT_CONNECT, SSL_ERROR_WANT_ACCEPT: SSLError("Unexpected error occured.") # This should just not happen. of SSL_ERROR_WANT_WRITE, SSL_ERROR_WANT_READ: - return false + return 0 of SSL_ERROR_WANT_X509_LOOKUP: SSLError("Function for x509 lookup has been called.") of SSL_ERROR_SYSCALL, SSL_ERROR_SSL: From 92f8f2e7766d6c5475f643a17df88aa680a4e5dc Mon Sep 17 00:00:00 2001 From: Araq Date: Sun, 9 Dec 2012 03:02:52 +0100 Subject: [PATCH 026/106] documented AST overloading and some TR optimizations --- doc/manual.txt | 85 ++++++++++++++++++++++++++++++++++++- tests/patterns/thoist.nim | 13 ++++++ tests/patterns/tpartial.nim | 11 +++++ todo.txt | 8 +--- web/index.txt | 1 - web/news.txt | 1 + 6 files changed, 109 insertions(+), 10 deletions(-) create mode 100644 tests/patterns/thoist.nim create mode 100644 tests/patterns/tpartial.nim diff --git a/doc/manual.txt b/doc/manual.txt index 4d0c3b01b..da66833ab 100755 --- a/doc/manual.txt +++ b/doc/manual.txt @@ -3973,18 +3973,95 @@ parameter is of the type ``varargs`` it is treated specially and it can match +Example: Partial evaluation +--------------------------- + +The following example shows how some simple partial evaluation can be +implemented with term rewriting: + +.. code-block:: nimrod + proc p(x, y: int; cond: bool): int = + result = if cond: x + y else: x - y + + template optP{p(x, y, true)}(x, y: expr): expr = x + y + template optP{p(x, y, false)}(x, y: expr): expr = x - y + + +Example: hoisting +----------------- + +The following example how some form of hoisting can be implemented: + +.. code-block:: nimrod + import pegs + + template optPeg{peg(pattern)}(pattern: string{lit}): TPeg = + var gl {.global, gensym.} = peg(pattern) + gl + + for i in 0 .. 3: + echo match("(a b c)", peg"'(' @ ')'") + echo match("W_HI_Le", peg"\y 'while'") + +The ``optPeg`` template optimizes the case of a peg constructor with a string +literal, so that the pattern will only be parsed once at program startup and +stored in global ``gl`` which is then re-used. This optimization is called +hoisting because it is comparable to classical loop hoisting. + + AST based overloading ===================== Parameter constraints can also be used for ordinary routine parameters; these constraints affect ordinary overloading resolution then: +.. code-block:: nimrod + proc optLit(a: string{lit|`const`}) = + echo "string literal" + proc optLit(a: string) = + echo "no string literal" + const + constant = "abc" + + var + variable = "xyz" + + optLit("literal") + optLit(constant) + optLit(variable) However, the constraints ``alias`` and ``noalias`` are not available in ordinary routines. +Move optimization +----------------- + +The ``call`` constraint is particular useful to implement a `move`:idx: +optimization for types that have copying semantics: + +.. code-block:: nimrod + proc `[]=`*(t: var TTable, key: string, val: string) = + ## puts a (key, value)-pair into `t`. The semantics of string require + ## a copy here: + let idx = findInsertionPosition(key) + t[idx] = key + t[idx] = val + + proc `[]=`*(t: var TTable, key: string{call}, val: string{call}) = + ## puts a (key, value)-pair into `t`. Optimized version that knows that + ## the strings are unique and thus don't need to be copied: + let idx = findInsertionPosition(key) + shallowCopy t[idx], key + shallowCopy t[idx], val + + var t: TTable + # overloading resolution ensures that the optimized []= is called here: + t["abc"] = "xyz" + + + Modules ======= Nimrod supports splitting a program into pieces by a `module`:idx: concept. @@ -4692,10 +4769,14 @@ Threadvar pragma ---------------- A global variable can be marked with the `threadvar`:idx: pragma; it is -a `thead-local`:idx: variable then: +a `thread-local`:idx: variable then: .. code-block:: nimrod - var checkpoints* {.threadvar.}: seq[string] = @[] + var checkpoints* {.threadvar.}: seq[string] + +Due to implementation restrictions thread local variables cannot be +initialized within the ``var`` section. (Every thread local variable needs to +be replicated at thread creation.) Actor model diff --git a/tests/patterns/thoist.nim b/tests/patterns/thoist.nim new file mode 100644 index 000000000..7d14c0abf --- /dev/null +++ b/tests/patterns/thoist.nim @@ -0,0 +1,13 @@ +discard """ + output: '''true +true''' +""" + +import pegs + +template optPeg{peg(pattern)}(pattern: string{lit}): TPeg = + var gl {.global, gensym.} = peg(pattern) + gl + +echo match("(a b c)", peg"'(' @ ')'") +echo match("W_HI_Le", peg"\y 'while'") diff --git a/tests/patterns/tpartial.nim b/tests/patterns/tpartial.nim new file mode 100644 index 000000000..1a5b1a953 --- /dev/null +++ b/tests/patterns/tpartial.nim @@ -0,0 +1,11 @@ +discard """ + output: '''-1''' +""" + +proc p(x, y: int; cond: bool): int = + result = if cond: x + y else: x - y + +template optP{p(x, y, true)}(x, y: expr): expr = x - y +template optP{p(x, y, false)}(x, y: expr): expr = x + y + +echo p(2, 4, true) diff --git a/todo.txt b/todo.txt index 3854102c2..141acfa93 100755 --- a/todo.txt +++ b/todo.txt @@ -1,18 +1,12 @@ version 0.9.2 ============= -- document overloading based on ASTs - - implement ``partial`` pragma for partial evaluation: easily done with AST - overloading - - ``hoist`` pragma for loop hoisting: can be easily done with - AST overloading + global - - move semantics - - test&finish first class iterators: * nested iterators * test generic iterators - fix closure bug finally - fix marshal bug +- make PDF generation of the docs work again version 0.9.X diff --git a/web/index.txt b/web/index.txt index b81ecae0d..207e425ae 100755 --- a/web/index.txt +++ b/web/index.txt @@ -101,7 +101,6 @@ Roadmap to 1.0 ============== Version 0.9.2 - * overloading based on ASTs (like already possible for term rewriting macros) * better interaction between macros, templates and overloading * the symbol binding rules for generics and templates may change again diff --git a/web/news.txt b/web/news.txt index 254e9a57a..802825515 100755 --- a/web/news.txt +++ b/web/news.txt @@ -56,6 +56,7 @@ Language Additions - An ``export`` statement has been added to the language: It can be used for symbol forwarding so client modules don't have to import a module's dependencies explicitly. +- Overloading based on ASTs has been implemented. 2012-09-23 Version 0.9.0 released From f96ec25ed26cc8868adba6c61af037056049e6be Mon Sep 17 00:00:00 2001 From: Araq Date: Sun, 9 Dec 2012 15:53:59 +0100 Subject: [PATCH 027/106] made some tests green; some small bugfixes --- compiler/semtypes.nim | 2 +- doc/manual.txt | 2 +- packages/docutils/rstgen.nim | 4 ++-- tests/patterns/tpartial.nim | 2 +- tests/reject/tenummix.nim | 2 +- todo.txt | 1 - 6 files changed, 6 insertions(+), 7 deletions(-) diff --git a/compiler/semtypes.nim b/compiler/semtypes.nim index 3da2a95b7..3ad275601 100755 --- a/compiler/semtypes.nim +++ b/compiler/semtypes.nim @@ -798,7 +798,7 @@ proc semTypeExpr(c: PContext, n: PNode): PType = proc freshType(res, prev: PType): PType {.inline.} = if prev.isNil: - result = copyType(result, result.owner, keepId=false) + result = copyType(res, res.owner, keepId=false) else: result = res diff --git a/doc/manual.txt b/doc/manual.txt index da66833ab..6e61395e3 100755 --- a/doc/manual.txt +++ b/doc/manual.txt @@ -2408,7 +2408,7 @@ simplicity (they require specialized semantic checking):: is, of, echo, shallowCopy, getAst Thus they act more like keywords than like ordinary identifiers; unlike a -keyword however, a redefinition may `shadow`:id: the definition in +keyword however, a redefinition may `shadow`:idx: the definition in the ``system`` module. diff --git a/packages/docutils/rstgen.nim b/packages/docutils/rstgen.nim index 492322f6f..92bd20cf8 100644 --- a/packages/docutils/rstgen.nim +++ b/packages/docutils/rstgen.nim @@ -474,8 +474,8 @@ proc renderRstToOut(d: PDoc, n: PRstNode, result: var string) = of rnTableRow: if len(n) >= 1: if d.target == outLatex: - var tmp = "" - renderRstToOut(d, n.sons[0], tmp) + #var tmp = "" + renderRstToOut(d, n.sons[0], result) for i in countup(1, len(n) - 1): result.add(" & ") renderRstToOut(d, n.sons[i], result) diff --git a/tests/patterns/tpartial.nim b/tests/patterns/tpartial.nim index 1a5b1a953..fdaa3414a 100644 --- a/tests/patterns/tpartial.nim +++ b/tests/patterns/tpartial.nim @@ -1,5 +1,5 @@ discard """ - output: '''-1''' + output: '''-2''' """ proc p(x, y: int; cond: bool): int = diff --git a/tests/reject/tenummix.nim b/tests/reject/tenummix.nim index 8a2a19c1e..ec99a2b6a 100644 --- a/tests/reject/tenummix.nim +++ b/tests/reject/tenummix.nim @@ -1,6 +1,6 @@ discard """ file: "system.nim" - line: 663 + line: 669 errormsg: "type mismatch" """ diff --git a/todo.txt b/todo.txt index 141acfa93..b162812bc 100755 --- a/todo.txt +++ b/todo.txt @@ -6,7 +6,6 @@ version 0.9.2 * test generic iterators - fix closure bug finally - fix marshal bug -- make PDF generation of the docs work again version 0.9.X From b0dc877cc3cb45cce40f7cd9ad281823bf91cd72 Mon Sep 17 00:00:00 2001 From: Araq Date: Sun, 9 Dec 2012 21:23:12 +0100 Subject: [PATCH 028/106] small documentation improvements --- doc/manual.txt | 12 ++++++------ packages/docutils/rst.nim | 4 ++-- packages/docutils/rstgen.nim | 3 ++- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/doc/manual.txt b/doc/manual.txt index 6e61395e3..2feb60ac0 100755 --- a/doc/manual.txt +++ b/doc/manual.txt @@ -2858,9 +2858,9 @@ compatibility: For a routine ``p`` the compiler uses inference rules to determine the set of possibly raised exceptions; the algorithm operates on ``p``'s call graph: -1. Every indirect call via some proc type ``T`` is assumed to - raise ``system.E_Base`` (the base type of the exception hierarchy) and thus - any exception unless ``T`` has an explicit ``raises`` list. +1. Every indirect call via some proc type ``T`` is assumed to + raise ``system.E_Base`` (the base type of the exception hierarchy) and + thus any exception unless ``T`` has an explicit ``raises`` list. 2. Every call to a proc ``q`` which has an unknown body (due to a forward declaration or an ``importc`` pragma) is assumed to raise ``system.E_Base`` unless ``q`` has an explicit ``raises`` list. @@ -2914,7 +2914,7 @@ effects up to the ``effects``'s position: proc p(what: bool) = if what: raise newException(EIO, "IO") - {.effect.} + {.effects.} else: raise newException(EOS, "OS") @@ -3522,7 +3522,7 @@ BindSym The above ``debug`` macro relies on the fact that ``write``, ``writeln`` and ``stdout`` are declared in the system module and thus visible in the instantiating context. There is a way to use bound identifiers -(aka `symbols`:idx) instead of using unbound identifiers. The ``bindSym`` +(aka `symbols`:idx:) instead of using unbound identifiers. The ``bindSym`` builtin can be used for that: .. code-block:: nimrod @@ -4038,7 +4038,7 @@ ordinary routines. Move optimization ----------------- -The ``call`` constraint is particular useful to implement a `move`:idx: +The ``call`` constraint is particularly useful to implement a `move`:idx: optimization for types that have copying semantics: .. code-block:: nimrod diff --git a/packages/docutils/rst.nim b/packages/docutils/rst.nim index f594a39f5..c752e5065 100755 --- a/packages/docutils/rst.nim +++ b/packages/docutils/rst.nim @@ -1012,10 +1012,10 @@ proc whichSection(p: TRstParser): TRstNodeKind = result = rnOptionList else: result = rnParagraph - of tkWord, tkOther, tkWhite: + of tkWord, tkOther, tkWhite: if match(p, tokenAfterNewLine(p), "ai"): result = rnHeadline - elif isDefList(p): result = rnDefList elif match(p, p.idx, "e) ") or match(p, p.idx, "e. "): result = rnEnumList + elif isDefList(p): result = rnDefList else: result = rnParagraph else: result = rnLeaf diff --git a/packages/docutils/rstgen.nim b/packages/docutils/rstgen.nim index 92bd20cf8..53bd8188e 100644 --- a/packages/docutils/rstgen.nim +++ b/packages/docutils/rstgen.nim @@ -390,7 +390,8 @@ proc renderField(d: PDoc, n: PRstNode, result: var string) = if d.target == outLatex: var fieldname = addNodes(n.sons[0]) var fieldval = esc(d.target, strip(addNodes(n.sons[1]))) - if cmpIgnoreStyle(fieldname, "author") == 0: + if cmpIgnoreStyle(fieldname, "author") == 0 or + cmpIgnoreStyle(fieldname, "authors") == 0: if d.meta[metaAuthor].len == 0: d.meta[metaAuthor] = fieldval b = true From a840a4ce53c2344f77c6e07b4f8cbed0af2820dd Mon Sep 17 00:00:00 2001 From: Araq Date: Sun, 9 Dec 2012 23:20:48 +0100 Subject: [PATCH 029/106] improvements for --babelPath --- compiler/babelcmd.nim | 24 +++++++++++++++++++----- compiler/condsyms.nim | 1 + config/nimrod.cfg | 10 ++++------ 3 files changed, 24 insertions(+), 11 deletions(-) diff --git a/compiler/babelcmd.nim b/compiler/babelcmd.nim index 99513e58b..97d3bcf6c 100644 --- a/compiler/babelcmd.nim +++ b/compiler/babelcmd.nim @@ -9,7 +9,7 @@ ## Implements some helper procs for Babel (Nimrod's package manager) support. -import parseutils, strtabs, os, options, msgs, lists +import parseutils, strutils, strtabs, os, options, msgs, lists proc addPath*(path: string, info: TLineInfo) = if not contains(options.searchPaths, path): @@ -58,6 +58,23 @@ iterator chosen(packages: PStringTable): string = let res = if val == latest: key else: key & '-' & val yield res +proc addPathWithNimFiles(p: string, info: TLineInfo) = + proc hasNimFile(dir: string): bool = + for kind, path in walkDir(dir): + if kind == pcFile and path.endsWith(".nim"): + return true + + proc addPath(p: string) = + if not contains(options.searchPaths, p): + Message(info, hintPath, p) + lists.PrependStr(options.searchPaths, p) + + if hasNimFile(p): + addPath(p) + else: + for kind, p2 in walkDir(p): + if hasNimFile(p2): addPath(p2) + proc addPathRec(dir: string, info: TLineInfo) = var packages = newStringTable(modeStyleInsensitive) var pos = dir.len-1 @@ -65,11 +82,8 @@ proc addPathRec(dir: string, info: TLineInfo) = for k,p in os.walkDir(dir): if k == pcDir and p[pos] != '.': addPackage(packages, p) - for p in packages.chosen: - if not contains(options.searchPaths, p): - Message(info, hintPath, p) - lists.PrependStr(options.searchPaths, p) + addPathWithNimFiles(p, info) proc babelPath*(path: string, info: TLineInfo) = addPathRec(path, info) diff --git a/compiler/condsyms.nim b/compiler/condsyms.nim index 234029ea9..17366f6e9 100755 --- a/compiler/condsyms.nim +++ b/compiler/condsyms.nim @@ -60,6 +60,7 @@ proc InitDefines*() = DefineSymbol("niminheritable") DefineSymbol("nimmixin") DefineSymbol("nimeffects") + DefineSymbol("nimbabel") # add platform specific symbols: case targetCPU diff --git a/config/nimrod.cfg b/config/nimrod.cfg index 3ae38133d..9db13e836 100755 --- a/config/nimrod.cfg +++ b/config/nimrod.cfg @@ -13,11 +13,6 @@ cc = gcc arm.linux.gcc.exe = "arm-linux-gcc" arm.linux.gcc.linkerexe = "arm-linux-gcc" -@if nim: - # use the old fixed library for bootstrapping with Nim: - lib = "nimlib" -@end - path="$lib/core" path="$lib/pure" path="$lib/pure/collections" @@ -36,7 +31,10 @@ path="$lib/windows" path="$lib/posix" path="$lib/ecmas" path="$lib/pure/unidecode" -path="$home/.babel/libs/" + +@if nimbabel: + babelpath="$home/.babel/libs/" +@end @if release or quick: obj_checks:off From c98696d7428346b53c8998bf8fab77fe08830e2e Mon Sep 17 00:00:00 2001 From: Araq Date: Tue, 11 Dec 2012 20:06:15 +0100 Subject: [PATCH 030/106] lazy paths for Babel support --- compiler/babelcmd.nim | 20 ++++++++++---------- compiler/commands.nim | 1 + compiler/lists.nim | 11 ++++++++++- compiler/main.nim | 6 +++--- compiler/options.nim | 29 ++++++++++++++++++++++------- compiler/patterns.nim | 9 +++++++++ config/nimrod.cfg | 2 +- doc/manual.txt | 2 +- 8 files changed, 57 insertions(+), 23 deletions(-) diff --git a/compiler/babelcmd.nim b/compiler/babelcmd.nim index 97d3bcf6c..956c6a6ae 100644 --- a/compiler/babelcmd.nim +++ b/compiler/babelcmd.nim @@ -58,22 +58,22 @@ iterator chosen(packages: PStringTable): string = let res = if val == latest: key else: key & '-' & val yield res +proc addBabelPath(p: string, info: TLineInfo) = + if not contains(options.searchPaths, p): + Message(info, hintPath, p) + lists.PrependStr(options.lazyPaths, p) + proc addPathWithNimFiles(p: string, info: TLineInfo) = proc hasNimFile(dir: string): bool = for kind, path in walkDir(dir): if kind == pcFile and path.endsWith(".nim"): - return true - - proc addPath(p: string) = - if not contains(options.searchPaths, p): - Message(info, hintPath, p) - lists.PrependStr(options.searchPaths, p) - + result = true + break if hasNimFile(p): - addPath(p) + addBabelPath(p, info) else: for kind, p2 in walkDir(p): - if hasNimFile(p2): addPath(p2) + if hasNimFile(p2): addBabelPath(p2, info) proc addPathRec(dir: string, info: TLineInfo) = var packages = newStringTable(modeStyleInsensitive) @@ -87,4 +87,4 @@ proc addPathRec(dir: string, info: TLineInfo) = proc babelPath*(path: string, info: TLineInfo) = addPathRec(path, info) - addPath(path, info) + addBabelPath(path, info) diff --git a/compiler/commands.nim b/compiler/commands.nim index 4a72e25f3..4b9a47d37 100755 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -221,6 +221,7 @@ proc processSwitch(switch, arg: string, pass: TCmdlinePass, info: TLineInfo) = expectArg(switch, arg, pass, info) let path = processPath(arg) lists.ExcludeStr(options.searchPaths, path) + lists.ExcludeStr(options.lazyPaths, path) of "nimcache": expectArg(switch, arg, pass, info) options.nimcacheDir = processPath(arg) diff --git a/compiler/lists.nim b/compiler/lists.nim index 9da4db177..67b32f919 100755 --- a/compiler/lists.nim +++ b/compiler/lists.nim @@ -90,6 +90,15 @@ proc Remove*(list: var TLinkedList, entry: PListEntry) = if entry.next != nil: entry.next.prev = entry.prev if entry.prev != nil: entry.prev.next = entry.next +proc bringToFront*(list: var TLinkedList, entry: PListEntry) = + if entry == list.head: return + if entry == list.tail: list.tail = entry.prev + if entry.next != nil: entry.next.prev = entry.prev + if entry.prev != nil: entry.prev.next = entry.next + entry.prev = nil + entry.next = list.head + list.head = entry + proc ExcludeStr*(list: var TLinkedList, data: string) = var it = list.head while it != nil: @@ -99,6 +108,6 @@ proc ExcludeStr*(list: var TLinkedList, data: string) = proc Find*(list: TLinkedList, fn: TCompareProc, closure: Pointer): PListEntry = result = list.head - while result != nil: + while result != nil: if fn(result, closure): return result = result.next diff --git a/compiler/main.nim b/compiler/main.nim index dabd5309b..edd4fda68 100755 --- a/compiler/main.nim +++ b/compiler/main.nim @@ -279,11 +279,11 @@ proc MainCommand = gCmd = cmdGenDepend wantMainModule() CommandGenDepend() - of "dump": + of "dump": gCmd = cmdDump condsyms.ListSymbols() - for it in iterSearchPath(): MsgWriteln(it) - of "check": + for it in iterSearchPath(searchPaths): MsgWriteln(it) + of "check": gCmd = cmdCheck wantMainModule() CommandCheck() diff --git a/compiler/options.nim b/compiler/options.nim index 42fca1ad1..b8e65f8e3 100755 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -88,7 +88,7 @@ var optPatterns} gGlobalOptions*: TGlobalOptions = {optRefcGC, optThreadAnalysis} gExitcode*: int8 - searchPaths*: TLinkedList + searchPaths*, lazyPaths*: TLinkedList outFile*: string = "" headerFile*: string = "" gCmd*: TCommands = cmdNone # the command @@ -194,22 +194,37 @@ proc completeGeneratedFilePath*(f: string, createSubDir: bool = true): string = result = joinPath(subdir, tail) #echo "completeGeneratedFilePath(", f, ") = ", result -iterator iterSearchPath*(): string = +iterator iterSearchPath*(SearchPaths: TLinkedList): string = var it = PStrEntry(SearchPaths.head) - while it != nil: + while it != nil: yield it.data it = PStrEntry(it.Next) proc rawFindFile(f: string): string = - for it in iterSearchPath(): + for it in iterSearchPath(SearchPaths): result = JoinPath(it, f) - if ExistsFile(result): + if existsFile(result): return result.canonicalizePath result = "" +proc rawFindFile2(f: string): string = + var it = PStrEntry(lazyPaths.head) + while it != nil: + result = JoinPath(it.data, f) + if existsFile(result): + bringToFront(lazyPaths, it) + return result.canonicalizePath + it = PStrEntry(it.Next) + result = "" + proc FindFile*(f: string): string {.procvar.} = - result = rawFindFile(f) - if len(result) == 0: result = rawFindFile(toLower(f)) + result = f.rawFindFile + if result.len == 0: + result = f.toLower.rawFindFile + if result.len == 0: + result = f.rawFindFile2 + if result.len == 0: + result = f.toLower.rawFindFile2 proc findModule*(modulename: string): string {.inline.} = # returns path to module diff --git a/compiler/patterns.nim b/compiler/patterns.nim index b7792100f..ff7f18ac0 100644 --- a/compiler/patterns.nim +++ b/compiler/patterns.nim @@ -237,6 +237,15 @@ proc addToArgList(result, n: PNode) = else: for i in 0 .. Date: Tue, 11 Dec 2012 21:39:05 +0100 Subject: [PATCH 031/106] Adds to split() a code example using a set of separators. --- lib/pure/strutils.nim | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/lib/pure/strutils.nim b/lib/pure/strutils.nim index 0f11b4d89..8a5061037 100755 --- a/lib/pure/strutils.nim +++ b/lib/pure/strutils.nim @@ -186,7 +186,24 @@ iterator split*(s: string, seps: set[char] = Whitespace): string = ## for word in split(";;this;is;an;;example;;;", {';'}): ## writeln(stdout, word) ## - ## produces the same output. + ## produces the same output. The code: + ## + ## .. code-block:: nimrod + ## let date = "2012-11-20T22:08:08.398990" + ## let separators = {' ', '-', ':', 'T'} + ## for number in split(date, separators): + ## writeln(stdout, number) + ## + ## Results in: + ## + ## .. code-block:: nimrod + ## "2012" + ## "11" + ## "20" + ## "22" + ## "08" + ## "08.398990" + ## var last = 0 assert(not ('\0' in seps)) while last < len(s): From d9dab30c1499dec081f553b887905009cd8f3e1a Mon Sep 17 00:00:00 2001 From: Grzegorz Adam Hankiewicz Date: Tue, 11 Dec 2012 21:51:03 +0100 Subject: [PATCH 032/106] Documents json [] accesors, raises explicit exception. --- lib/pure/json.nim | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/lib/pure/json.nim b/lib/pure/json.nim index 941d88dfc..e7945eac3 100755 --- a/lib/pure/json.nim +++ b/lib/pure/json.nim @@ -607,14 +607,14 @@ proc len*(n: PJsonNode): int = else: nil proc `[]`*(node: PJsonNode, name: String): PJsonNode = - ## Gets a field from a `JObject`. + ## Gets a field from a `JObject`. Returns nil if the key is not found. assert(node.kind == JObject) for key, item in items(node.fields): if key == name: return item return nil -proc `[]`*(node: PJsonNode, index: Int): PJsonNode = +proc `[]`*(node: PJsonNode, index: Int): PJsonNode {.raises: [EInvalidIndex].} = ## Gets the node at `index` in an Array. assert(node.kind == JArray) return node.elems[index] @@ -893,6 +893,10 @@ when isMainModule: echo(parsed["keyÄÖöoßß"]) echo() echo(pretty(parsed2)) + try: + echo(parsed["key2"][12123]) + raise newException(EInvalidValue, "That line was expected to fail") + except EInvalidIndex: echo() discard """ while true: From d3eaddc961092d50bc02c2f5cdd3e940c876a3fc Mon Sep 17 00:00:00 2001 From: Grzegorz Adam Hankiewicz Date: Tue, 11 Dec 2012 21:59:41 +0100 Subject: [PATCH 033/106] Transforms httpclient into hyperlink for ease of use. --- lib/impure/web.nim | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/impure/web.nim b/lib/impure/web.nim index 417fe9746..5f04422d1 100755 --- a/lib/impure/web.nim +++ b/lib/impure/web.nim @@ -17,7 +17,8 @@ ## Currently only requesting URLs is implemented. The implementation depends ## on the libcurl library! ## -## **Deprecated since version 0.8.8:** Use the ``httpclient`` module instead. +## **Deprecated since version 0.8.8:** Use the +## `httpclient `_ module instead. ## {.deprecated.} From 545a7577e55597595a15c98a092f5e5da847c33e Mon Sep 17 00:00:00 2001 From: Grzegorz Adam Hankiewicz Date: Tue, 11 Dec 2012 22:15:50 +0100 Subject: [PATCH 034/106] Adds json usage example. --- lib/pure/json.nim | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/lib/pure/json.nim b/lib/pure/json.nim index e7945eac3..cee135f08 100755 --- a/lib/pure/json.nim +++ b/lib/pure/json.nim @@ -13,6 +13,23 @@ ## (unlike XML). It is easy for machines to parse and generate. ## JSON is based on a subset of the JavaScript Programming Language, ## Standard ECMA-262 3rd Edition - December 1999. +## +## Usage example: +## +## .. code-block:: nimrod +## let +## small_json = """{"test": 1.3, "key2": true}""" +## jobj = parseJson(small_json) +## assert (jobj.kind == JObject) +## echo($jobj["test"].fnum) +## echo($jobj["key2"].bval) +## +## Results in: +## +## .. code-block:: nimrod +## +## 1.3000000000000000e+00 +## true import hashes, strutils, lexbase, streams, unicode From afff026dbd0466cabbce17a1026f7927598cceea Mon Sep 17 00:00:00 2001 From: Grzegorz Adam Hankiewicz Date: Tue, 11 Dec 2012 22:24:07 +0100 Subject: [PATCH 035/106] Adds hyperlink to system module to explain TFileMode type. --- lib/pure/streams.nim | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/pure/streams.nim b/lib/pure/streams.nim index 5db21d76a..212aab493 100755 --- a/lib/pure/streams.nim +++ b/lib/pure/streams.nim @@ -263,7 +263,8 @@ proc newFileStream*(f: TFile): PFileStream = proc newFileStream*(filename: string, mode: TFileMode): PFileStream = ## creates a new stream from the file named `filename` with the mode `mode`. - ## If the file cannot be opened, nil is returned. + ## If the file cannot be opened, nil is returned. See the `system + ## `_ module for a list of available TFileMode enums. var f: TFile if Open(f, filename, mode): result = newFileStream(f) From b1b01e9f995e09dae945d69370124de04c285f27 Mon Sep 17 00:00:00 2001 From: Grzegorz Adam Hankiewicz Date: Fri, 14 Dec 2012 22:11:13 +0100 Subject: [PATCH 036/106] Removes misleading comment in build script. A true program would rather compile the main nim project file which imports other modules. --- examples/cross_calculator/ios/scripts/xcode_prebuild.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/examples/cross_calculator/ios/scripts/xcode_prebuild.sh b/examples/cross_calculator/ios/scripts/xcode_prebuild.sh index 7626ba0b9..c6d38f164 100755 --- a/examples/cross_calculator/ios/scripts/xcode_prebuild.sh +++ b/examples/cross_calculator/ios/scripts/xcode_prebuild.sh @@ -21,7 +21,6 @@ DEST_NIMBASE=build/nimcache/nimbase.h # Ok, are we out now? if [ -d src ] then - # You may want to use a wildcard here to compile all nimrod files. $PATH_TO_NIMROD objc --noMain --app:lib \ --nimcache:build/nimcache --compileOnly \ --header --cpu:i386 ../nimrod_backend/backend.nim From da825a0fe2842468f7e925de1f088e62c74bcd9b Mon Sep 17 00:00:00 2001 From: Keita Haga Date: Sun, 16 Dec 2012 03:57:17 +0900 Subject: [PATCH 037/106] cd to build/, before trying to extract csources.zip --- readme.md | 6 ++++-- readme.txt | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/readme.md b/readme.md index fe4542b56..d5cb35820 100644 --- a/readme.md +++ b/readme.md @@ -34,7 +34,9 @@ If you are on a fairly modern *nix system, the following steps should work: ``` $ git clone git://github.com/Araq/Nimrod.git $ cd Nimrod -$ unzip build/csources.zip +$ cd build +$ unzip csources.zip +$ cd .. $ ./build.sh $ bin/nimrod c koch $ ./koch boot -d:release @@ -61,4 +63,4 @@ developed with Nimrod, allowing you to create commercial applications. Read copying.txt for more details. Copyright (c) 2004-2012 Andreas Rumpf. -All rights reserved. \ No newline at end of file +All rights reserved. diff --git a/readme.txt b/readme.txt index fe4542b56..d5cb35820 100755 --- a/readme.txt +++ b/readme.txt @@ -34,7 +34,9 @@ If you are on a fairly modern *nix system, the following steps should work: ``` $ git clone git://github.com/Araq/Nimrod.git $ cd Nimrod -$ unzip build/csources.zip +$ cd build +$ unzip csources.zip +$ cd .. $ ./build.sh $ bin/nimrod c koch $ ./koch boot -d:release @@ -61,4 +63,4 @@ developed with Nimrod, allowing you to create commercial applications. Read copying.txt for more details. Copyright (c) 2004-2012 Andreas Rumpf. -All rights reserved. \ No newline at end of file +All rights reserved. From a22c1f444fa76cc06fee1ae31c79986e9c3b459f Mon Sep 17 00:00:00 2001 From: Araq Date: Sat, 15 Dec 2012 22:35:16 +0100 Subject: [PATCH 038/106] fixes #271 --- compiler/msgs.nim | 2 +- compiler/sem.nim | 2 ++ compiler/semcall.nim | 40 +++++++++++++++++++++++++++++----------- compiler/semexprs.nim | 1 + compiler/sigmatch.nim | 34 ++++++++++++++-------------------- config/nimrod.cfg | 2 +- doc/lib.txt | 4 ++-- tests/run/titer9.nim | 6 +++--- todo.txt | 1 + 9 files changed, 54 insertions(+), 38 deletions(-) diff --git a/compiler/msgs.nim b/compiler/msgs.nim index 0f795c07d..240115817 100755 --- a/compiler/msgs.nim +++ b/compiler/msgs.nim @@ -132,7 +132,7 @@ const errNumberOutOfRange: "number $1 out of valid range", errNnotAllowedInCharacter: "\\n not allowed in character literal", errClosingBracketExpected: "closing ']' expected, but end of file reached", - errMissingFinalQuote: "missing final \'", + errMissingFinalQuote: "missing final \' for character literal", errIdentifierExpected: "identifier expected, but found \'$1\'", errNewlineExpected: "newline expected, but found \'$1\'", errInvalidModuleName: "invalid module name: '$1'", diff --git a/compiler/sem.nim b/compiler/sem.nim index 911eafb08..c8340a139 100755 --- a/compiler/sem.nim +++ b/compiler/sem.nim @@ -47,6 +47,8 @@ proc fixImmediateParams(n: PNode): PNode proc activate(c: PContext, n: PNode) proc semQuoteAst(c: PContext, n: PNode): PNode +proc IndexTypesMatch(c: PContext, f, a: PType, arg: PNode): PNode + proc typeMismatch(n: PNode, formal, actual: PType) = if formal.kind != tyError and actual.kind != tyError: LocalError(n.Info, errGenerated, msgKindToString(errTypeMismatch) & diff --git a/compiler/semcall.nim b/compiler/semcall.nim index 962e4d3cc..0a5f19822 100755 --- a/compiler/semcall.nim +++ b/compiler/semcall.nim @@ -84,15 +84,34 @@ proc resolveOverloads(c: PContext, n, orig: PNode, getProcHeader(best.calleeSym), getProcHeader(alt.calleeSym), args]) -proc instantiateGenericConverters(c: PContext, n: PNode, x: TCandidate) {. - noinline.}= - for i in 1 .. `_ Nimrod thread support. **Note**: This is part of the system module. Do not - import it explicitely. + import it explicitly. * `channels `_ Nimrod message passing support for threads. **Note**: This is part of the - system module. Do not import it explicitely. + system module. Do not import it explicitly. * `locks `_ Locks and condition variables for Nimrod. diff --git a/tests/run/titer9.nim b/tests/run/titer9.nim index 0d6c466c1..99874e70a 100644 --- a/tests/run/titer9.nim +++ b/tests/run/titer9.nim @@ -4,17 +4,17 @@ discard """ 0''' """ -iterator count(x: int, skip: bool): int {.closure.} = +iterator count[T](x: T, skip: bool): int {.closure.} = if skip: return x+10 else: yield x+1 if skip: return x+10 else: yield x+2 -proc takeProc(x: iterator (x: int, skip: bool): int) = +proc takeProc[T](x: iterator (x: T, skip: bool): int) = echo x(4, false) echo x(4, true) echo x(4, false) -takeProc(count) +takeProc(count[int]) diff --git a/todo.txt b/todo.txt index b162812bc..f5e6d8092 100755 --- a/todo.txt +++ b/todo.txt @@ -87,6 +87,7 @@ GC Optimizations ============= +- optimize 'if' with a constant condition - escape analysis for string/seq seems to be easy to do too; even further write barrier specialization - inlining of first class functions From cd006792891f2d8aafce73e7a3f29b6254410d59 Mon Sep 17 00:00:00 2001 From: Araq Date: Sat, 15 Dec 2012 23:39:27 +0100 Subject: [PATCH 039/106] some fixes for generic first class iterators --- compiler/sem.nim | 3 +-- compiler/seminst.nim | 6 +++--- compiler/semstmts.nim | 12 +++++++----- todo.txt | 2 +- 4 files changed, 12 insertions(+), 11 deletions(-) diff --git a/compiler/sem.nim b/compiler/sem.nim index c8340a139..c70cbf61e 100755 --- a/compiler/sem.nim +++ b/compiler/sem.nim @@ -39,8 +39,7 @@ proc semTypeNode(c: PContext, n: PNode, prev: PType): PType proc semStmt(c: PContext, n: PNode): PNode proc semParamList(c: PContext, n, genericParams: PNode, s: PSym) proc addParams(c: PContext, n: PNode, kind: TSymKind) -proc addResult(c: PContext, t: PType, info: TLineInfo, owner: TSymKind) -proc addResultNode(c: PContext, n: PNode) +proc maybeAddResult(c: PContext, s: PSym, n: PNode) proc instGenericContainer(c: PContext, n: PNode, header: PType): PType proc tryExpr(c: PContext, n: PNode, flags: TExprFlags = {}): PNode proc fixImmediateParams(n: PNode): PNode diff --git a/compiler/seminst.nim b/compiler/seminst.nim index 8e164531a..0a1a17f72 100755 --- a/compiler/seminst.nim +++ b/compiler/seminst.nim @@ -93,9 +93,7 @@ proc instantiateBody(c: PContext, n: PNode, result: PSym) = # add it here, so that recursive generic procs are possible: addDecl(c, result) pushProcCon(c, result) - if result.kind in {skProc, skMethod, skConverter, skMacro}: - addResult(c, result.typ.sons[0], n.info, result.kind) - addResultNode(c, n) + maybeAddResult(c, result, n) var b = n.sons[bodyPos] var symMap: TIdTable InitIdTable symMap @@ -168,6 +166,8 @@ proc generateInstance(c: PContext, fn: PSym, pt: TIdTable, result.typ = newTypeS(tyProc, c) rawAddSon(result.typ, nil) result.typ.callConv = fn.typ.callConv + if result.kind == skIterator: result.typ.flags.incl(tfIterator) + var oldPrc = GenericCacheGet(c, entry) if oldPrc == nil: c.generics.generics.add(entry) diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index 68d485f48..712a6d2af 100755 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -713,6 +713,12 @@ proc doDestructorStuff(c: PContext, s: PSym, n: PNode) = useSym(t.sons[i].destructor), n.sons[paramsPos][1][0]])) +proc maybeAddResult(c: PContext, s: PSym, n: PNode) = + if s.typ.sons[0] != nil and + (s.kind != skIterator or s.typ.callConv == ccClosure): + addResult(c, s.typ.sons[0], n.info, s.kind) + addResultNode(c, n) + proc semProcAux(c: PContext, n: PNode, kind: TSymKind, validPragmas: TSpecialWords): PNode = result = semProcAnnotation(c, n) @@ -794,17 +800,13 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind, if n.sons[genericParamsPos].kind == nkEmpty: ParamsTypeCheck(c, s.typ) pushProcCon(c, s) - if s.typ.sons[0] != nil and - (kind != skIterator or s.typ.callConv == ccClosure): - addResult(c, s.typ.sons[0], n.info, kind) - addResultNode(c, n) + maybeAddResult(c, s, n) if sfImportc notin s.flags: # no semantic checking for importc: let semBody = hloBody(c, semProcBody(c, n.sons[bodyPos])) # unfortunately we cannot skip this step when in 'system.compiles' # context as it may even be evaluated in 'system.compiles': n.sons[bodyPos] = transformBody(c.module, semBody, s) - #if s.typ.sons[0] != nil and kind != skIterator: addResultNode(c, n) popProcCon(c) else: if s.typ.sons[0] != nil and kind != skIterator: diff --git a/todo.txt b/todo.txt index f5e6d8092..b79b617c0 100755 --- a/todo.txt +++ b/todo.txt @@ -3,7 +3,7 @@ version 0.9.2 - test&finish first class iterators: * nested iterators - * test generic iterators + * document them - fix closure bug finally - fix marshal bug From d16c04d8f8fe8d61016e99c6fcccc45fff66d075 Mon Sep 17 00:00:00 2001 From: Araq Date: Sun, 16 Dec 2012 20:12:56 +0100 Subject: [PATCH 040/106] don't invoke destructors for .global. variables --- compiler/semstmts.nim | 3 ++- todo.txt | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index 712a6d2af..abd064a62 100755 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -1078,7 +1078,8 @@ proc insertDestructors(c: PContext, varSection: PNode): varTyp = varId.sym.typ info = varId.info - if varTyp != nil and instantiateDestructor(c, varTyp): + if varTyp != nil and instantiateDestructor(c, varTyp) and + sfGlobal notin varId.sym.flags: var tryStmt = newNodeI(nkTryStmt, info) if j < totalVars - 1: diff --git a/todo.txt b/todo.txt index b79b617c0..cd4b63942 100755 --- a/todo.txt +++ b/todo.txt @@ -1,11 +1,12 @@ version 0.9.2 ============= +- fix closure bug finally - test&finish first class iterators: * nested iterators * document them -- fix closure bug finally - fix marshal bug +- investigate nimgame bug version 0.9.X From 3be576222a2e2a774570eb408d43c35ab94c5f15 Mon Sep 17 00:00:00 2001 From: Araq Date: Mon, 17 Dec 2012 19:30:43 +0100 Subject: [PATCH 041/106] fixes #277; documentation improvements --- compiler/ccgstmts.nim | 4 + compiler/cgen.nim | 2 - doc/manual.txt | 299 ++++++++++++++++++++++++++---------------- doc/tut1.txt | 7 +- todo.txt | 1 - 5 files changed, 198 insertions(+), 115 deletions(-) diff --git a/compiler/ccgstmts.nim b/compiler/ccgstmts.nim index 089f7fb83..9d5659f5e 100755 --- a/compiler/ccgstmts.nim +++ b/compiler/ccgstmts.nim @@ -124,6 +124,7 @@ proc genBreakState(p: BProc, n: PNode) = lineF(p, cpsStmts, "if ((((NI*) $1.ClEnv)[0]) < 0) break;$n", [rdLoc(a)]) # lineF(p, cpsStmts, "if (($1) < 0) break;$n", [rdLoc(a)]) +proc genVarPrototypeAux(m: BModule, sym: PSym) proc genSingleVar(p: BProc, a: PNode) = var v = a.sons[0].sym if sfCompileTime in v.flags: return @@ -142,6 +143,9 @@ proc genSingleVar(p: BProc, a: PNode) = genObjectInit(p.module.preInitProc, cpsInit, v.typ, v.loc, true) # Alternative construction using default constructor (which may zeromem): # if sfImportc notin v.flags: constructLoc(p.module.preInitProc, v.loc) + if sfExportc in v.flags and generatedHeader != nil: + genVarPrototypeAux(generatedHeader, v) + else: assignLocalVar(p, v) initLocalVar(p, v, immediateAsgn) diff --git a/compiler/cgen.nim b/compiler/cgen.nim index 5bdbefde1..d024f3479 100755 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -799,8 +799,6 @@ proc genVarPrototypeAux(m: BModule, sym: PSym) = proc genVarPrototype(m: BModule, sym: PSym) = genVarPrototypeAux(m, sym) - if sfExportc in sym.flags and generatedHeader != nil: - genVarPrototypeAux(generatedHeader, sym) proc addIntTypes(result: var PRope) {.inline.} = appf(result, "#define NIM_INTBITS $1", [ diff --git a/doc/manual.txt b/doc/manual.txt index d8d53085c..2a79c0f99 100755 --- a/doc/manual.txt +++ b/doc/manual.txt @@ -479,11 +479,8 @@ The grammar's start symbol is ``module``. -Semantics -========= - Types ------ +===== All expressions have a `type`:idx: which is known at compile time. Nimrod is statically typed. One can declare new types, which is in essence defining @@ -502,7 +499,7 @@ These are the major type classes: Ordinal types -~~~~~~~~~~~~~ +------------- `Ordinal types`:idx: have the following characteristics: - Ordinal types are countable and ordered. This property allows @@ -519,7 +516,7 @@ the types ``uint`` and ``uint64`` are no ordinal types. Pre-defined integer types -~~~~~~~~~~~~~~~~~~~~~~~~~ +------------------------- These integer types are pre-defined: ``int`` @@ -602,7 +599,7 @@ For further details, see `Convertible relation`_. Subrange types -~~~~~~~~~~~~~~ +-------------- A `subrange`:idx: type is a range of values from an ordinal type (the base type). To define a subrange type, one must specify it's limiting values: the lowest and highest value of the type: @@ -642,7 +639,7 @@ This means that the following code is accepted: Pre-defined floating point types -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +-------------------------------- The following floating point types are pre-defined: @@ -703,7 +700,7 @@ the ``+``, ``-``, ``*``, ``/`` operators for floating point types. Boolean type -~~~~~~~~~~~~ +------------ The `boolean`:idx: type is named `bool`:idx: in Nimrod and can be one of the two pre-defined values ``true`` and ``false``. Conditions in while, if, elif, when statements need to be of type bool. @@ -727,7 +724,7 @@ The size of the bool type is one byte. Character type -~~~~~~~~~~~~~~ +-------------- The `character type`:idx: is named ``char`` in Nimrod. Its size is one byte. Thus it cannot represent an UTF-8 character, but a part of it. The reason for this is efficiency: for the overwhelming majority of use-cases, @@ -741,7 +738,7 @@ character. ``TRune`` is declared in the ``unicode`` module. Enumeration types -~~~~~~~~~~~~~~~~~ +----------------- `Enumeration`:idx: types define a new type whose values consist of the ones specified. The values are ordered. Example: @@ -811,7 +808,7 @@ via ``TMyEnum.value``: String type -~~~~~~~~~~~ +----------- All string literals are of the type `string`:idx:. A string in Nimrod is very similar to a sequence of characters. However, strings in Nimrod are both zero-terminated and have a length field. One can retrieve the length with the @@ -838,7 +835,7 @@ module can be used for iteration over all Unicode characters. CString type -~~~~~~~~~~~~ +------------ The `cstring`:idx: type represents a pointer to a zero-terminated char array compatible to the type ``char*`` in Ansi C. Its primary purpose lies in easy interfacing with C. The index operation ``s[i]`` means the i-th *char* of @@ -864,13 +861,13 @@ not work. Structured types -~~~~~~~~~~~~~~~~ +---------------- A variable of a `structured type`:idx: can hold multiple values at the same time. Structured types can be nested to unlimited levels. Arrays, sequences, tuples, objects and sets belong to the structured types. Array and sequence types -~~~~~~~~~~~~~~~~~~~~~~~~ +------------------------ `Arrays`:idx: are a homogeneous type, meaning that each element in the array has the same type. Arrays always have a fixed length which is specified at compile time (except for open arrays). They can be indexed by any ordinal type. @@ -918,7 +915,7 @@ The current implementation does not support nested open arrays. Varargs -~~~~~~~ +------- A `varargs`:idx: parameter is an openarray parameter that additionally allows to pass a variable number of arguments to a procedure. The compiler @@ -954,7 +951,7 @@ parameter ``a``. (Note that ``$`` applied to strings is a nop.) Tuples and object types -~~~~~~~~~~~~~~~~~~~~~~~ +----------------------- A variable of a `tuple`:idx: or `object`:idx: type is a heterogeneous storage container. A tuple or object defines various named *fields* of a type. A tuple also @@ -1022,7 +1019,7 @@ introduce new object roots apart from ``system.TObject``. Object variants -~~~~~~~~~~~~~~~ +--------------- Often an object hierarchy is overkill in certain situations where simple `variant`:idx: types are needed. @@ -1069,7 +1066,7 @@ the ``case`` statement: The branches in a ``case`` section may be indented too. Set type -~~~~~~~~ +-------- The `set type`:idx: models the mathematical notion of a set. The set's basetype can only be an ordinal type. The reason is that sets are implemented as high performance bit vectors. @@ -1104,7 +1101,7 @@ operation meaning Reference and pointer types -~~~~~~~~~~~~~~~~~~~~~~~~~~~ +--------------------------- References (similar to `pointers`:idx: in other programming languages) are a way to introduce many-to-one relationships. This means different references can point to and modify the same location in memory (also called `aliasing`:idx:). @@ -1193,14 +1190,14 @@ mysterious crashes. **Note**: The example only works because the memory is initialized to zero (``alloc0`` instead of ``alloc`` does this): ``d.s`` is thus initialized to -``nil`` which the string assignment can handle. You need to know low level +``nil`` which the string assignment can handle. One needs to know low level details like this when mixing garbage collected data with unmanaged memory. .. XXX finalizers for traced objects Not nil annotation -~~~~~~~~~~~~~~~~~~ +------------------ All types for that ``nil`` is a valid value can be annotated to exclude ``nil`` as a valid value with the `not nil`:idx: annotation: @@ -1225,7 +1222,7 @@ for now the compiler can only catch the most trivial type violations. Procedural type -~~~~~~~~~~~~~~~ +--------------- A `procedural type`:idx: is internally a pointer to a procedure. ``nil`` is an allowed value for variables of a procedural type. Nimrod uses procedural types to achieve `functional`:idx: programming techniques. @@ -1335,7 +1332,7 @@ accesses its environment. If it does so, it has the calling convention Distinct type -~~~~~~~~~~~~~ +------------- A distinct type is new type derived from a `base type`:idx: that is incompatible with its base type. In particular, it is an essential property @@ -1431,9 +1428,9 @@ currency. This can be solved with templates_. Void type -~~~~~~~~~ +--------- -The `void`:idx: type denotes the absence of any type. Parameters of +The `void`:idx: type denotes the absense of any type. Parameters of type ``void`` are treated as non-existent, ``void`` as a return type means that the procedure does not return a value: @@ -1471,14 +1468,14 @@ cannot have the type ``void``. Type relations --------------- +============== The following section defines several relations on types that are needed to describe the type checking done by the compiler. Type equality -~~~~~~~~~~~~~ +------------- Nimrod uses structural type equivalence for most types. Only for objects, enumerations and distinct types name equivalence is used. The following algorithm (in pseudo-code) determines type equality: @@ -1523,7 +1520,7 @@ auxiliary set ``s`` to detect this case. Type equality modulo type distinction -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +------------------------------------- The following algorithm (in pseudo-code) determines whether two types are equal with no respect to ``distinct`` types. For brevity the cycle check @@ -1565,7 +1562,7 @@ with an auxiliary set ``s`` is omitted: Subtype relation -~~~~~~~~~~~~~~~~ +---------------- If object ``a`` inherits from ``b``, ``a`` is a subtype of ``b``. This subtype relation is extended to the types ``var``, ``ref``, ``ptr``: @@ -1584,7 +1581,7 @@ relation is extended to the types ``var``, ``ref``, ``ptr``: Convertible relation -~~~~~~~~~~~~~~~~~~~~ +-------------------- A type ``a`` is **implicitly** convertible to type ``b`` iff the following algorithm returns true: @@ -1651,20 +1648,21 @@ The type conversion ``T(a)`` is an L-value if ``a`` is an L-value and Assignment compatibility -~~~~~~~~~~~~~~~~~~~~~~~~ +------------------------ An expression ``b`` can be assigned to an expression ``a`` iff ``a`` is an `l-value` and ``isImplicitlyConvertible(b.typ, a.typ)`` holds. Overloading resolution -~~~~~~~~~~~~~~~~~~~~~~ +---------------------- To be written. Statements and expressions --------------------------- +========================== + Nimrod uses the common statement/expression paradigm: `Statements`:idx: do not produce a value in contrast to expressions. Call expressions are statements. If the called procedure returns a value, it is not a valid statement @@ -1698,7 +1696,7 @@ statements always have to be intended:: Discard statement -~~~~~~~~~~~~~~~~~ +----------------- Syntax:: @@ -1729,7 +1727,7 @@ been declared with the `discardable`:idx: pragma: Var statement -~~~~~~~~~~~~~ +------------- Syntax:: @@ -1790,7 +1788,7 @@ If a proc is annotated with the ``noinit`` pragma this refers to its implicit let statement -~~~~~~~~~~~~~ +------------- A `Let`:idx: statement declares new local and global `single assignment`:idx: variables and binds a value to them. The syntax is the of the ``var`` @@ -1802,7 +1800,7 @@ For let variables the same pragmas are available as for ordinary variables. Const section -~~~~~~~~~~~~~ +------------- Syntax:: @@ -1840,7 +1838,7 @@ they contain such a type. Static statement/expression -~~~~~~~~~~~~~~~~~~~~~~~~~~~ +--------------------------- Syntax:: staticExpr ::= 'static' '(' optInd expr optPar ')' @@ -1865,7 +1863,7 @@ support the FFI at compile time. If statement -~~~~~~~~~~~~ +------------ Syntax:: @@ -1895,7 +1893,7 @@ part, execution continues with the statement after the ``if`` statement. Case statement -~~~~~~~~~~~~~~ +-------------- Syntax:: @@ -1961,7 +1959,7 @@ a list of its elements: When statement -~~~~~~~~~~~~~~ +-------------- Syntax:: @@ -1995,7 +1993,7 @@ within ``object`` definitions. Return statement -~~~~~~~~~~~~~~~~ +---------------- Syntax:: @@ -2026,7 +2024,7 @@ variables, ``result`` is initialized to (binary) zero: Yield statement -~~~~~~~~~~~~~~~ +--------------- Syntax:: @@ -2046,7 +2044,7 @@ for further information. Block statement -~~~~~~~~~~~~~~~ +--------------- Syntax:: @@ -2071,7 +2069,7 @@ block to specify which block is to leave. Break statement -~~~~~~~~~~~~~~~ +--------------- Syntax:: @@ -2088,7 +2086,7 @@ absent, the innermost block is left. While statement -~~~~~~~~~~~~~~~ +--------------- Syntax:: @@ -2110,7 +2108,7 @@ so that they can be left with a ``break`` statement. Continue statement -~~~~~~~~~~~~~~~~~~ +------------------ Syntax:: @@ -2137,7 +2135,7 @@ Is equivalent to: Assembler statement -~~~~~~~~~~~~~~~~~~~ +------------------- Syntax:: asmStmt ::= 'asm' [pragma] (STR_LIT | RSTR_LIT | TRIPLESTR_LIT) @@ -2159,7 +2157,7 @@ specified in the statement's pragmas. The default special character is ``'`'``: """ If expression -~~~~~~~~~~~~~ +------------- An `if expression` is almost like an if statement, but it is an expression. Example: @@ -2172,12 +2170,12 @@ required. ``Elif`` parts are also allowed (but unlikely to be good style). When expression -~~~~~~~~~~~~~~~ +--------------- Just like an `if expression`, but corresponding to the when statement. Case expression -~~~~~~~~~~~~~~~ +--------------- The `case expression` is again very similar to the case statement: @@ -2195,7 +2193,7 @@ effects. When multiple statements are given for a branch, Nimrod will use the last expression as the result value, much like in an `expr` template. Table constructor -~~~~~~~~~~~~~~~~~ +----------------- A `table constructor`:idx: is syntactic sugar for an array constructor: @@ -2223,7 +2221,7 @@ has lots of advantages: Type conversions -~~~~~~~~~~~~~~~~ +---------------- Syntactically a `type conversion` is like a procedure call, but a type name replaces the procedure name. A type conversion is always safe in the sense that a failure to convert a type to another @@ -2231,7 +2229,7 @@ results in an exception (if it cannot be determined statically). Type casts -~~~~~~~~~~ +---------- Example: @@ -2244,7 +2242,7 @@ only needed for low-level programming and are inherently unsafe. The addr operator -~~~~~~~~~~~~~~~~~ +----------------- The `addr` operator returns the address of an l-value. If the type of the location is ``T``, the `addr` operator result is of the type ``ptr T``. Taking the address of an object that resides @@ -2253,7 +2251,8 @@ object on the stack and can thus reference a non-existing object. Procedures -~~~~~~~~~~ +========== + What most programming languages call `methods`:idx: or `functions`:idx: are called `procedures`:idx: in Nimrod (which is the correct terminology). A procedure declaration defines an identifier and associates it with a block @@ -2332,7 +2331,7 @@ notation. (Thus an operator can have more than two parameters): assert `*+`(3, 4, 6) == `*`(a, `+`(b, c)) Closures -~~~~~~~~ +-------- Procedures can appear at the top level in a module as well as inside other scopes, in which case they are called nested procs. A nested proc can access @@ -2344,7 +2343,7 @@ visible in both places). The closure environment may be allocated on the heap or on the stack if the compiler determines that this would be safe. Anonymous Procs -~~~~~~~~~~~~~~~ +--------------- Procs can also be treated as expressions, in which case it's allowed to omit the proc's name. @@ -2360,7 +2359,7 @@ Procs as expressions can appear both as nested procs and inside top level executable code. Do notation -~~~~~~~~~~~ +----------- As a special more convenient notation, proc expressions involved in procedure calls can use the ``do`` keyword: @@ -2399,7 +2398,7 @@ The compatibility works in the other direction too as the ``do`` syntax can be used with macros and templates expecting ``stmt`` blocks. Nonoverloadable builtins -~~~~~~~~~~~~~~~~~~~~~~~~ +------------------------ The following builtin procs cannot be overloaded for reasons of implementation simplicity (they require specialized semantic checking):: @@ -2413,7 +2412,7 @@ the ``system`` module. Var parameters -~~~~~~~~~~~~~~ +-------------- The type of a parameter may be prefixed with the ``var`` keyword: .. code-block:: nimrod @@ -2465,7 +2464,7 @@ One can use `tuple unpacking`:idx: to access the tuple's fields: Var return type -~~~~~~~~~~~~~~~ +--------------- A proc, converter or iterator may return a ``var`` type which means that the returned value is an l-value and can be modified by the caller: @@ -2499,7 +2498,7 @@ starts with the prefix ``m`` per convention. Overloading of the subscript operator -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +------------------------------------- The ``[]`` subscript operator for arrays/openarrays/sequences can be overloaded. Overloading support is only possible if the first parameter has no type that @@ -2508,7 +2507,7 @@ does not check this restriction. Multi-methods -~~~~~~~~~~~~~ +============= Procedures always use static dispatch. `Multi-methods`:idx: use dynamic dispatch. @@ -2580,7 +2579,7 @@ evaluation or dead code elimination do not work with methods. Iterators and the for statement -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +=============================== Syntax:: @@ -2598,8 +2597,9 @@ Syntax:: The `for`:idx: statement is an abstract mechanism to iterate over the elements of a container. It relies on an `iterator`:idx: to do so. Like ``while`` statements, ``for`` statements open an `implicit block`:idx:, so that they -can be left with a ``break`` statement. The ``for`` loop declares +can be left with a ``break`` statement. +The ``for`` loop declares iteration variables (``x`` in the example) - their scope reaches until the end of the loop body. The iteration variables' types are inferred by the return type of the iterator. @@ -2632,18 +2632,13 @@ The compiler generates code as if the programmer would have written this: echo(ch) inc(i) -The current implementation always inlines the iterator code leading to zero -overhead for the abstraction. But this may increase the code size. Later -versions of the compiler will only inline iterators which have the calling -convention ``inline``. - If the iterator yields a tuple, there have to be as many iteration variables as there are components in the tuple. The i'th iteration variable's type is the type of the i'th component. Implict items/pairs invocations -+++++++++++++++++++++++++++++++ +------------------------------- If the for loop expression ``e`` does not denote an iterator and the for loop has exactly 1 variable, the for loop expression is rewritten to ``items(e)``; @@ -2660,8 +2655,90 @@ the rewriting step, so that all overloadings of ``items``/``pairs`` are taken into account. +First class iterators +--------------------- + +There are 2 kinds of iterators in Nimrod: *inline* and *closure* iterators. +An `inline iterator`:idx: is an iterator that's always inlined by the compiler +leading to zero overhead for the abstraction, but may result in a heavy +increasee in code size. Inline iterators are second class +citizens; one cannot pass them around like first class procs. + +In contrast to that, a `closure iterator`:idx: can be passed around: + +.. code-block:: nimrod + iterator count0(): int {.closure.} = + yield 0 + + iterator count2(): int {.closure.} = + var x = 1 + yield x + inc x + yield x + + proc invoke(iter: iterator(): int {.closure.}) = + for x in iter(): echo x + + invoke(count0) + invoke(count2) + + +Closure iterators have other restrictions than inline iterators: + +1.) ``yield`` in a closure iterator can not occur in a ``try`` statement. +2.) For now, a closure iterator cannot be evaluated at compile time. +3.) ``return`` is allowed in a closure iterator (but rarely useful). +4.) Since closure iterators can be used as a collaborative tasking + system, ``void`` is a valid return type for them. +5.) Both inline and closure iterators cannot be recursive. + + +Iterators that are neither marked ``{.closure.}`` nor ``{.inline.}`` explicitly +default to being inline, but that this may change in future versions of the +implementation. + +The ``iterator`` type is always of the calling convention ``closure`` +implicitly; the following example shows how to use iterators to implement +a `collaborative tasking`:idx: system: + +.. code-block:: nimrod + # simple tasking: + type + TTask = iterator (ticker: int) + + iterator a1(ticker: int) {.closure.} = + echo "a1: A" + yield + echo "a1: B" + yield + echo "a1: C" + yield + echo "a1: D" + + iterator a2(ticker: int) {.closure.} = + echo "a2: A" + yield + echo "a2: B" + yield + echo "a2: C" + + proc runTasks(t: varargs[TTask]) = + var ticker = 0 + while true: + let x = t[ticker mod t.len] + if finished(x): break + x(ticker) + inc ticker + + runTasks(a1, a2) + +The builtin ``system.finished`` can be used to determine if an iterator has +finished its operation; no exception is raised on an attempt to invoke an +iterator that has already finished its work. + + Type sections -~~~~~~~~~~~~~ +============= Syntax:: @@ -2697,10 +2774,10 @@ possible within a single ``type`` section. Exception handling ------------------- +================== Try statement -~~~~~~~~~~~~~ +------------- Syntax:: @@ -2755,7 +2832,7 @@ is not executed (if an exception occurs). Except and finally statements -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +----------------------------- `except`:idx: and `finally`:idx: can also be used as a stand-alone statements. Any statements following them in the current block will be considered to be @@ -2768,7 +2845,7 @@ in an implicit try block: Raise statement -~~~~~~~~~~~~~~~ +--------------- Syntax:: @@ -2791,7 +2868,7 @@ exception (unless a raise hook has been provided). OnRaise builtin -~~~~~~~~~~~~~~~ +--------------- ``system.onRaise`` can be used to override the behaviour of ``raise`` for a single ``try`` statement. `onRaise`:idx: has to be called within the ``try`` @@ -2815,10 +2892,10 @@ This allows for a Lisp-like `condition system`:idx:\: Effect system -------------- +============= Exception tracking -~~~~~~~~~~~~~~~~~~ +------------------ Nimrod supports `exception tracking`:idx:. The `raises`:idx: pragma can used to explicitly define which exceptions a proc/iterator/method/converter is allowed @@ -2873,7 +2950,7 @@ possibly raised exceptions; the algorithm operates on ``p``'s call graph: Tag tracking -~~~~~~~~~~~~ +------------ The exception tracking is part of Nimrod's `effect system`:idx:. Raising an exception is an *effect*. Other effects can also be defined. A user defined @@ -2895,7 +2972,7 @@ exception tracking. Read/Write tracking -~~~~~~~~~~~~~~~~~~~ +------------------- **Note**: Read/write tracking is not yet implemented! @@ -2904,7 +2981,7 @@ exception tracking. Effects pragma -~~~~~~~~~~~~~~ +-------------- The `effects`:idx: pragma has been designed to assist the programmer with the effects analysis. It is a statement that makes the compiler output all inferred @@ -2923,7 +3000,7 @@ listed as it cannot be raised in the branch the ``effects`` pragma appears in. Generics --------- +======== Example: @@ -2981,7 +3058,7 @@ introduce type parameters or to instantiate a generic proc, iterator or type. Is operator -~~~~~~~~~~~ +----------- The `is`:idx: operator checks for type equivalence at compile time. It is therefore very useful for type specialization within generic code: @@ -2996,7 +3073,7 @@ therefore very useful for type specialization within generic code: Type operator -~~~~~~~~~~~~~ +------------- The `type`:idx: (in many other languages called `typeof`:idx:) operator can be used to get the type of an expression: @@ -3019,7 +3096,7 @@ other interpretations: Type Classes -~~~~~~~~~~~~ +------------ A `type class`:idx: is a special pseudo-type that can be used to match against types in the context of overload resolution or the ``is`` operator. @@ -3108,13 +3185,15 @@ the dot syntax: If anonymous type classes are used, the ``type`` operator can be used to discover the instantiated type of each param. + User defined type classes -~~~~~~~~~~~~~~~~~~~~~~~~~ +------------------------- To be written. + Return Type Inference -~~~~~~~~~~~~~~~~~~~~~ +--------------------- If a type class is used as the return type of a proc and it won't be bound to a concrete type by some of the proc params, Nimrod will infer the return type @@ -3132,7 +3211,7 @@ also influence the inferred return type. Symbol lookup in generics -~~~~~~~~~~~~~~~~~~~~~~~~~ +------------------------- The symbol binding rules in generics are slightly subtle: There are "open" and "closed" symbols. A "closed" symbol cannot be re-bound in the instantiation @@ -3171,7 +3250,7 @@ A symbol can be forced to be open by a `mixin`:idx: declaration: Templates ---------- +========= A `template`:idx: is a simple form of a macro: It is a simple substitution mechanism that operates on Nimrod's abstract syntax trees. It is processed in @@ -3202,7 +3281,7 @@ Real types can be used too; this implies that expressions are expected. Ordinary vs immediate templates -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +------------------------------- There are two different kinds of templates: `immediate`:idx: templates and ordinary templates. Ordinary templates take part in overloading resolution. As @@ -3229,7 +3308,7 @@ receive undeclared identifiers: Scoping in templates -~~~~~~~~~~~~~~~~~~~~ +-------------------- The template body does not open a new scope. To open a new scope a ``block`` statement can be used: @@ -3251,7 +3330,7 @@ statement can be used: Passing a code block to a template -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +---------------------------------- If there is a ``stmt`` parameter it should be the last in the template declaration, because statements are passed to a template via a @@ -3298,7 +3377,7 @@ Symbol binding within templates happens after template instantiation: Bind statement -~~~~~~~~~~~~~~ +-------------- Syntax:: @@ -3329,7 +3408,7 @@ A ``bind`` statement can also be used in generics for the same purpose. Identifier construction -~~~~~~~~~~~~~~~~~~~~~~~ +----------------------- In templates identifiers can be constructed with the backticks notation: @@ -3348,7 +3427,7 @@ In the example ``name`` is instantiated with ``myint``, so \`T name\` becomes Lookup rules for template parameters -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +------------------------------------ A parameter ``p`` in a template is even substituted in the expression ``x.p``. Thus template arguments can be used as field names and a global symbol can be @@ -3389,7 +3468,7 @@ But the global symbol can properly be captured by a ``bind`` statement: Hygiene in templates -~~~~~~~~~~~~~~~~~~~~ +-------------------- Per default templates are `hygienic`:idx:\: Local identifiers declared in a template cannot be accessed in the instantiation context: @@ -3444,7 +3523,7 @@ a template. ``inject`` and ``gensym`` have no effect in ``dirty`` templates. Macros ------- +====== A `macro`:idx: is a special kind of low level template. Macros can be used to implement `domain specific languages`:idx:. Like templates, macros come in @@ -3463,7 +3542,7 @@ There are two ways to invoke a macro: Expression Macros -~~~~~~~~~~~~~~~~~ +----------------- The following example implements a powerful ``debug`` command that accepts a variable number of arguments: @@ -3517,7 +3596,7 @@ children. BindSym -~~~~~~~ +------- The above ``debug`` macro relies on the fact that ``write``, ``writeln`` and ``stdout`` are declared in the system module and thus visible in the @@ -3565,7 +3644,7 @@ overloaded symbols implicitly. Statement Macros -~~~~~~~~~~~~~~~~ +---------------- Statement macros are defined just as expression macros. However, they are invoked by an expression following a colon:: @@ -3609,7 +3688,7 @@ powerful programming construct that still suffices. So the "check list" is: Macros as pragmas -~~~~~~~~~~~~~~~~~ +----------------- Whole routines (procs, iterators etc.) can also be passed to a template or a macro via the pragma notation: @@ -3629,17 +3708,17 @@ This is a simple syntactic transformation into: Special Types -------------- +============= typedesc -~~~~~~~~ +-------- -`typedesc` is a special type allowing you to treat types as compile-time values +`typedesc` is a special type allowing one to treat types as compile-time values (i.e. if types are compile-time values and all values have a type, then typedesc must be their type). When used as a regular proc param, typedesc acts as a type class. The proc -will be instantiated for each unique type parameter and you can refer to the +will be instantiated for each unique type parameter and one can refer to the instantiation type using the param name: .. code-block:: nimrod @@ -3654,8 +3733,8 @@ instantiation type using the param name: When used with macros and .compileTime. procs on the other hand, the compiler don't need to instantiate the code multiple times, because types then can be manipulated using the unified internal symbol representation. In such context -typedesc acts as any other type. You can create variables, store typedesc -values inside containers and so on. For example, here is how we can create +typedesc acts as any other type. One can create variables, store typedesc +values inside containers and so on. For example, here is how one can create a type-safe wrapper for the unsafe `printf` function from C: .. code-block:: nimrod diff --git a/doc/tut1.txt b/doc/tut1.txt index 95d13ceb9..d8d04bc93 100755 --- a/doc/tut1.txt +++ b/doc/tut1.txt @@ -780,12 +780,15 @@ important differences: * Iterators cannot contain a ``return`` statement and procs cannot contain a ``yield`` statement. * Iterators have no implicit ``result`` variable. -* Iterators do not support recursion. (This restriction will be gone in a - future version of the compiler.) +* Iterators do not support recursion. * Iterators cannot be forward declared, because the compiler must be able to inline an iterator. (This restriction will be gone in a future version of the compiler.) +However, you can also use a ``closure`` iterator to get a different set of +restrictions. See `first class iterators `_ +for details. + Basic types =========== diff --git a/todo.txt b/todo.txt index cd4b63942..3d8859fc2 100755 --- a/todo.txt +++ b/todo.txt @@ -4,7 +4,6 @@ version 0.9.2 - fix closure bug finally - test&finish first class iterators: * nested iterators - * document them - fix marshal bug - investigate nimgame bug From acfe55ab906ba2fbe14f3c19f085cabe2cbe25c5 Mon Sep 17 00:00:00 2001 From: Grzegorz Adam Hankiewicz Date: Fri, 14 Dec 2012 23:32:11 +0100 Subject: [PATCH 042/106] Adds macosx dynamic library name for libiconv. Also adds some verbosity to the implicit test. --- lib/pure/encodings.nim | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/lib/pure/encodings.nim b/lib/pure/encodings.nim index 6421b6f44..ce4238409 100755 --- a/lib/pure/encodings.nim +++ b/lib/pure/encodings.nim @@ -254,6 +254,8 @@ when defined(windows): else: when defined(haiku): const iconvDll = "(libc.so.6|libiconv.so|libtextencoding.so)" + elif defined(macosx): + const iconvDll = "libiconv.dylib" else: const iconvDll = "(libc.so.6|libiconv.so)" @@ -449,8 +451,13 @@ proc convert*(s: string, destEncoding = "UTF-8", close(c) when IsMainModule: - var orig = "öäüß" - var crap = convert(orig, "CP1252", "UTF-8") - echo convert(crap, "ibm850", "CP1252") - echo getCurrentEncoding() + let + orig = "öäüß" + cp1252 = convert(orig, "CP1252", "UTF-8") + ibm850 = convert(cp1252, "ibm850", "CP1252") + current = getCurrentEncoding() + echo "Original string from source code: ", orig + echo "Forced ibm850 encoding: ", ibm850 + echo "Current encoding: ", current + echo "From ibm850 to current: ", convert(ibm850, current, "ibm850") From da944e331fd7b02cc380dd2112a4d3559ce4104b Mon Sep 17 00:00:00 2001 From: Grzegorz Adam Hankiewicz Date: Wed, 28 Nov 2012 23:01:22 +0100 Subject: [PATCH 043/106] Adds tools/nimweb to .gitignore. --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 7f6d35f71..81bfbad89 100755 --- a/.gitignore +++ b/.gitignore @@ -6,6 +6,7 @@ nimcache lib/nimcache tools/nimcache +tools/nimweb tests/nimcache tests/accept/run/nimcache tests/accept/compile/nimcache From cb43de875aa611d9c5eab49b59984202f0e11edc Mon Sep 17 00:00:00 2001 From: Grzegorz Adam Hankiewicz Date: Tue, 18 Dec 2012 19:33:04 +0100 Subject: [PATCH 044/106] Adds link to external nimrod-crossplatform-todo repository. --- examples/cross_todo/readme.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/examples/cross_todo/readme.txt b/examples/cross_todo/readme.txt index acea8973a..326ed816f 100644 --- a/examples/cross_todo/readme.txt +++ b/examples/cross_todo/readme.txt @@ -3,3 +3,5 @@ called by different native user interfaces. This example builds on the knowledge learned from the cross_calculator example. Check it out first to learn how to set up nimrod on different platforms. +Additional implementations are provided at the external +https://github.com/gradha/nimrod-crossplatform-todo github repository. From 7148812524d26b89dab7efde2e86d318677b502b Mon Sep 17 00:00:00 2001 From: Araq Date: Wed, 19 Dec 2012 02:22:39 +0100 Subject: [PATCH 045/106] first steps for FFI support at compile time --- compiler/cgen.nim | 11 --- compiler/condsyms.nim | 2 + compiler/evalffi.nim | 198 ++++++++++++++++++++++++++++++++++++++++ compiler/evals.nim | 96 +++++++++++-------- compiler/options.nim | 12 +++ koch.nim | 1 + lib/wrappers/libffi.nim | 149 ++++++++++++++++++++++++++++++ 7 files changed, 420 insertions(+), 49 deletions(-) create mode 100644 compiler/evalffi.nim create mode 100644 lib/wrappers/libffi.nim diff --git a/compiler/cgen.nim b/compiler/cgen.nim index d024f3479..001a6fbee 100755 --- a/compiler/cgen.nim +++ b/compiler/cgen.nim @@ -469,17 +469,6 @@ include "ccgexprs.nim", "ccgstmts.nim" # ----------------------------- dynamic library handling ----------------- # We don't finalize dynamic libs as this does the OS for us. -proc libCandidates(s: string, dest: var TStringSeq) = - var le = strutils.find(s, '(') - var ri = strutils.find(s, ')', le+1) - if le >= 0 and ri > le: - var prefix = substr(s, 0, le - 1) - var suffix = substr(s, ri + 1) - for middle in split(substr(s, le + 1, ri - 1), '|'): - libCandidates(prefix & middle & suffix, dest) - else: - add(dest, s) - proc isGetProcAddr(lib: PLib): bool = let n = lib.path result = n.kind in nkCallKinds and n.typ != nil and diff --git a/compiler/condsyms.nim b/compiler/condsyms.nim index 17366f6e9..4cac7c847 100755 --- a/compiler/condsyms.nim +++ b/compiler/condsyms.nim @@ -61,6 +61,8 @@ proc InitDefines*() = DefineSymbol("nimmixin") DefineSymbol("nimeffects") DefineSymbol("nimbabel") + when defined(useFFI): + DefineSymbol("nimffi") # add platform specific symbols: case targetCPU diff --git a/compiler/evalffi.nim b/compiler/evalffi.nim new file mode 100644 index 000000000..66ed3a87e --- /dev/null +++ b/compiler/evalffi.nim @@ -0,0 +1,198 @@ +# +# +# The Nimrod Compiler +# (c) Copyright 2012 Andreas Rumpf +# +# See the file "copying.txt", included in this +# distribution, for details about the copyright. +# + +## This file implements the FFI part of the evaluator for Nimrod code. + +import ast, astalgo, ropes, types, options, tables, dynlib, libffi, msgs + +when defined(windows): + const libcDll = "msvcrt.dll" +else: + const libcDll = "libc.so(.6|.5|)" + +type + TDllCache* = tables.TTable[string, TLibHandle] +var + gDllCache = initTable[string, TLibHandle]() + +proc getDll(cache: var TDllCache; dll: string): pointer = + result = cache[dll] + if result.isNil: + var libs: seq[string] = @[] + libCandidates(dll, libs) + for c in libs: + result = LoadLib(c) + if not result.isNil: break + if result.isNil: + InternalError("cannot load: " & dll) + cache[dll] = result + +proc importcSymbol*(sym: PSym): PNode = + let lib = sym.annex + if lib != nil and lib.path.kind notin {nkStrLit..nkTripleStrLit}: + InternalError("dynlib needs to be a string literal for the REPL") + + let dllpath = if lib.isNil: libcDll else: lib.path.strVal + let dllhandle = gDllCache.getDll(dllpath) + let name = ropeToStr(sym.loc.r) + let theAddr = dllhandle.checkedSymAddr(name) + + # the AST does not support untyped pointers directly, so we use an nkIntLit + # that contains the address instead: + result = newNodeIT(nkIntLit, sym.info, sym.typ) + result.intVal = cast[TAddress](theAddr) + +proc mapType(t: ast.PType): ptr libffi.TType = + if t == nil: return addr libffi.type_void + + case t.kind + of tyBool, tyEnum, tyChar, tyInt..tyInt64, tyUInt..tyUInt64, tySet: + case t.getSize + of 1: result = addr libffi.type_uint8 + of 2: result = addr libffi.type_sint16 + of 4: result = addr libffi.type_sint32 + of 8: result = addr libffi.type_sint64 + else: + InternalError("cannot map type to FFI") + of tyFloat, tyFloat64: result = addr libffi.type_double + of tyFloat32: result = addr libffi.type_float + of tyVar, tyPointer, tyPtr, tyRef, tyCString, tySequence, tyString, tyExpr, + tyStmt, tyTypeDesc, tyProc, tyArray, tyArrayConstr: + result = addr libffi.type_pointer + else: + InternalError("cannot map type to FFI") + # too risky: + #of tyFloat128: result = addr libffi.type_longdouble + +proc mapCallConv(cc: TCallingConvention): TABI = + case cc + of ccDefault: result = DEFAULT_ABI + of ccStdCall: result = when defined(windows): STDCALL else: DEFAULT_ABI + of ccCDecl: result = DEFAULT_ABI + else: InternalError("cannot map calling convention to FFI") + +template rd(T, p: expr): expr {.immediate.} = (cast[ptr T](p))[] +template wr(T, p, v: expr) {.immediate.} = (cast[ptr T](p))[] = v + +proc pack(v: PNode): pointer = + template awr(T, v: expr) {.immediate, dirty.} = + result = alloc0(sizeof(T)) + wr(T, result, v) + + case v.typ.kind + of tyBool: awr(bool, v.intVal != 0) + of tyChar: awr(char, v.intVal.chr) + of tyInt: awr(int, v.intVal.int) + of tyInt8: awr(int8, v.intVal.int8) + of tyInt16: awr(int16, v.intVal.int16) + of tyInt32: awr(int32, v.intVal.int32) + of tyInt64: awr(int64, v.intVal.int64) + of tyUInt: awr(uint, v.intVal.uint) + of tyUInt8: awr(uint8, v.intVal.uint8) + of tyUInt16: awr(uint16, v.intVal.uint16) + of tyUInt32: awr(uint32, v.intVal.uint32) + of tyUInt64: awr(uint64, v.intVal.uint64) + of tyEnum, tySet: + case v.typ.getSize + of 1: awr(uint8, v.intVal.uint8) + of 2: awr(uint16, v.intVal.uint16) + of 4: awr(int32, v.intVal.int32) + of 8: awr(int64, v.intVal.int64) + else: + InternalError("cannot map value to FFI (tyEnum, tySet)") + of tyFloat: awr(float, v.floatVal) + of tyFloat32: awr(float32, v.floatVal) + of tyFloat64: awr(float64, v.floatVal) + + of tyPointer, tyProc, tyPtr, tyRef: + if v.kind == nkNilLit: + result = alloc0(sizeof(pointer)) + else: + awr(pointer, cast[pointer](v.intVal)) + of tyCString, tyString: + if v.kind == nkNilLit: + result = alloc0(sizeof(pointer)) + else: + awr(cstring, cstring(v.strVal)) + else: + InternalError("cannot map value to FFI " & typeToString(v.typ)) + +proc unpack(x: pointer, typ: PType, info: TLineInfo): PNode = + template aw(kind, v, field: expr) {.immediate, dirty.} = + result = newNodeIT(kind, info, typ) + result.field = v + + template awi(kind, v: expr) {.immediate, dirty.} = aw(kind, v, intVal) + template awf(kind, v: expr) {.immediate, dirty.} = aw(kind, v, floatVal) + template aws(kind, v: expr) {.immediate, dirty.} = aw(kind, v, strVal) + + case typ.kind + of tyBool: awi(nkIntLit, rd(bool, x).ord) + of tyChar: awi(nkIntLit, rd(char, x).ord) + of tyInt: awi(nkIntLit, rd(int, x)) + of tyInt8: awi(nkIntLit, rd(int8, x)) + of tyInt16: awi(nkIntLit, rd(int16, x)) + of tyInt32: awi(nkIntLit, rd(int32, x)) + of tyInt64: awi(nkIntLit, rd(int64, x)) + of tyUInt: awi(nkIntLit, rd(uint, x).biggestInt) + of tyUInt8: awi(nkIntLit, rd(uint8, x).biggestInt) + of tyUInt16: awi(nkIntLit, rd(uint16, x).biggestInt) + of tyUInt32: awi(nkIntLit, rd(uint32, x).biggestInt) + of tyUInt64: awi(nkIntLit, rd(uint64, x).biggestInt) + of tyEnum: + case typ.getSize + of 1: awi(nkIntLit, rd(uint8, x).biggestInt) + of 2: awi(nkIntLit, rd(uint16, x).biggestInt) + of 4: awi(nkIntLit, rd(int32, x).biggestInt) + of 8: awi(nkIntLit, rd(int64, x).biggestInt) + else: + InternalError("cannot map value from FFI (tyEnum, tySet)") + of tyFloat: awf(nkFloatLit, rd(float, x)) + of tyFloat32: awf(nkFloatLit, rd(float32, x)) + of tyFloat64: awf(nkFloatLit, rd(float64, x)) + of tyPointer, tyProc, tyPtr: + let p = rd(pointer, x) + if p.isNil: + result = newNodeIT(nkNilLit, info, typ) + else: + awi(nkIntLit, cast[TAddress](p)) + of tyCString, tyString: + let p = rd(cstring, x) + if p.isNil: + result = newNodeIT(nkNilLit, info, typ) + else: + aws(nkStrLit, $p) + else: + InternalError("cannot map value from FFI " & typeToString(typ)) + +proc callForeignFunction*(call: PNode): PNode = + InternalAssert call.sons[0].kind == nkIntLit + let typ = call.sons[0].typ + + var cif: TCif + var sig: TParamList + for i in 1..typ.len-1: sig[i-1] = mapType(typ.sons[i]) + + if prep_cif(cif, mapCallConv(typ.callConv), cuint(typ.len-1), + mapType(typ.sons[0]), sig) != OK: + InternalError(call.info, "error in FFI call") + + var args: TArgList + let fn = cast[pointer](call.sons[0].intVal) + for i in 0 .. call.len-1: + args[i] = pack(call.sons[i+1]) + let retVal = alloc(typ.sons[0].getSize.int) + + libffi.call(cif, fn, retVal, args) + + if isEmptyType(typ.sons[0]): result = emptyNode + else: result = unpack(retVal, typ.sons[0], call.info) + + dealloc retVal + for i in countdown(call.len-1, 0): dealloc args[i] diff --git a/compiler/evals.nim b/compiler/evals.nim index 20d354aaf..535d9aebd 100755 --- a/compiler/evals.nim +++ b/compiler/evals.nim @@ -18,6 +18,9 @@ import msgs, os, condsyms, idents, renderer, types, passes, semfold, transf, parser, ropes, rodread, idgen, osproc, streams, evaltempl +when hasFFI: + import evalffi + type PStackFrame* = ref TStackFrame TStackFrame*{.final.} = object @@ -307,42 +310,6 @@ proc evalVar(c: PEvalContext, n: PNode): PNode = for i in countup(0, sonsLen(result) - 1): addSon(x, result.sons[i]) result = emptyNode -proc evalCall(c: PEvalContext, n: PNode): PNode = - var d = newStackFrame() - d.call = n - var prc = n.sons[0] - let isClosure = prc.kind == nkClosure - setlen(d.params, sonsLen(n) + ord(isClosure)) - if isClosure: - #debug prc - result = evalAux(c, prc.sons[1], {efLValue}) - if isSpecial(result): return - d.params[sonsLen(n)] = result - result = evalAux(c, prc.sons[0], {}) - else: - result = evalAux(c, prc, {}) - - if isSpecial(result): return - prc = result - # bind the actual params to the local parameter of a new binding - if prc.kind != nkSym: - InternalError(n.info, "evalCall " & n.renderTree) - return - d.prc = prc.sym - if prc.sym.kind notin {skProc, skConverter, skMacro}: - InternalError(n.info, "evalCall") - return - for i in countup(1, sonsLen(n) - 1): - result = evalAux(c, n.sons[i], {}) - if isSpecial(result): return - d.params[i] = result - if n.typ != nil: d.params[0] = getNullValue(n.typ, n.info) - pushStackFrame(c, d) - result = evalAux(c, prc.sym.getBody, {}) - if result.kind == nkExceptBranch: return - if n.typ != nil: result = d.params[0] - popStackFrame(c) - proc aliasNeeded(n: PNode, flags: TEvalFlags): bool = result = efLValue in flags or n.typ == nil or n.typ.kind in {tyExpr, tyStmt, tyTypeDesc} @@ -374,7 +341,14 @@ proc evalGlobalVar(c: PEvalContext, s: PSym, flags: TEvalFlags): PNode = else: result = s.ast if result == nil or result.kind == nkEmpty: - result = getNullValue(s.typ, s.info) + when hasFFI: + # for 'stdin' etc. we need to support 'importc' for variables: + if sfImportc in s.flags: + result = importcSymbol(s) + else: + result = getNullValue(s.typ, s.info) + else: + result = getNullValue(s.typ, s.info) else: result = evalAux(c, result, {}) if isSpecial(result): return @@ -382,6 +356,51 @@ proc evalGlobalVar(c: PEvalContext, s: PSym, flags: TEvalFlags): PNode = else: result = raiseCannotEval(nil, s.info) +proc evalCall(c: PEvalContext, n: PNode): PNode = + var d = newStackFrame() + d.call = n + var prc = n.sons[0] + let isClosure = prc.kind == nkClosure + setlen(d.params, sonsLen(n) + ord(isClosure)) + if isClosure: + #debug prc + result = evalAux(c, prc.sons[1], {efLValue}) + if isSpecial(result): return + d.params[sonsLen(n)] = result + result = evalAux(c, prc.sons[0], {}) + else: + result = evalAux(c, prc, {}) + + if isSpecial(result): return + prc = result + # bind the actual params to the local parameter of a new binding + if prc.kind != nkSym: + InternalError(n.info, "evalCall " & n.renderTree) + return + d.prc = prc.sym + if prc.sym.kind notin {skProc, skConverter, skMacro}: + InternalError(n.info, "evalCall") + return + for i in countup(1, sonsLen(n) - 1): + result = evalAux(c, n.sons[i], {}) + if isSpecial(result): return + d.params[i] = result + if n.typ != nil: d.params[0] = getNullValue(n.typ, n.info) + + when hasFFI: + if sfImportc in prc.sym.flags: + var newCall = newNodeI(nkCall, n.info, n.len) + newCall.sons[0] = evalGlobalVar(c, prc.sym, {}) + for i in 1 .. = 0 and ri > le: + var prefix = substr(s, 0, le - 1) + var suffix = substr(s, ri + 1) + for middle in split(substr(s, le + 1, ri - 1), '|'): + libCandidates(prefix & middle & suffix, dest) + else: + add(dest, s) + proc binaryStrSearch*(x: openarray[string], y: string): int = var a = 0 var b = len(x) - 1 diff --git a/koch.nim b/koch.nim index 6bc63a6ce..029e8a733 100755 --- a/koch.nim +++ b/koch.nim @@ -49,6 +49,7 @@ Boot options: -d:tinyc include the Tiny C backend (not supported on Windows) -d:useGnuReadline use the GNU readline library for interactive mode (not needed on Windows) + -d:useFFI build Nimrod with FFI support at compile time -d:nativeStacktrace use native stack traces (only for Mac OS X or Linux) """ diff --git a/lib/wrappers/libffi.nim b/lib/wrappers/libffi.nim new file mode 100644 index 000000000..514ce024f --- /dev/null +++ b/lib/wrappers/libffi.nim @@ -0,0 +1,149 @@ +# -----------------------------------------------------------------*-C-*- +# libffi 3.0.10 - Copyright (c) 2011 Anthony Green +# - Copyright (c) 1996-2003, 2007, 2008 Red Hat, Inc. +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation +# files (the ``Software''), to deal in the Software without +# restriction, including without limitation the rights to use, copy, +# modify, merge, publish, distribute, sublicense, and/or sell copies +# of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED ``AS IS'', WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT +# HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, +# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +# DEALINGS IN THE SOFTWARE. +# +# ----------------------------------------------------------------------- + +{.deadCodeElim: on.} + +when defined(windows): + const libffidll* = "libffi.dll" +elif defined(macosx): + const libffidll* = "libffi.dylib" +else: + const libffidll* = "libffi.so" + +type + TArg* = int + TSArg* = int + +when defined(windows) and defined(x86): + type + TABI* {.size: sizeof(cint).} = enum + FIRST_ABI, SYSV, STDCALL + + const DEFAULT_ABI* = SYSV +elif defined(amd64) and defined(windows): + type + TABI* {.size: sizeof(cint).} = enum + FIRST_ABI, WIN64 + const DEFAULT_ABI* = WIN64 +else: + type + TABI* {.size: sizeof(cint).} = enum + FIRST_ABI, SYSV, UNIX64 + + when defined(i386): + const DEFAULT_ABI* = SYSV + else: + const DEFAULT_ABI* = UNIX64 + +const + tkVOID* = 0 + tkINT* = 1 + tkFLOAT* = 2 + tkDOUBLE* = 3 + tkLONGDOUBLE* = 4 + tkUINT8* = 5 + tkSINT8* = 6 + tkUINT16* = 7 + tkSINT16* = 8 + tkUINT32* = 9 + tkSINT32* = 10 + tkUINT64* = 11 + tkSINT64* = 12 + tkSTRUCT* = 13 + tkPOINTER* = 14 + + tkLAST = tkPOINTER + tkSMALL_STRUCT_1B* = (tkLAST + 1) + tkSMALL_STRUCT_2B* = (tkLAST + 2) + tkSMALL_STRUCT_4B* = (tkLAST + 3) + +type + TType* = object + size*: int + alignment*: uint16 + typ*: uint16 + elements*: ptr ptr TType + +var + type_void* {.importc: "ffi_type_void", dynlib: libffidll.}: TType + type_uint8* {.importc: "ffi_type_uint8", dynlib: libffidll.}: TType + type_sint8* {.importc: "ffi_type_sint8", dynlib: libffidll.}: TType + type_uint16* {.importc: "ffi_type_uint16", dynlib: libffidll.}: TType + type_sint16* {.importc: "ffi_type_sint16", dynlib: libffidll.}: TType + type_uint32* {.importc: "ffi_type_uint32", dynlib: libffidll.}: TType + type_sint32* {.importc: "ffi_type_sint32", dynlib: libffidll.}: TType + type_uint64* {.importc: "ffi_type_uint64", dynlib: libffidll.}: TType + type_sint64* {.importc: "ffi_type_sint64", dynlib: libffidll.}: TType + type_float* {.importc: "ffi_type_float", dynlib: libffidll.}: TType + type_double* {.importc: "ffi_type_double", dynlib: libffidll.}: TType + type_pointer* {.importc: "ffi_type_pointer", dynlib: libffidll.}: TType + type_longdouble* {.importc: "ffi_type_longdouble", dynlib: libffidll.}: TType + +type + Tstatus* {.size: sizeof(cint).} = enum + OK, BAD_TYPEDEF, BAD_ABI + TTypeKind* = cuint + TCif* {.pure, final.} = object + abi*: TABI + nargs*: cuint + arg_types*: ptr ptr TType + rtype*: ptr TType + bytes*: cuint + flags*: cuint + +type + TRaw* = object + sint*: TSArg + +proc raw_call*(cif: var Tcif; fn: proc () {.cdecl.}; rvalue: pointer; + avalue: ptr TRaw) {.cdecl, importc: "ffi_raw_call", + dynlib: libffidll.} +proc ptrarray_to_raw*(cif: var Tcif; args: ptr pointer; raw: ptr TRaw) {.cdecl, + importc: "ffi_ptrarray_to_raw", dynlib: libffidll.} +proc raw_to_ptrarray*(cif: var Tcif; raw: ptr TRaw; args: ptr pointer) {.cdecl, + importc: "ffi_raw_to_ptrarray", dynlib: libffidll.} +proc raw_size*(cif: var Tcif): int {.cdecl, importc: "ffi_raw_size", + dynlib: libffidll.} + +proc prep_cif*(cif: var Tcif; abi: TABI; nargs: cuint; rtype: ptr TType; + atypes: ptr ptr TType): TStatus {.cdecl, importc: "ffi_prep_cif", + dynlib: libffidll.} +proc call*(cif: var Tcif; fn: proc () {.cdecl.}; rvalue: pointer; + avalue: ptr pointer) {.cdecl, importc: "ffi_call", dynlib: libffidll.} + +# the same with an easier interface: +type + TParamList* = array[0..100, ptr TType] + TArgList* = array[0..100, pointer] + +proc prep_cif*(cif: var Tcif; abi: TABI; nargs: cuint; rtype: ptr TType; + atypes: TParamList): TStatus {.cdecl, importc: "ffi_prep_cif", + dynlib: libffidll.} +proc call*(cif: var Tcif; fn, rvalue: pointer; + avalue: TArgList) {.cdecl, importc: "ffi_call", dynlib: libffidll.} + +# Useful for eliminating compiler warnings +##define FFI_FN(f) ((void (*)(void))f) From c73d9fdd4c5d216b75ae4dc7659ea6c19c13ad15 Mon Sep 17 00:00:00 2001 From: Araq Date: Wed, 19 Dec 2012 08:44:20 +0100 Subject: [PATCH 046/106] small examples work with the FFI --- compiler/evalffi.nim | 57 ++++++++++++++++++++++++++++---------------- compiler/evals.nim | 17 +++++++------ koch.nim | 2 +- todo.txt | 9 +++++-- 4 files changed, 53 insertions(+), 32 deletions(-) diff --git a/compiler/evalffi.nim b/compiler/evalffi.nim index 66ed3a87e..35659ed26 100644 --- a/compiler/evalffi.nim +++ b/compiler/evalffi.nim @@ -34,19 +34,25 @@ proc getDll(cache: var TDllCache; dll: string): pointer = cache[dll] = result proc importcSymbol*(sym: PSym): PNode = - let lib = sym.annex - if lib != nil and lib.path.kind notin {nkStrLit..nkTripleStrLit}: - InternalError("dynlib needs to be a string literal for the REPL") - - let dllpath = if lib.isNil: libcDll else: lib.path.strVal - let dllhandle = gDllCache.getDll(dllpath) let name = ropeToStr(sym.loc.r) - let theAddr = dllhandle.checkedSymAddr(name) # the AST does not support untyped pointers directly, so we use an nkIntLit # that contains the address instead: result = newNodeIT(nkIntLit, sym.info, sym.typ) - result.intVal = cast[TAddress](theAddr) + case name + of "stdin": result.intVal = cast[TAddress](system.stdin) + of "stdout": result.intVal = cast[TAddress](system.stdout) + of "stderr": result.intVal = cast[TAddress](system.stderr) + else: + let lib = sym.annex + if lib != nil and lib.path.kind notin {nkStrLit..nkTripleStrLit}: + InternalError(sym.info, "dynlib needs to be a string literal for the REPL") + + let dllpath = if lib.isNil: libcDll else: lib.path.strVal + let dllhandle = gDllCache.getDll(dllpath) + let theAddr = dllhandle.checkedSymAddr(name) + + result.intVal = cast[TAddress](theAddr) proc mapType(t: ast.PType): ptr libffi.TType = if t == nil: return addr libffi.type_void @@ -63,8 +69,10 @@ proc mapType(t: ast.PType): ptr libffi.TType = of tyFloat, tyFloat64: result = addr libffi.type_double of tyFloat32: result = addr libffi.type_float of tyVar, tyPointer, tyPtr, tyRef, tyCString, tySequence, tyString, tyExpr, - tyStmt, tyTypeDesc, tyProc, tyArray, tyArrayConstr: + tyStmt, tyTypeDesc, tyProc, tyArray, tyArrayConstr, tyNil: result = addr libffi.type_pointer + of tyDistinct: + result = mapType(t.sons[0]) else: InternalError("cannot map type to FFI") # too risky: @@ -80,12 +88,12 @@ proc mapCallConv(cc: TCallingConvention): TABI = template rd(T, p: expr): expr {.immediate.} = (cast[ptr T](p))[] template wr(T, p, v: expr) {.immediate.} = (cast[ptr T](p))[] = v -proc pack(v: PNode): pointer = +proc pack(v: PNode, typ: PType): pointer = template awr(T, v: expr) {.immediate, dirty.} = result = alloc0(sizeof(T)) wr(T, result, v) - case v.typ.kind + case typ.kind of tyBool: awr(bool, v.intVal != 0) of tyChar: awr(char, v.intVal.chr) of tyInt: awr(int, v.intVal.int) @@ -120,6 +128,10 @@ proc pack(v: PNode): pointer = result = alloc0(sizeof(pointer)) else: awr(cstring, cstring(v.strVal)) + of tyNil: + result = alloc0(sizeof(pointer)) + of tyDistinct, tyGenericInst: + result = pack(v, typ.sons[0]) else: InternalError("cannot map value to FFI " & typeToString(v.typ)) @@ -168,31 +180,36 @@ proc unpack(x: pointer, typ: PType, info: TLineInfo): PNode = result = newNodeIT(nkNilLit, info, typ) else: aws(nkStrLit, $p) + of tyNil: result = newNodeIT(nkNilLit, info, typ) + of tyDistinct, tyGenericInst: + result = unpack(x, typ.sons[0], info) else: InternalError("cannot map value from FFI " & typeToString(typ)) proc callForeignFunction*(call: PNode): PNode = InternalAssert call.sons[0].kind == nkIntLit - let typ = call.sons[0].typ var cif: TCif var sig: TParamList - for i in 1..typ.len-1: sig[i-1] = mapType(typ.sons[i]) + # use the arguments' types for varargs support: + for i in 1..call.len-1: sig[i-1] = mapType(call.sons[i].typ) - if prep_cif(cif, mapCallConv(typ.callConv), cuint(typ.len-1), + let typ = call.sons[0].typ + if prep_cif(cif, mapCallConv(typ.callConv), cuint(call.len-1), mapType(typ.sons[0]), sig) != OK: InternalError(call.info, "error in FFI call") var args: TArgList let fn = cast[pointer](call.sons[0].intVal) - for i in 0 .. call.len-1: - args[i] = pack(call.sons[i+1]) - let retVal = alloc(typ.sons[0].getSize.int) + for i in 1 .. call.len-1: + args[i-1] = pack(call.sons[i], call.sons[i].typ) + let retVal = if isEmptyType(typ.sons[0]): pointer(nil) + else: alloc(typ.sons[0].getSize.int) libffi.call(cif, fn, retVal, args) - if isEmptyType(typ.sons[0]): result = emptyNode + if retVal.isNil: result = emptyNode else: result = unpack(retVal, typ.sons[0], call.info) - dealloc retVal - for i in countdown(call.len-1, 0): dealloc args[i] + if retVal != nil: dealloc retVal + for i in countdown(call.len-2, 0): dealloc args[i] diff --git a/compiler/evals.nim b/compiler/evals.nim index 535d9aebd..ab6218be0 100755 --- a/compiler/evals.nim +++ b/compiler/evals.nim @@ -339,16 +339,15 @@ proc evalGlobalVar(c: PEvalContext, s: PSym, flags: TEvalFlags): PNode = if not aliasNeeded(result, flags): result = copyTree(result) else: + when hasFFI: + if sfImportc in s.flags: + result = importcSymbol(s) + IdNodeTablePut(c.globals, s, result) + return result + result = s.ast if result == nil or result.kind == nkEmpty: - when hasFFI: - # for 'stdin' etc. we need to support 'importc' for variables: - if sfImportc in s.flags: - result = importcSymbol(s) - else: - result = getNullValue(s.typ, s.info) - else: - result = getNullValue(s.typ, s.info) + result = getNullValue(s.typ, s.info) else: result = evalAux(c, result, {}) if isSpecial(result): return @@ -392,7 +391,7 @@ proc evalCall(c: PEvalContext, n: PNode): PNode = var newCall = newNodeI(nkCall, n.info, n.len) newCall.sons[0] = evalGlobalVar(c, prc.sym, {}) for i in 1 .. Date: Wed, 19 Dec 2012 21:45:18 +0100 Subject: [PATCH 047/106] bugfix: openssl import for macosx --- compiler/condsyms.nim | 2 -- compiler/evalffi.nim | 3 +++ compiler/main.nim | 3 +++ lib/wrappers/openssl.nim | 10 +++++----- 4 files changed, 11 insertions(+), 7 deletions(-) diff --git a/compiler/condsyms.nim b/compiler/condsyms.nim index 4cac7c847..17366f6e9 100755 --- a/compiler/condsyms.nim +++ b/compiler/condsyms.nim @@ -61,8 +61,6 @@ proc InitDefines*() = DefineSymbol("nimmixin") DefineSymbol("nimeffects") DefineSymbol("nimbabel") - when defined(useFFI): - DefineSymbol("nimffi") # add platform specific symbols: case targetCPU diff --git a/compiler/evalffi.nim b/compiler/evalffi.nim index 35659ed26..464c932f9 100644 --- a/compiler/evalffi.nim +++ b/compiler/evalffi.nim @@ -33,6 +33,9 @@ proc getDll(cache: var TDllCache; dll: string): pointer = InternalError("cannot load: " & dll) cache[dll] = result +const + nkPtrLit = nkIntLit # hopefully we can get rid of this hack soon + proc importcSymbol*(sym: PSym): PNode = let name = ropeToStr(sym.loc.r) diff --git a/compiler/main.nim b/compiler/main.nim index edd4fda68..a65660380 100755 --- a/compiler/main.nim +++ b/compiler/main.nim @@ -157,6 +157,9 @@ proc CommandInteractive = #setTarget(osNimrodVM, cpuNimrodVM) initDefines() DefineSymbol("nimrodvm") + when hasFFI: + DefineSymbol("nimffi") + registerPass(verbosePass()) registerPass(sem.semPass()) registerPass(evals.evalPass()) # load system module: diff --git a/lib/wrappers/openssl.nim b/lib/wrappers/openssl.nim index 752e594d1..f692db13a 100755 --- a/lib/wrappers/openssl.nim +++ b/lib/wrappers/openssl.nim @@ -48,14 +48,14 @@ when defined(WINDOWS): else: const versions = "(|.1.0.0|.0.9.9|.0.9.8|.0.9.7|.0.9.6|.0.9.5|.0.9.4)" - when defined(posix): - const - DLLSSLName = "libssl.so" & versions - DLLUtilName = "libcrypto.so" & versions - else: + when defined(macosx): const DLLSSLName = "libssl.dylib" & versions DLLUtilName = "libcrypto.dylib" & versions + else: + const + DLLSSLName = "libssl.so" & versions + DLLUtilName = "libcrypto.so" & versions type SslStruct {.final, pure.} = object From 6cb8edfce94c9236dfd8a30380d7474a8ede6f87 Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Sat, 22 Dec 2012 23:03:28 +0000 Subject: [PATCH 048/106] recvLine now works with unbuffered ssl sockets. Added higher level recv functions. --- lib/pure/sockets.nim | 77 +++++++++++++++++++++++++------------------- 1 file changed, 44 insertions(+), 33 deletions(-) diff --git a/lib/pure/sockets.nim b/lib/pure/sockets.nim index 5f7ba6ac2..5ec831bcc 100755 --- a/lib/pure/sockets.nim +++ b/lib/pure/sockets.nim @@ -62,6 +62,8 @@ type sslHandle: PSSL sslContext: PSSLContext sslNoHandshake: bool # True if needs handshake. + sslHasPeekChar: bool + sslPeekChar: char of false: nil TSocket* = ref TSocketImpl @@ -291,6 +293,7 @@ when defined(ssl): socket.sslContext = ctx socket.sslHandle = SSLNew(PSSLCTX(socket.sslContext)) socket.sslNoHandshake = false + socket.sslHasPeekChar = false if socket.sslHandle == nil: SSLError() @@ -849,11 +852,8 @@ proc checkBuffer(readfds: var seq[TSocket]): int = var res: seq[TSocket] = @[] result = 0 for s in readfds: - if s.isBuffered: - if s.bufLen <= 0 or s.currPos == s.bufLen: - res.add(s) - else: - inc(result) + if hasDataBuffered(s): + inc(result) else: res.add(s) readfds = res @@ -975,42 +975,46 @@ template retRead(flags, readBytes: int) = proc recv*(socket: TSocket, data: pointer, size: int): int {.tags: [FReadIO].} = ## receives data from a socket + if size == 0: return if socket.isBuffered: if socket.bufLen == 0: retRead(0'i32, 0) - when true: - var read = 0 - while read < size: - if socket.currPos >= socket.bufLen: - retRead(0'i32, read) - - let chunk = min(socket.bufLen-socket.currPos, size-read) - var d = cast[cstring](data) - copyMem(addr(d[read]), addr(socket.buffer[socket.currPos]), chunk) - read.inc(chunk) - socket.currPos.inc(chunk) - else: - var read = 0 - while read < size: - if socket.currPos >= socket.bufLen: - retRead(0'i32, read) - - var d = cast[cstring](data) - d[read] = socket.buffer[socket.currPos] - read.inc(1) - socket.currPos.inc(1) + var read = 0 + while read < size: + if socket.currPos >= socket.bufLen: + retRead(0'i32, read) + let chunk = min(socket.bufLen-socket.currPos, size-read) + var d = cast[cstring](data) + copyMem(addr(d[read]), addr(socket.buffer[socket.currPos]), chunk) + read.inc(chunk) + socket.currPos.inc(chunk) + result = read else: when defined(ssl): if socket.isSSL: - result = SSLRead(socket.sslHandle, data, size) + if socket.sslHasPeekChar: + copyMem(data, addr(socket.sslPeekChar), 1) + socket.sslHasPeekChar = false + if size-1 > 0: + var d = cast[cstring](data) + result = SSLRead(socket.sslHandle, addr(d[1]), size-1) + 1 + else: + result = 1 + else: + result = SSLRead(socket.sslHandle, data, size) else: result = recv(socket.fd, data, size.cint, 0'i32) else: result = recv(socket.fd, data, size.cint, 0'i32) +proc recv*(socket: TSocket, data: var string, size: int): int = + ## higher-level version of the above + data.setLen(size) + result = recv(socket, cstring(data), size) + proc waitFor(socket: TSocket, waited: var float, timeout: int): int {. tags: [FTime].} = ## returns the number of characters available to be read. In unbuffered @@ -1045,6 +1049,11 @@ proc recv*(socket: TSocket, data: pointer, size: int, timeout: int): int {. result = read +proc recv*(socket: TSocket, data: var string, size: int, timeout: int): int = + # higher-level version of the above + data.setLen(size) + result = recv(socket, cstring(data), size, timeout) + proc peekChar(socket: TSocket, c: var char): int {.tags: [FReadIO].} = if socket.isBuffered: result = 1 @@ -1057,8 +1066,12 @@ proc peekChar(socket: TSocket, c: var char): int {.tags: [FReadIO].} = else: when defined(ssl): if socket.isSSL: - raise newException(ESSL, "Sorry, you cannot use recvLine on an unbuffered SSL socket.") - + if not socket.sslHasPeekChar: + result = SSLRead(socket.sslHandle, addr(socket.sslPeekChar), 1) + socket.sslHasPeekChar = true + + c = socket.sslPeekChar + return result = recv(socket.fd, addr(c), 1, MSG_PEEK) proc recvLine*(socket: TSocket, line: var TaintedString): bool {. @@ -1068,13 +1081,11 @@ proc recvLine*(socket: TSocket, line: var TaintedString): bool {. ## will be set to it. ## ## ``True`` is returned if data is available. ``False`` usually suggests an - ## error, EOS exceptions are not raised in favour of this. + ## error, EOS exceptions are not raised and ``False`` is simply returned + ## instead. ## ## If the socket is disconnected, ``line`` will be set to ``""`` and ``True`` ## will be returned. - ## - ## **Warning:** Using this function on a unbuffered ssl socket will result - ## in an error. template addNLIfEmpty(): stmt = if line.len == 0: line.add("\c\L") From 3cbac135465d63a320ff5fcdb5d3ba4ba89da4c8 Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Sun, 23 Dec 2012 11:22:42 +0000 Subject: [PATCH 049/106] Rewrote the implementation of parsing chunked transfer coding in httpclient. Fixes #272. --- lib/pure/httpclient.nim | 92 ++++++++++++++++------------------------- lib/pure/sockets.nim | 22 ++++++++-- 2 files changed, 55 insertions(+), 59 deletions(-) diff --git a/lib/pure/httpclient.nim b/lib/pure/httpclient.nim index 20b448123..5be4af8a4 100755 --- a/lib/pure/httpclient.nim +++ b/lib/pure/httpclient.nim @@ -80,62 +80,43 @@ proc fileError(msg: string) = e.msg = msg raise e -proc charAt(d: var string, i: var int, s: TSocket): char {.inline.} = - result = d[i] - while result == '\0': - d = string(s.recv()) - i = 0 - result = d[i] - proc parseChunks(s: TSocket): string = - # get chunks: - var i = 0 result = "" - var d = s.recv().string + var ri = 0 while true: + var chunkSizeStr = "" var chunkSize = 0 - var digitFound = false - while true: - case d[i] - of '0'..'9': - digitFound = true - chunkSize = chunkSize shl 4 or (ord(d[i]) - ord('0')) - of 'a'..'f': - digitFound = true - chunkSize = chunkSize shl 4 or (ord(d[i]) - ord('a') + 10) - of 'A'..'F': - digitFound = true - chunkSize = chunkSize shl 4 or (ord(d[i]) - ord('A') + 10) - of '\0': - d = string(s.recv()) - i = -1 - else: break - inc(i) - if not digitFound: httpError("Chunksize expected") + if s.recvLine(chunkSizeStr): + var i = 0 + if chunkSizeStr == "": + httpError("Server terminated connection prematurely") + while true: + case chunkSizeStr[i] + of '0'..'9': + chunkSize = chunkSize shl 4 or (ord(chunkSizeStr[i]) - ord('0')) + of 'a'..'f': + chunkSize = chunkSize shl 4 or (ord(chunkSizeStr[i]) - ord('a') + 10) + of 'A'..'F': + chunkSize = chunkSize shl 4 or (ord(chunkSizeStr[i]) - ord('A') + 10) + of '\0': + break + of ';': + # http://tools.ietf.org/html/rfc2616#section-3.6.1 + # We don't care about chunk-extensions. + break + else: + httpError("Invalid chunk size: " & chunkSizeStr) + inc(i) if chunkSize <= 0: break - while charAt(d, i, s) notin {'\C', '\L', '\0'}: inc(i) - if charAt(d, i, s) == '\C': inc(i) - if charAt(d, i, s) == '\L': inc(i) - else: httpError("CR-LF after chunksize expected") - - var x = substr(d, i, i+chunkSize-1) - var size = x.len - result.add(x) - inc(i, size) - if size < chunkSize: - # read in the rest: - var missing = chunkSize - size - var L = result.len - setLen(result, L + missing) - while missing > 0: - var bytesRead = s.recv(addr(result[L]), missing) - inc(L, bytesRead) - dec(missing, bytesRead) - # next chunk: - d = string(s.recv()) - i = 0 - # skip trailing CR-LF: - while charAt(d, i, s) in {'\C', '\L'}: inc(i) + result.setLen(ri+chunkSize) + var bytesRead = 0 + while bytesRead != chunkSize: + let ret = recv(s, addr(result[ri]), chunkSize-bytesRead) + ri += ret + bytesRead += ret + s.skip(2) # Skip \c\L + # Trailer headers will only be sent if the request specifies that we want + # them: http://tools.ietf.org/html/rfc2616#section-3.6.1 proc parseBody(s: TSocket, headers: PStringTable): string = @@ -250,7 +231,6 @@ proc request*(url: string, httpMethod = httpGET, extraHeaders = "", ## | Requests ``url`` with the specified ``httpMethod``. ## | Extra headers can be specified and must be seperated by ``\c\L`` var r = parseUrl(url) - var headers = substr($httpMethod, len("http")) headers.add(" /" & r.path & r.query) @@ -285,7 +265,7 @@ proc redirection(status: string): bool = return True proc get*(url: string, maxRedirects = 5, sslContext: PSSLContext = defaultSSLContext): TResponse = - ## | GET's the ``url`` and returns a ``TResponse`` object + ## | GETs the ``url`` and returns a ``TResponse`` object ## | This proc also handles redirection result = request(url) for i in 1..maxRedirects: @@ -295,7 +275,7 @@ proc get*(url: string, maxRedirects = 5, sslContext: PSSLContext = defaultSSLCon result = request(locationHeader, sslContext = sslContext) proc getContent*(url: string, sslContext: PSSLContext = defaultSSLContext): string = - ## | GET's the body and returns it as a string. + ## | GETs the body and returns it as a string. ## | Raises exceptions for the status codes ``4xx`` and ``5xx`` var r = get(url, sslContext = sslContext) if r.status[0] in {'4','5'}: @@ -305,7 +285,7 @@ proc getContent*(url: string, sslContext: PSSLContext = defaultSSLContext): stri proc post*(url: string, extraHeaders = "", body = "", maxRedirects = 5, sslContext: PSSLContext = defaultSSLContext): TResponse = - ## | POST's ``body`` to the ``url`` and returns a ``TResponse`` object. + ## | POSTs ``body`` to the ``url`` and returns a ``TResponse`` object. ## | This proc adds the necessary Content-Length header. ## | This proc also handles redirection. var xh = extraHeaders & "Content-Length: " & $len(body) & "\c\L" @@ -319,7 +299,7 @@ proc post*(url: string, extraHeaders = "", body = "", proc postContent*(url: string, extraHeaders = "", body = "", sslContext: PSSLContext = defaultSSLContext): string = - ## | POST's ``body`` to ``url`` and returns the response's body as a string + ## | POSTs ``body`` to ``url`` and returns the response's body as a string ## | Raises exceptions for the status codes ``4xx`` and ``5xx`` var r = post(url, extraHeaders, body) if r.status[0] in {'4','5'}: diff --git a/lib/pure/sockets.nim b/lib/pure/sockets.nim index 5ec831bcc..fa7112f5b 100755 --- a/lib/pure/sockets.nim +++ b/lib/pure/sockets.nim @@ -1164,11 +1164,13 @@ proc recvLineAsync*(socket: TSocket, elif c == '\L': return RecvFullLine add(line.string, c) -proc recv*(socket: TSocket): TaintedString {.tags: [FReadIO].} = +proc recv*(socket: TSocket): TaintedString {.tags: [FReadIO], deprecated.} = ## receives all the available data from the socket. ## Socket errors will result in an ``EOS`` error. ## If socket is not a connectionless socket and socket is not connected ## ``""`` will be returned. + ## + ## **Deprecated since version 0.9.2**: This function is not safe for use. const bufSize = 4000 result = newStringOfCap(bufSize).TaintedString var pos = 0 @@ -1194,10 +1196,12 @@ proc recv*(socket: TSocket): TaintedString {.tags: [FReadIO].} = if bytesRead != bufSize-1: break proc recvTimeout*(socket: TSocket, timeout: int): TaintedString {. - tags: [FReadIO].} = + tags: [FReadIO], deprecated.} = ## overloaded variant to support a ``timeout`` parameter, the ``timeout`` ## parameter specifies the amount of miliseconds to wait for data on the ## socket. + ## + ## **Deprecated since version 0.9.2**: This function is not safe for use. if socket.bufLen == 0: var s = @[socket] if s.select(timeout) != 1: @@ -1298,13 +1302,25 @@ proc recvFromAsync*(socket: TSocket, data: var String, length: int, return False else: OSError() -proc skip*(socket: TSocket) {.tags: [FReadIO].} = +proc skip*(socket: TSocket) {.tags: [FReadIO], deprecated.} = ## skips all the data that is pending for the socket + ## + ## **Deprecated since version 0.9.2**: This function is not safe for use. const bufSize = 1000 var buf = alloc(bufSize) while recv(socket, buf, bufSize) == bufSize: nil dealloc(buf) +proc skip*(socket: TSocket, size: int) = + ## Skips ``size`` amount of bytes. + ## + ## Returns the number of skipped bytes. + var dummy = alloc(size) + var bytesSkipped = 0 + while bytesSkipped != size: + bytesSkipped += recv(socket, dummy, size-bytesSkipped) + dealloc(dummy) + proc send*(socket: TSocket, data: pointer, size: int): int {. tags: [FWriteIO].} = ## sends data to a socket. From da609fc4454a8e57cbdd9d0415bb7fed1f7d1ce1 Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Sun, 23 Dec 2012 14:05:16 +0000 Subject: [PATCH 050/106] Fixed many deprecation warnings. asyncio.recvLine now throws an exception when an error occurs. Added sockets.SocketError. --- lib/pure/asyncio.nim | 8 +++- lib/pure/ftpclient.nim | 78 ++++++++++++-------------------- lib/pure/redis.nim | 48 ++++++++++++-------- lib/pure/sockets.nim | 93 ++++++++++++++++++++++++++++++++++++--- tests/compile/tircbot.nim | 15 ++++--- 5 files changed, 160 insertions(+), 82 deletions(-) diff --git a/lib/pure/asyncio.nim b/lib/pure/asyncio.nim index 133d8c24e..be375a1a1 100644 --- a/lib/pure/asyncio.nim +++ b/lib/pure/asyncio.nim @@ -440,6 +440,9 @@ proc recvLine*(s: PAsyncSocket, line: var TaintedString): bool = ## sockets properly. This function guarantees that ``line`` is a full line, ## if this function can only retrieve some data; it will save this data and ## add it to the result when a full line is retrieved. + ## + ## Unlike ``sockets.recvLine`` this function will raise an EOS or ESSL + ## exception if an error occurs. setLen(line.string, 0) var dataReceived = "".TaintedString var ret = s.socket.recvLineAsync(dataReceived) @@ -458,6 +461,7 @@ proc recvLine*(s: PAsyncSocket, line: var TaintedString): bool = of RecvDisconnected: result = true of RecvFail: + s.SocketError(async = true) result = false proc send*(sock: PAsyncSocket, data: string) = @@ -594,7 +598,9 @@ when isMainModule: proc testRead(s: PAsyncSocket, no: int) = echo("Reading! " & $no) - var data = s.getSocket.recv() + var data = "" + if not s.recvLine(data): + OSError() if data == "": echo("Closing connection. " & $no) s.close() diff --git a/lib/pure/ftpclient.nim b/lib/pure/ftpclient.nim index ad3d7a3bb..e656d001e 100644 --- a/lib/pure/ftpclient.nim +++ b/lib/pure/ftpclient.nim @@ -96,8 +96,9 @@ type EFTP* = object of ESynch proc FTPClient*(address: string, port = TPort(21), - user, pass = ""): TFTPClient = - ## Create a ``TFTPClient`` object. + user, pass = ""): PFTPClient = + ## Create a ``PFTPClient`` object. + new(result) result.user = user result.pass = pass result.address = address @@ -278,11 +279,20 @@ proc getLines(ftp: PFTPClient, async: bool = false): bool = ## It doesn't if `async` is true, because it doesn't check for 226 then. if ftp.dsockConnected: var r = TaintedString"" - if getDSock(ftp).recvAsync(r): - if r.string != "": - ftp.job.lines.add(r.string) - else: - ftp.dsockConnected = False + if ftp.isAsync: + if ftp.asyncDSock.recvLine(r): + if r.string == "": + ftp.dsockConnected = false + else: + ftp.job.lines.add(r.string & "\n") + else: + assert(not async) + if ftp.dsock.recvLine(r): + if r.string == "": + ftp.dsockConnected = false + else: + ftp.job.lines.add(r.string & "\n") + else: OSError() if not async: var readSocks: seq[TSocket] = @[ftp.getCSock()] @@ -389,7 +399,7 @@ proc list*(ftp: PFTPClient, dir: string = "", async = false): string = proc retrText*(ftp: PFTPClient, file: string, async = false): string = ## Retrieves ``file``. File must be ASCII text. ## If ``async`` is true, this function will return immediately and - ## it will be your job to call ``poll`` to progress this operation. + ## it will be your job to call asyncio's ``poll`` to progress this operation. ftp.createJob(getLines, JRetrText) ftp.pasv() assertReply ftp.send("RETR " & file.normalizePathSep), ["125", "150"] @@ -404,12 +414,14 @@ proc retrText*(ftp: PFTPClient, file: string, async = false): string = proc getFile(ftp: PFTPClient, async = false): bool = if ftp.dsockConnected: var r = "".TaintedString + var bytesRead = 0 var returned = false if async: if not ftp.isAsync: raise newException(EFTP, "FTPClient must be async.") - returned = ftp.AsyncDSock.recvAsync(r) + bytesRead = ftp.AsyncDSock.recvAsync(r, BufferSize) + returned = bytesRead != -1 else: - r = getDSock(ftp).recv() + bytesRead = getDSock(ftp).recv(r, BufferSize) returned = true let r2 = r.string if r2 != "": @@ -429,8 +441,9 @@ proc getFile(ftp: PFTPClient, async = false): bool = proc retrFile*(ftp: PFTPClient, file, dest: string, async = false) = ## Downloads ``file`` and saves it to ``dest``. Usage of this function ## asynchronously is recommended to view the progress of the download. - ## The ``EvRetr`` event is given by ``poll`` when the download is finished, - ## and the ``filename`` field will be equal to ``file``. + ## The ``EvRetr`` event is passed to the specified ``handleEvent`` function + ## when the download is finished, and the ``filename`` field will be equal + ## to ``file``. ftp.createJob(getFile, JRetr) ftp.job.file = open(dest, mode = fmWrite) ftp.pasv() @@ -492,8 +505,9 @@ proc store*(ftp: PFTPClient, file, dest: string, async = false) = ## Uploads ``file`` to ``dest`` on the remote FTP server. Usage of this ## function asynchronously is recommended to view the progress of ## the download. - ## The ``EvStore`` event is given by ``poll`` when the upload is finished, - ## and the ``filename`` field will be equal to ``file``. + ## The ``EvStore`` event is passed to the specified ``handleEvent`` function + ## when the upload is finished, and the ``filename`` field will be + ## equal to ``file``. ftp.createJob(doUpload, JStore) ftp.job.file = open(file) ftp.job.total = ftp.job.file.getFileSize() @@ -518,16 +532,6 @@ proc close*(ftp: PFTPClient) = ftp.csock.close() ftp.dsock.close() -discard """proc getSocket(h: PObject): tuple[info: TInfo, sock: TSocket] = - result = (SockIdle, InvalidSocket) - var ftp = PAsyncFTPClient(h) - if ftp.jobInProgress: - case ftp.job.typ - of JRetrText, JRetr, JStore: - if ftp.dsockStatus == SockConnecting or ftp.dsockStatus == SockConnected: - result = (ftp.dsockStatus, ftp.dsock) - else: result = (SockIdle, ftp.dsock)""" - proc csockHandleRead(s: PAsyncSocket, ftp: PAsyncFTPClient) = if ftp.jobInProgress: assertReply ftp.expectReply(), "226" # Make sure the transfer completed. @@ -550,32 +554,6 @@ proc csockHandleRead(s: PAsyncSocket, ftp: PAsyncFTPClient) = ftp.handleEvent(ftp, r) -discard """proc handleConnect(h: PObject) = - var ftp = PAsyncFTPClient(h) - ftp.dsockStatus = SockConnected - assert(ftp.jobInProgress) - if ftp.job.typ == JStore: - ftp.dele.mode = MWriteable - else: - ftp.dele.mode = MReadable""" - -discard """proc handleRead(h: PObject) = - var ftp = PAsyncFTPClient(h) - assert(ftp.jobInProgress) - assert(ftp.job.typ != JStore) - # This can never return true, because it shouldn't check for code - # 226 from csock. - assert(not ftp.job.prc(ftp[], true)) -""" - -discard """proc csockGetSocket(h: PObject): tuple[info: TInfo, sock: TSocket] = - # This only returns the csock if a job is in progress. Otherwise handle read - # would capture data which is not for it to capture. - result = (SockIdle, InvalidSocket) - var ftp = PAsyncFTPClient(h) - if ftp.jobInProgress: - result = (SockConnected, ftp.csock)""" - proc AsyncFTPClient*(address: string, port = TPort(21), user, pass = "", handleEvent: proc (ftp: PAsyncFTPClient, ev: TFTPEvent) {.closure.} = diff --git a/lib/pure/redis.nim b/lib/pure/redis.nim index 921f9a3d8..bb6ea6768 100755 --- a/lib/pure/redis.nim +++ b/lib/pure/redis.nim @@ -49,26 +49,35 @@ proc raiseNoOK(status: string) = raise newException(EInvalidReply, "Expected \"OK\" got \"$1\"" % status) proc parseStatus(r: TRedis): TRedisStatus = - var line = r.socket.recv.string + var line = "" + if r.socket.recvLine(line): + if line == "": + raise newException(ERedis, "Server closed connection prematurely") - if line[0] == '-': - raise newException(ERedis, strip(line)) - if line[0] != '+': - raiseInvalidReply('+', line[0]) - - return line.substr(1, line.len-3) # Strip '+' and \c\L. + if line[0] == '-': + raise newException(ERedis, strip(line)) + if line[0] != '+': + raiseInvalidReply('+', line[0]) + + return line.substr(1) # Strip '+' + else: + OSError() proc parseInteger(r: TRedis): TRedisInteger = - var line = r.socket.recv.string + var line = "" + if r.socket.recvLine(line): + if line == "": + raise newException(ERedis, "Server closed connection prematurely") - if line[0] == '-': - raise newException(ERedis, strip(line)) - if line[0] != ':': - raiseInvalidReply(':', line[0]) - - # Strip ':' and \c\L. - if parseBiggestInt(line, result, 1) == 0: - raise newException(EInvalidReply, "Unable to parse integer.") + if line[0] == '-': + raise newException(ERedis, strip(line)) + if line[0] != ':': + raiseInvalidReply(':', line[0]) + + # Strip ':' + if parseBiggestInt(line, result, 1) == 0: + raise newException(EInvalidReply, "Unable to parse integer.") + else: OSError() proc recv(sock: TSocket, size: int): TaintedString = result = newString(size).TaintedString @@ -838,8 +847,11 @@ proc save*(r: TRedis) = proc shutdown*(r: TRedis) = ## Synchronously save the dataset to disk and then shut down the server r.sendCommand("SHUTDOWN") - var s = r.socket.recv() - if s.string.len != 0: raise newException(ERedis, s.string) + var s = "".TaintedString + if r.socket.recvLine(s): + if s.string.len != 0: raise newException(ERedis, s.string) + else: + OSError() proc slaveof*(r: TRedis, host: string, port: string) = ## Make the server a slave of another instance, or promote it as master diff --git a/lib/pure/sockets.nim b/lib/pure/sockets.nim index fa7112f5b..053c1711a 100755 --- a/lib/pure/sockets.nim +++ b/lib/pure/sockets.nim @@ -47,12 +47,15 @@ when defined(ssl): TSSLAcceptResult* = enum AcceptNoClient = 0, AcceptNoHandshake, AcceptSuccess +const + BufferSize*: int = 4000 ## size of a buffered socket's buffer + type TSocketImpl = object ## socket type fd: cint case isBuffered: bool # determines whether this socket is buffered. of true: - buffer: array[0..4000, char] + buffer: array[0..BufferSize, char] currPos: int # current index in buffer bufLen: int # current length of buffer of false: nil @@ -300,6 +303,48 @@ when defined(ssl): if SSLSetFd(socket.sslHandle, socket.fd) != 1: SSLError() +proc SocketError*(socket: TSocket, err: int = -1, async = false) = + ## Raises proper errors based on return values of ``recv`` functions. + ## + ## If ``async`` is ``True`` no error will be thrown in the case when the + ## error was caused by no data being available to be read. + ## + ## If ``err`` is not lower than 0 no exception will be raised. + when defined(ssl): + if socket.isSSL: + if err <= 0: + var ret = SSLGetError(socket.sslHandle, err.cint) + case ret + of SSL_ERROR_ZERO_RETURN: + SSLError("TLS/SSL connection failed to initiate, socket closed prematurely.") + of SSL_ERROR_WANT_CONNECT, SSL_ERROR_WANT_ACCEPT: + if async: + return + else: SSLError("Not enough data on socket.") + of SSL_ERROR_WANT_WRITE, SSL_ERROR_WANT_READ: + if async: + return + else: SSLError("Not enough data on socket.") + of SSL_ERROR_WANT_X509_LOOKUP: + SSLError("Function for x509 lookup has been called.") + of SSL_ERROR_SYSCALL, SSL_ERROR_SSL: + SSLError() + else: SSLError("Unknown Error") + + if err == -1 and not (when defined(ssl): socket.isSSL else: false): + if async: + when defined(windows): + # TODO: Test on Windows + var err = WSAGetLastError() + if err == WSAEWOULDBLOCK: + return + else: OSError() + else: + if errno == EAGAIN or errno == EWOULDBLOCK: + return + else: OSError() + else: OSError() + proc listen*(socket: TSocket, backlog = SOMAXCONN) {.tags: [FReadIO].} = ## Marks ``socket`` as accepting connections. ## ``Backlog`` specifies the maximum length of the @@ -1012,14 +1057,39 @@ proc recv*(socket: TSocket, data: pointer, size: int): int {.tags: [FReadIO].} = proc recv*(socket: TSocket, data: var string, size: int): int = ## higher-level version of the above + ## + ## When 0 is returned the socket's connection has been closed. + ## + ## This function will throw an EOS exception when an error occurs. A value + ## lower than 0 is never returned. + ## + ## **Note**: ``data`` must be initialised. data.setLen(size) result = recv(socket, cstring(data), size) + if result < 0: + data.setLen(0) + socket.SocketError(result) + data.setLen(result) + +proc recvAsync*(socket: TSocket, data: var string, size: int): int = + ## Async version of the above. + ## + ## When socket is non-blocking and no data is available on the socket, + ## ``-1`` will be returned and ``data`` will be ``""``. + ## + ## **Note**: ``data`` must be initialised. + data.setLen(size) + result = recv(socket, cstring(data), size) + if result < 0: + data.setLen(0) + socket.SocketError(async = true) + result = -1 + data.setLen(result) proc waitFor(socket: TSocket, waited: var float, timeout: int): int {. tags: [FTime].} = ## returns the number of characters available to be read. In unbuffered - ## sockets this is always 1, otherwise this may as big as the buffer, currently - ## 4000. + ## sockets this is always 1, otherwise this may as big as ``BufferSize``. result = 1 if socket.isBuffered and socket.bufLen != 0 and socket.bufLen != socket.currPos: result = socket.bufLen - socket.currPos @@ -1050,9 +1120,16 @@ proc recv*(socket: TSocket, data: pointer, size: int, timeout: int): int {. result = read proc recv*(socket: TSocket, data: var string, size: int, timeout: int): int = - # higher-level version of the above + ## higher-level version of the above. + ## + ## Similar to the non-timeout version this will throw an EOS exception + ## when an error occurs. data.setLen(size) result = recv(socket, cstring(data), size, timeout) + if result < 0: + data.setLen(0) + socket.SocketError() + data.setLen(result) proc peekChar(socket: TSocket, c: var char): int {.tags: [FReadIO].} = if socket.isBuffered: @@ -1080,7 +1157,7 @@ proc recvLine*(socket: TSocket, line: var TaintedString): bool {. ## added to ``line``, however if solely ``\r\L`` is received then ``line`` ## will be set to it. ## - ## ``True`` is returned if data is available. ``False`` usually suggests an + ## ``True`` is returned if data is available. ``False`` suggests an ## error, EOS exceptions are not raised and ``False`` is simply returned ## instead. ## @@ -1112,6 +1189,8 @@ proc recvLine*(socket: TSocket, line: var TaintedString, timeout: int): bool {. tags: [FReadIO, FTime].} = ## variant with a ``timeout`` parameter, the timeout parameter specifies ## how many miliseconds to wait for data. + ## + ## ``ETimeout`` will be raised if ``timeout`` is exceeded. template addNLIfEmpty(): stmt = if line.len == 0: line.add("\c\L") @@ -1210,12 +1289,14 @@ proc recvTimeout*(socket: TSocket, timeout: int): TaintedString {. return socket.recv proc recvAsync*(socket: TSocket, s: var TaintedString): bool {. - tags: [FReadIO].} = + tags: [FReadIO], deprecated.} = ## receives all the data from a non-blocking socket. If socket is non-blocking ## and there are no messages available, `False` will be returned. ## Other socket errors will result in an ``EOS`` error. ## If socket is not a connectionless socket and socket is not connected ## ``s`` will be set to ``""``. + ## + ## **Deprecated since version 0.9.2**: This function is not safe for use. const bufSize = 1000 # ensure bufSize capacity: setLen(s.string, bufSize) diff --git a/tests/compile/tircbot.nim b/tests/compile/tircbot.nim index 10482c3f6..d16c99b69 100644 --- a/tests/compile/tircbot.nim +++ b/tests/compile/tircbot.nim @@ -272,7 +272,7 @@ proc handleWebMessage(state: PState, line: string) = message.add(limitCommitMsg(commit["message"].str)) # Send message to #nimrod. - state.ircClient[].privmsg(joinChans[0], message) + state.ircClient.privmsg(joinChans[0], message) elif json.existsKey("redisinfo"): assert json["redisinfo"].existsKey("port") #let redisPort = json["redisinfo"]["port"].num @@ -336,10 +336,11 @@ proc hubConnect(state: PState) = state.dispatcher.register(state.sock) -proc handleIrc(irc: var TAsyncIRC, event: TIRCEvent, state: PState) = +proc handleIrc(irc: PAsyncIRC, event: TIRCEvent, state: PState) = case event.typ + of EvConnected: nil of EvDisconnected: - while not state.ircClient[].isConnected: + while not state.ircClient.isConnected: try: state.ircClient.connect() except: @@ -355,12 +356,12 @@ proc handleIrc(irc: var TAsyncIRC, event: TIRCEvent, state: PState) = let msg = event.params[event.params.len-1] let words = msg.split(' ') template pm(msg: string): stmt = - state.ircClient[].privmsg(event.origin, msg) + state.ircClient.privmsg(event.origin, msg) case words[0] of "!ping": pm("pong") of "!lag": - if state.ircClient[].getLag != -1.0: - var lag = state.ircClient[].getLag + if state.ircClient.getLag != -1.0: + var lag = state.ircClient.getLag lag = lag * 1000.0 pm($int(lag) & "ms between me and the server.") else: @@ -433,7 +434,7 @@ proc open(port: TPort = TPort(5123)): PState = res.hubPort = port res.hubConnect() let hirc = - proc (a: var TAsyncIRC, ev: TIRCEvent) = + proc (a: PAsyncIRC, ev: TIRCEvent) = handleIrc(a, ev, res) # Connect to the irc server. res.ircClient = AsyncIrc(ircServer, nick = botNickname, user = botNickname, From f83881dd4e42edc3f6c2b3dfdbe661defa096c60 Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Sun, 23 Dec 2012 14:31:16 +0000 Subject: [PATCH 051/106] Fixed symbol conflict in the sockets module on Windows. --- lib/pure/sockets.nim | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/pure/sockets.nim b/lib/pure/sockets.nim index 053c1711a..86f06324a 100755 --- a/lib/pure/sockets.nim +++ b/lib/pure/sockets.nim @@ -1504,8 +1504,7 @@ proc sendTo*(socket: TSocket, address: string, port: TPort, result = socket.sendTo(address, port, cstring(data), data.len) when defined(Windows): - const - SOCKET_ERROR = -1 + const IOCPARM_MASK = 127 IOC_IN = int(-2147483648) FIONBIO = int(IOC_IN or ((sizeof(int) and IOCPARM_MASK) shl 16) or @@ -1518,7 +1517,7 @@ when defined(Windows): proc setBlocking(s: TSocket, blocking: bool) = when defined(Windows): var mode = clong(ord(not blocking)) # 1 for non-blocking, 0 for blocking - if SOCKET_ERROR == ioctlsocket(TWinSocket(s.fd), FIONBIO, addr(mode)): + if ioctlsocket(TWinSocket(s.fd), FIONBIO, addr(mode)) == -1: OSError() else: # BSD sockets var x: int = fcntl(s.fd, F_GETFL, 0) From d7d4caf45a835975b5f9c328ed8ca408d55cd4fc Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Tue, 25 Dec 2012 14:06:18 +0000 Subject: [PATCH 052/106] Added set_markup to gtk2 wrapper. --- lib/wrappers/gtk/gtk2.nim | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/wrappers/gtk/gtk2.nim b/lib/wrappers/gtk/gtk2.nim index a1bfa7fe1..7094bfd5d 100755 --- a/lib/wrappers/gtk/gtk2.nim +++ b/lib/wrappers/gtk/gtk2.nim @@ -16596,6 +16596,8 @@ proc message_dialog_new*(parent: PWindow, flags: TDialogFlags, thetype: TMessageType, buttons: TButtonsType, message_format: cstring): PMessageDialog{.varargs, cdecl, importc: "gtk_message_dialog_new", dynlib: lib.} +proc set_markup*(msgDialog: PMessageDialog, str: cstring) {.cdecl, + importc: "gtk_message_dialog_set_markup", dynlib: lib.} proc signal_new*(name: cstring, signal_flags: TSignalRunType, object_type: TType, function_offset: guint, marshaller: TSignalMarshaller, return_val: TType, n_args: guint): guint{. From 07c8a6206f6a257f170a677efcb44e5d6234711a Mon Sep 17 00:00:00 2001 From: Grzegorz Adam Hankiewicz Date: Mon, 24 Dec 2012 00:25:37 +0100 Subject: [PATCH 053/106] Documents repr proc and nested array syntax. --- doc/tut1.txt | 79 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) diff --git a/doc/tut1.txt b/doc/tut1.txt index d8d04bc93..2ae7e9540 100755 --- a/doc/tut1.txt +++ b/doc/tut1.txt @@ -919,6 +919,37 @@ types automatically and vice versa. The ``toInt`` and ``toFloat`` procs can be used for these conversions. +Internal type representation +============================ + +As mentioned earlier, the built-in ``$`` (stringify) operator turns any basic +type into a string, which you can then print to the screen with the ``echo`` +proc. However, advanced types, or types you may define yourself won't work with +the ``$`` operator until you define one for them. Sometimes you just want to +debug the current value of a complex type without having to write its ``$`` +operator. You can use then the ``repr`` proc which works with any type and +even complex data graphs with cycles. The following example shows that even for +basic types there is a difference between the ``$`` and ``repr`` outputs: + +.. code-block:: nimrod + var + myBool = true + myCharacter = 'n' + myString = "nimrod" + myInteger = 42 + myFloat = 3.14 + echo($myBool, ":", repr(myBool)) + # --> true:true + echo($myCharacter, ":", repr(myCharacter)) + # --> n:'n' + echo($myString, ":", repr(myString)) + # --> nimrod:0x10fa8c050"nimrod" + echo($myInteger, ":", repr(myInteger)) + # --> 42:42 + echo($myFloat, ":", repr(myFloat)) + # --> 3.1400000000000001e+00:3.1400000000000001e+00 + + Advanced types ============== @@ -1092,6 +1123,54 @@ copies the whole array contents. The built-in ``len`` proc returns the array's length. ``low(a)`` returns the lowest valid index for the array `a` and ``high(a)`` the highest valid index. +.. code-block:: nimrod + type + TDirection = enum + north, east, south, west + TBlinkLights = enum + off, on, slowBlink, mediumBlink, fastBlink + TLevelSetting = array[north..west, TBlinkLights] + var + level : TLevelSetting + level[north] = on + level[south] = slowBlink + level[east] = fastBlink + echo repr(level) # --> [on, fastBlink, slowBlink, off] + echo low(level) # --> north + echo len(level) # --> 4 + echo high(level) # --> west + +The syntax for nested arrays (multidimensional) in other languages is a matter +of appending more brackets because usually each dimension is restricted to the +same index type as the others. In nimrod you can have different dimensions with +different index types, so the nesting syntax is slightly different. Building on +the previous example where a level is defined as an array of enums indexed by +yet another enum, we can add the following lines to add a light tower type +subdivided in height levels accessed through their integer index: + +.. code-block:: nimrod + type + TLightTower = array[1..10, TLevelSetting] + var + tower: TLightTower + tower[1][north] = slowBlink + tower[1][east] = mediumBlink + echo len(tower) # --> 10 + echo len(tower[1]) # --> 4 + echo repr(tower) # --> [[slowBlink, mediumBlink, ...more output.. + # The following lines don't compile due to type mistmatch errors + #tower[north][east] = on + #tower[0][1] = on + +Note how the built-in ``len`` proc returns only the array's first dimension +length. Another way of defining the ``TLightTower`` to show better its +nested nature would be to omit the previous definition of the ``TLevelSetting`` +type and instead write it embedded directly as the type of the first dimension: + +.. code-block:: nimrod + type + TLightTower = array[1..10, array[north..west, TBlinkLights]] + Sequences --------- From 81b718641c6a01581925423fa85be9b97d28bcdd Mon Sep 17 00:00:00 2001 From: Araq Date: Wed, 26 Dec 2012 22:48:34 +0100 Subject: [PATCH 054/106] small improvements for FFI --- compiler/evalffi.nim | 68 +++++++++++++++++++++++++++++++------------- lib/pure/dynlib.nim | 9 +++++- lib/system.nim | 13 ++++++--- 3 files changed, 66 insertions(+), 24 deletions(-) diff --git a/compiler/evalffi.nim b/compiler/evalffi.nim index 464c932f9..0c5301b76 100644 --- a/compiler/evalffi.nim +++ b/compiler/evalffi.nim @@ -17,11 +17,12 @@ else: const libcDll = "libc.so(.6|.5|)" type - TDllCache* = tables.TTable[string, TLibHandle] + TDllCache = tables.TTable[string, TLibHandle] var gDllCache = initTable[string, TLibHandle]() + gExeHandle = LoadLib() -proc getDll(cache: var TDllCache; dll: string): pointer = +proc getDll(cache: var TDllCache; dll: string; info: TLineInfo): pointer = result = cache[dll] if result.isNil: var libs: seq[string] = @[] @@ -30,7 +31,7 @@ proc getDll(cache: var TDllCache; dll: string): pointer = result = LoadLib(c) if not result.isNil: break if result.isNil: - InternalError("cannot load: " & dll) + GlobalError(info, errGenerated, "cannot load: " & dll) cache[dll] = result const @@ -41,7 +42,7 @@ proc importcSymbol*(sym: PSym): PNode = # the AST does not support untyped pointers directly, so we use an nkIntLit # that contains the address instead: - result = newNodeIT(nkIntLit, sym.info, sym.typ) + result = newNodeIT(nkPtrLit, sym.info, sym.typ) case name of "stdin": result.intVal = cast[TAddress](system.stdin) of "stdout": result.intVal = cast[TAddress](system.stdout) @@ -49,12 +50,19 @@ proc importcSymbol*(sym: PSym): PNode = else: let lib = sym.annex if lib != nil and lib.path.kind notin {nkStrLit..nkTripleStrLit}: - InternalError(sym.info, "dynlib needs to be a string literal for the REPL") - - let dllpath = if lib.isNil: libcDll else: lib.path.strVal - let dllhandle = gDllCache.getDll(dllpath) - let theAddr = dllhandle.checkedSymAddr(name) - + GlobalError(sym.info, errGenerated, + "dynlib needs to be a string lit for the REPL") + var theAddr: pointer + if lib.isNil and not gExehandle.isNil: + # first try this exe itself: + theAddr = gExehandle.symAddr(name) + # then try libc: + if theAddr.isNil: + let dllhandle = gDllCache.getDll(libcDll) + theAddr = dllhandle.checkedSymAddr(name) + else: + let dllhandle = gDllCache.getDll(lib.path.strVal, sym.info) + theAddr = dllhandle.checkedSymAddr(name) result.intVal = cast[TAddress](theAddr) proc mapType(t: ast.PType): ptr libffi.TType = @@ -90,11 +98,12 @@ proc mapCallConv(cc: TCallingConvention): TABI = template rd(T, p: expr): expr {.immediate.} = (cast[ptr T](p))[] template wr(T, p, v: expr) {.immediate.} = (cast[ptr T](p))[] = v +template `+!`(x, y: expr): expr {.immediate.} = + cast[pointer](cast[TAddress](x) + y) -proc pack(v: PNode, typ: PType): pointer = +proc pack(v: PNode, typ: PType, res: pointer) = template awr(T, v: expr) {.immediate, dirty.} = - result = alloc0(sizeof(T)) - wr(T, result, v) + wr(T, res, v) case typ.kind of tyBool: awr(bool, v.intVal != 0) @@ -121,18 +130,37 @@ proc pack(v: PNode, typ: PType): pointer = of tyFloat32: awr(float32, v.floatVal) of tyFloat64: awr(float64, v.floatVal) - of tyPointer, tyProc, tyPtr, tyRef: + of tyPointer, tyProc: if v.kind == nkNilLit: - result = alloc0(sizeof(pointer)) - else: + # nothing to do since the memory is 0 initialized anyway + nil + elif v.kind == nkPtrLit: awr(pointer, cast[pointer](v.intVal)) + else: + InternalError("cannot map pointer/proc value to FFI") + of tyPtr, tyRef, tyVar: + if v.kind == nkNilLit: + # nothing to do since the memory is 0 initialized anyway + nil + elif v.kind == nkPtrLit: + awr(pointer, cast[pointer](v.intVal)) + else: + # XXX this is pretty hard: we need to allocate a new buffer and store it + # somewhere to be freed; we also need to write back any changes to the + # data! + InternalError("cannot map pointer/proc value to FFI") of tyCString, tyString: if v.kind == nkNilLit: - result = alloc0(sizeof(pointer)) + nil else: awr(cstring, cstring(v.strVal)) + of tyArray, tyArrayConstr: + let baseSize = typ.sons[1].getSize + assert(v.len == lengthOrd(typ.sons[0])) + for i in 0 .. ".} proc LoadLib(path: string): TLibHandle = return dlopen(path, RTLD_NOW) + proc LoadLib(): TLibHandle = return dlopen(nil, RTLD_NOW) proc UnloadLib(lib: TLibHandle) = dlclose(lib) proc symAddr(lib: TLibHandle, name: cstring): pointer = return dlsym(lib, name) @@ -78,6 +83,8 @@ elif defined(windows) or defined(dos): proc LoadLib(path: string): TLibHandle = result = cast[TLibHandle](winLoadLibrary(path)) + proc LoadLib(): TLibHandle = + result = cast[TLibHandle](winLoadLibrary(nil)) proc UnloadLib(lib: TLibHandle) = FreeLibrary(cast[THINSTANCE](lib)) proc symAddr(lib: TLibHandle, name: cstring): pointer = diff --git a/lib/system.nim b/lib/system.nim index e199d7611..cbca90a3b 100755 --- a/lib/system.nim +++ b/lib/system.nim @@ -1716,8 +1716,8 @@ when not defined(EcmaScript) and not defined(NimrodVM): proc initStackBottom() {.inline, compilerproc.} = # WARNING: This is very fragile! An array size of 8 does not work on my - # Linux 64bit system. Very strange, but we are at the will of GCC's - # optimizer... + # Linux 64bit system. -- That's because the stack direction is the other + # way round. when defined(setStackBottom): var locals {.volatile.}: pointer locals = addr(locals) @@ -1754,7 +1754,6 @@ when not defined(EcmaScript) and not defined(NimrodVM): proc endbStep() # ----------------- IO Part ------------------------------------------------ - type CFile {.importc: "FILE", nodecl, final.} = object # empty record for # data hiding @@ -1783,7 +1782,7 @@ when not defined(EcmaScript) and not defined(NimrodVM): ## The standard error stream. ## ## Note: In my opinion, this should not be used -- the concept of a - ## separate error stream is a design flaw of UNIX. A seperate *message + ## separate error stream is a design flaw of UNIX. A separate *message ## stream* is a good idea, but since it is named ``stderr`` there are few ## programs out there that distinguish properly between ``stdout`` and ## ``stderr``. So, that's what you get if you don't name your variables @@ -2121,6 +2120,8 @@ when not defined(EcmaScript) and not defined(NimrodVM): elif defined(ecmaScript) or defined(NimrodVM): # Stubs: + proc nimGCvisit(d: pointer, op: int) {.compilerRtl.} = nil + proc GC_disable() = nil proc GC_enable() = nil proc GC_fullCollect() = nil @@ -2151,6 +2152,10 @@ elif defined(ecmaScript) or defined(NimrodVM): if x == y: return 0 if x < y: return -1 return 1 + + when defined(nimffi): + include "system/sysio" + proc quit*(errormsg: string, errorcode = QuitFailure) {.noReturn.} = ## a shorthand for ``echo(errormsg); quit(errorcode)``. From 3020257559d3aa2455309ee3727599fe68274e32 Mon Sep 17 00:00:00 2001 From: Araq Date: Thu, 27 Dec 2012 00:05:23 +0100 Subject: [PATCH 055/106] implemented --listFullPaths option for aporia --- compiler/commands.nim | 3 +++ compiler/msgs.nim | 14 +++++++++++--- compiler/options.nim | 1 + doc/advopt.txt | 1 + 4 files changed, 16 insertions(+), 3 deletions(-) diff --git a/compiler/commands.nim b/compiler/commands.nim index 4b9a47d37..8c0e9a784 100755 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -458,6 +458,9 @@ proc processSwitch(switch, arg: string, pass: TCmdlinePass, info: TLineInfo) = of "stdout": expectNoArg(switch, arg, pass, info) incl(gGlobalOptions, optStdout) + of "listfullpaths": + expectNoArg(switch, arg, pass, info) + gListFullPaths = true else: if strutils.find(switch, '.') >= 0: options.setConfigVar(switch, arg) else: InvalidCmdLineOption(pass, switch, info) diff --git a/compiler/msgs.nim b/compiler/msgs.nim index 240115817..d88e4a513 100755 --- a/compiler/msgs.nim +++ b/compiler/msgs.nim @@ -531,7 +531,15 @@ proc ToFilename*(fileIdx: int32): string = proc toFullPath*(info: TLineInfo): string = if info.fileIndex < 0: result = "???" else: result = fileInfos[info.fileIndex].fullPath - + +proc ToMsgFilename*(info: TLineInfo): string = + if info.fileIndex < 0: result = "???" + else: + if gListFullPaths: + result = fileInfos[info.fileIndex].fullPath + else: + result = fileInfos[info.fileIndex].projPath + proc ToLinenumber*(info: TLineInfo): int {.inline.} = result = info.line @@ -622,7 +630,7 @@ proc writeContext(lastinfo: TLineInfo) = var info = lastInfo for i in countup(0, len(msgContext) - 1): if msgContext[i] != lastInfo and msgContext[i] != info: - MsgWriteln(posContextFormat % [toFilename(msgContext[i]), + MsgWriteln(posContextFormat % [toMsgFilename(msgContext[i]), coordToStr(msgContext[i].line), coordToStr(msgContext[i].col), getMessageStr(errInstantiationFrom, "")]) @@ -676,7 +684,7 @@ proc liMessage(info: TLineInfo, msg: TMsgKind, arg: string, ignoreMsg = optHints notin gOptions or msg notin gNotes frmt = posHintFormat inc(gHintCounter) - let s = frmt % [toFilename(info), coordToStr(info.line), + let s = frmt % [toMsgFilename(info), coordToStr(info.line), coordToStr(info.col), getMessageStr(msg, arg)] if not ignoreMsg: MsgWriteln(s) diff --git a/compiler/options.nim b/compiler/options.nim index 6df29d85f..28382f9ba 100755 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -97,6 +97,7 @@ var gNumberOfProcessors*: int # number of processors gWholeProject*: bool # for 'doc2': output any dependency + gListFullPaths*: bool const genSubDir* = "nimcache" diff --git a/doc/advopt.txt b/doc/advopt.txt index bd387e041..5d3544a76 100755 --- a/doc/advopt.txt +++ b/doc/advopt.txt @@ -21,6 +21,7 @@ Advanced options: -m, --mainmodule:FILE set the project main module -o, --out:FILE set the output filename --stdout output to stdout + --listFullPaths list full paths in messages -w, --warnings:on|off turn all warnings on|off --warning[X]:on|off turn specific warning X on|off --hints:on|off turn all hints on|off From 7d24a43e6191472171af623b85a5265550814f3a Mon Sep 17 00:00:00 2001 From: Araq Date: Thu, 27 Dec 2012 00:15:04 +0100 Subject: [PATCH 056/106] better errors for FFI --- compiler/evalffi.nim | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/compiler/evalffi.nim b/compiler/evalffi.nim index 0c5301b76..5a2f71042 100644 --- a/compiler/evalffi.nim +++ b/compiler/evalffi.nim @@ -58,7 +58,7 @@ proc importcSymbol*(sym: PSym): PNode = theAddr = gExehandle.symAddr(name) # then try libc: if theAddr.isNil: - let dllhandle = gDllCache.getDll(libcDll) + let dllhandle = gDllCache.getDll(libcDll, sym.info) theAddr = dllhandle.checkedSymAddr(name) else: let dllhandle = gDllCache.getDll(lib.path.strVal, sym.info) @@ -125,7 +125,8 @@ proc pack(v: PNode, typ: PType, res: pointer) = of 4: awr(int32, v.intVal.int32) of 8: awr(int64, v.intVal.int64) else: - InternalError("cannot map value to FFI (tyEnum, tySet)") + GlobalError(v.info, errGenerated, + "cannot map value to FFI (tyEnum, tySet)") of tyFloat: awr(float, v.floatVal) of tyFloat32: awr(float32, v.floatVal) of tyFloat64: awr(float64, v.floatVal) @@ -137,7 +138,7 @@ proc pack(v: PNode, typ: PType, res: pointer) = elif v.kind == nkPtrLit: awr(pointer, cast[pointer](v.intVal)) else: - InternalError("cannot map pointer/proc value to FFI") + GlobalError(v.info, errGenerated, "cannot map pointer/proc value to FFI") of tyPtr, tyRef, tyVar: if v.kind == nkNilLit: # nothing to do since the memory is 0 initialized anyway @@ -148,7 +149,7 @@ proc pack(v: PNode, typ: PType, res: pointer) = # XXX this is pretty hard: we need to allocate a new buffer and store it # somewhere to be freed; we also need to write back any changes to the # data! - InternalError("cannot map pointer/proc value to FFI") + GlobalError(v.info, errGenerated, "cannot map pointer/proc value to FFI") of tyCString, tyString: if v.kind == nkNilLit: nil @@ -162,9 +163,10 @@ proc pack(v: PNode, typ: PType, res: pointer) = of tyNil: nil of tyDistinct, tyGenericInst: - result = pack(v, typ.sons[0]) + pack(v, typ.sons[0], res) else: - InternalError("cannot map value to FFI " & typeToString(v.typ)) + GlobalError(v.info, errGenerated, "cannot map value to FFI " & + typeToString(v.typ)) proc unpack(x: pointer, typ: PType, info: TLineInfo): PNode = template aw(kind, v, field: expr) {.immediate, dirty.} = @@ -195,7 +197,8 @@ proc unpack(x: pointer, typ: PType, info: TLineInfo): PNode = of 4: awi(nkIntLit, rd(int32, x).biggestInt) of 8: awi(nkIntLit, rd(int64, x).biggestInt) else: - InternalError("cannot map value from FFI (tyEnum, tySet)") + GlobalError(info, errGenerated, + "cannot map value from FFI (tyEnum, tySet)") of tyFloat: awf(nkFloatLit, rd(float, x)) of tyFloat32: awf(nkFloatLit, rd(float32, x)) of tyFloat64: awf(nkFloatLit, rd(float64, x)) @@ -215,7 +218,8 @@ proc unpack(x: pointer, typ: PType, info: TLineInfo): PNode = of tyDistinct, tyGenericInst: result = unpack(x, typ.sons[0], info) else: - InternalError("cannot map value from FFI " & typeToString(typ)) + GlobalError(info, errGenerated, "cannot map value from FFI " & + typeToString(typ)) proc callForeignFunction*(call: PNode): PNode = InternalAssert call.sons[0].kind == nkIntLit @@ -228,7 +232,7 @@ proc callForeignFunction*(call: PNode): PNode = let typ = call.sons[0].typ if prep_cif(cif, mapCallConv(typ.callConv), cuint(call.len-1), mapType(typ.sons[0]), sig) != OK: - InternalError(call.info, "error in FFI call") + GlobalError(call.info, errGenerated, "error in FFI call") var args: TArgList let fn = cast[pointer](call.sons[0].intVal) From 9ef367db62e6b0acff8d6c845375830080d0d77d Mon Sep 17 00:00:00 2001 From: Araq Date: Mon, 31 Dec 2012 17:53:37 +0100 Subject: [PATCH 057/106] fixed a newly introduced c2nim bug; many untested improvements to the FFI at compile time --- compiler/c2nim/cpp.nim | 2 + compiler/c2nim/tests/systest.c | 2 + compiler/evalffi.nim | 230 +++++++++++++++++++++++++++------ compiler/evals.nim | 29 ++++- compiler/msgs.nim | 3 + todo.txt | 6 +- 6 files changed, 225 insertions(+), 47 deletions(-) diff --git a/compiler/c2nim/cpp.nim b/compiler/c2nim/cpp.nim index 3b7f58fcc..c210eca3a 100755 --- a/compiler/c2nim/cpp.nim +++ b/compiler/c2nim/cpp.nim @@ -42,6 +42,7 @@ proc parseDefine(p: var TParser): PNode = result = newNodeP(nkTemplateDef, p) getTok(p) addSon(result, skipIdentExport(p)) + addSon(result, ast.emptyNode) eat(p, pxParLe) var params = newNodeP(nkFormalParams, p) # return type; not known yet: @@ -60,6 +61,7 @@ proc parseDefine(p: var TParser): PNode = addSon(result, ast.emptyNode) # no generic parameters addSon(result, params) addSon(result, ast.emptyNode) # no pragmas + addSon(result, ast.emptyNode) var kind = parseDefineBody(p, result) params.sons[0] = newIdentNodeP(kind, p) eatNewLine(p, result) diff --git a/compiler/c2nim/tests/systest.c b/compiler/c2nim/tests/systest.c index 241526e07..2a9dd6c28 100755 --- a/compiler/c2nim/tests/systest.c +++ b/compiler/c2nim/tests/systest.c @@ -17,6 +17,8 @@ int aw_instance_callback_set (AW_CALLBACK c, callback_t callback); unsigned long int wawa; +#define MAX(x, y) ((x) < (y)? (y) : (x)) + #define AW_BUILD 85 // AW 5.0 // Limits #define AW_MAX_AVCHANGE_PER_SECOND 10 diff --git a/compiler/evalffi.nim b/compiler/evalffi.nim index 5a2f71042..8f708b76c 100644 --- a/compiler/evalffi.nim +++ b/compiler/evalffi.nim @@ -31,7 +31,7 @@ proc getDll(cache: var TDllCache; dll: string; info: TLineInfo): pointer = result = LoadLib(c) if not result.isNil: break if result.isNil: - GlobalError(info, errGenerated, "cannot load: " & dll) + GlobalError(info, "cannot load: " & dll) cache[dll] = result const @@ -50,8 +50,7 @@ proc importcSymbol*(sym: PSym): PNode = else: let lib = sym.annex if lib != nil and lib.path.kind notin {nkStrLit..nkTripleStrLit}: - GlobalError(sym.info, errGenerated, - "dynlib needs to be a string lit for the REPL") + GlobalError(sym.info, "dynlib needs to be a string lit for the REPL") var theAddr: pointer if lib.isNil and not gExehandle.isNil: # first try this exe itself: @@ -75,8 +74,7 @@ proc mapType(t: ast.PType): ptr libffi.TType = of 2: result = addr libffi.type_sint16 of 4: result = addr libffi.type_sint32 of 8: result = addr libffi.type_sint64 - else: - InternalError("cannot map type to FFI") + else: result = nil of tyFloat, tyFloat64: result = addr libffi.type_double of tyFloat32: result = addr libffi.type_float of tyVar, tyPointer, tyPtr, tyRef, tyCString, tySequence, tyString, tyExpr, @@ -85,22 +83,80 @@ proc mapType(t: ast.PType): ptr libffi.TType = of tyDistinct: result = mapType(t.sons[0]) else: - InternalError("cannot map type to FFI") + result = nil # too risky: #of tyFloat128: result = addr libffi.type_longdouble -proc mapCallConv(cc: TCallingConvention): TABI = +proc mapCallConv(cc: TCallingConvention, info: TLineInfo): TABI = case cc of ccDefault: result = DEFAULT_ABI of ccStdCall: result = when defined(windows): STDCALL else: DEFAULT_ABI of ccCDecl: result = DEFAULT_ABI - else: InternalError("cannot map calling convention to FFI") + else: + GlobalError(info, "cannot map calling convention to FFI") template rd(T, p: expr): expr {.immediate.} = (cast[ptr T](p))[] template wr(T, p, v: expr) {.immediate.} = (cast[ptr T](p))[] = v template `+!`(x, y: expr): expr {.immediate.} = cast[pointer](cast[TAddress](x) + y) +proc packSize(v: PNode, typ: PType): int = + ## computes the size of the blob + case typ.kind + of tyPtr, tyRef, tyVar: + if v.kind in {nkNilLit, nkPtrLit}: + result = sizeof(pointer) + else: + result = sizeof(pointer) + packSize(v.sons[0], typ.sons[0]) + of tyDistinct, tyGenericInst: + result = packSize(v, typ.sons[0]) + of tyArray, tyArrayConstr: + # consider: ptr array[0..1000_000, int] which is common for interfacing; + # we use the real length here instead + if v.kind in {nkNilLit, nkPtrLit}: + result = sizeof(pointer) + elif v.len != 0: + result = v.len * packSize(v.sons[0], typ.sons[1]) + else: + result = typ.getSize.int + +proc pack(v: PNode, typ: PType, res: pointer) + +proc getField(n: PNode; position: int): PSym = + case n.kind + of nkRecList: + for i in countup(0, sonsLen(n) - 1): + result = getField(n.sons[i], position) + if result != nil: return + of nkRecCase: + result = getField(n.sons[0], position) + if result != nil: return + for i in countup(1, sonsLen(n) - 1): + case n.sons[i].kind + of nkOfBranch, nkElse: + result = getField(lastSon(n.sons[i]), position) + if result != nil: return + else: internalError(n.info, "getField(record case branch)") + of nkSym: + if n.sym.position == position: result = n.sym + else: nil + +proc packObject(x: PNode, typ: PType, res: pointer) = + InternalAssert x.kind == nkPar + # compute the field's offsets: + discard typ.getSize + for i in countup(0, sonsLen(x) - 1): + var it = x.sons[i] + if it.kind == nkExprColonExpr: + internalAssert it.sons[0].kind == nkSym + let field = it.sons[0].sym + pack(it.sons[1], field.typ, res +! field.offset) + elif typ.n != nil: + let field = getField(typ.n, i) + pack(it, field.typ, res +! field.offset) + else: + GlobalError(x.info, "cannot pack unnamed tuple") + proc pack(v: PNode, typ: PType, res: pointer) = template awr(T, v: expr) {.immediate, dirty.} = wr(T, res, v) @@ -125,8 +181,7 @@ proc pack(v: PNode, typ: PType, res: pointer) = of 4: awr(int32, v.intVal.int32) of 8: awr(int64, v.intVal.int64) else: - GlobalError(v.info, errGenerated, - "cannot map value to FFI (tyEnum, tySet)") + GlobalError(v.info, "cannot map value to FFI (tyEnum, tySet)") of tyFloat: awr(float, v.floatVal) of tyFloat32: awr(float32, v.floatVal) of tyFloat64: awr(float64, v.floatVal) @@ -138,7 +193,7 @@ proc pack(v: PNode, typ: PType, res: pointer) = elif v.kind == nkPtrLit: awr(pointer, cast[pointer](v.intVal)) else: - GlobalError(v.info, errGenerated, "cannot map pointer/proc value to FFI") + GlobalError(v.info, "cannot map pointer/proc value to FFI") of tyPtr, tyRef, tyVar: if v.kind == nkNilLit: # nothing to do since the memory is 0 initialized anyway @@ -146,33 +201,106 @@ proc pack(v: PNode, typ: PType, res: pointer) = elif v.kind == nkPtrLit: awr(pointer, cast[pointer](v.intVal)) else: - # XXX this is pretty hard: we need to allocate a new buffer and store it - # somewhere to be freed; we also need to write back any changes to the - # data! - GlobalError(v.info, errGenerated, "cannot map pointer/proc value to FFI") + pack(v.sons[0], typ.sons[0], res +! sizeof(pointer)) + awr(pointer, res +! sizeof(pointer)) of tyCString, tyString: if v.kind == nkNilLit: nil - else: + elif v.kind in {nkStrLit..nkTripleStrLit}: awr(cstring, cstring(v.strVal)) + else: + GlobalError(v.info, "cannot map string value to FFI") of tyArray, tyArrayConstr: let baseSize = typ.sons[1].getSize - assert(v.len == lengthOrd(typ.sons[0])) for i in 0 .. Date: Wed, 2 Jan 2013 01:17:53 +0100 Subject: [PATCH 058/106] next steps for FFI at compile time --- compiler/evalffi.nim | 42 +++-- compiler/evals.nim | 23 ++- compiler/main.nim | 1 - lib/system.nim | 312 +++++++++++++++++---------------- lib/wrappers/sdl/sdl_image.nim | 18 +- todo.txt | 1 + 6 files changed, 215 insertions(+), 182 deletions(-) diff --git a/compiler/evalffi.nim b/compiler/evalffi.nim index 8f708b76c..b55f2baad 100644 --- a/compiler/evalffi.nim +++ b/compiler/evalffi.nim @@ -279,6 +279,13 @@ proc unpackArray(x: pointer, typ: PType, n: PNode): PNode = for i in 0 .. < result.len: result.sons[i] = unpack(x +! i * baseSize, typ.sons[1], result.sons[i]) +proc canonNodeKind(k: TNodeKind): TNodeKind = + case k + of nkCharLit..nkUInt64Lit: result = nkIntLit + of nkFloatLit..nkFloat128Lit: result = nkFloatLit + of nkStrLit..nkTripleStrLit: result = nkStrLit + else: result = k + proc unpack(x: pointer, typ: PType, n: PNode): PNode = template aw(k, v, field: expr) {.immediate, dirty.} = if n.isNil: @@ -287,7 +294,8 @@ proc unpack(x: pointer, typ: PType, n: PNode): PNode = else: # check we have the right field: result = n - if result.kind != k: + if result.kind.canonNodeKind != k.canonNodeKind: + echo "expected ", k, " but got ", result.kind GlobalError(n.info, "cannot map value from FFI") result.field = v @@ -309,15 +317,15 @@ proc unpack(x: pointer, typ: PType, n: PNode): PNode = of tyBool: awi(nkIntLit, rd(bool, x).ord) of tyChar: awi(nkIntLit, rd(char, x).ord) of tyInt: awi(nkIntLit, rd(int, x)) - of tyInt8: awi(nkIntLit, rd(int8, x)) - of tyInt16: awi(nkIntLit, rd(int16, x)) - of tyInt32: awi(nkIntLit, rd(int32, x)) - of tyInt64: awi(nkIntLit, rd(int64, x)) - of tyUInt: awi(nkIntLit, rd(uint, x).biggestInt) - of tyUInt8: awi(nkIntLit, rd(uint8, x).biggestInt) - of tyUInt16: awi(nkIntLit, rd(uint16, x).biggestInt) - of tyUInt32: awi(nkIntLit, rd(uint32, x).biggestInt) - of tyUInt64: awi(nkIntLit, rd(uint64, x).biggestInt) + of tyInt8: awi(nkInt8Lit, rd(int8, x)) + of tyInt16: awi(nkInt16Lit, rd(int16, x)) + of tyInt32: awi(nkInt32Lit, rd(int32, x)) + of tyInt64: awi(nkInt64Lit, rd(int64, x)) + of tyUInt: awi(nkUIntLit, rd(uint, x).biggestInt) + of tyUInt8: awi(nkUInt8Lit, rd(uint8, x).biggestInt) + of tyUInt16: awi(nkUInt16Lit, rd(uint16, x).biggestInt) + of tyUInt32: awi(nkUInt32Lit, rd(uint32, x).biggestInt) + of tyUInt64: awi(nkUInt64Lit, rd(uint64, x).biggestInt) of tyEnum: case typ.getSize of 1: awi(nkIntLit, rd(uint8, x).biggestInt) @@ -327,8 +335,8 @@ proc unpack(x: pointer, typ: PType, n: PNode): PNode = else: GlobalError(n.info, "cannot map value from FFI (tyEnum, tySet)") of tyFloat: awf(nkFloatLit, rd(float, x)) - of tyFloat32: awf(nkFloatLit, rd(float32, x)) - of tyFloat64: awf(nkFloatLit, rd(float64, x)) + of tyFloat32: awf(nkFloat32Lit, rd(float32, x)) + of tyFloat64: awf(nkFloat64Lit, rd(float64, x)) of tyPointer, tyProc: let p = rd(pointer, x) if p.isNil: @@ -339,7 +347,7 @@ proc unpack(x: pointer, typ: PType, n: PNode): PNode = let p = rd(pointer, x) if p.isNil: setNil() - elif n != nil and n.kind == nkPtrLit: + elif n == nil or n.kind == nkPtrLit: awi(nkPtrLit, cast[TAddress](p)) elif n != nil and n.len == 1: n.sons[0] = unpack(rd(pointer, x), typ.sons[0], n.sons[0]) @@ -363,6 +371,14 @@ proc unpack(x: pointer, typ: PType, n: PNode): PNode = # XXX what to do with 'array' here? GlobalError(n.info, "cannot map value from FFI " & typeToString(typ)) +proc fficast*(x: PNode, destTyp: PType): PNode = + # we play safe here and allocate the max possible size: + let allocSize = max(packSize(x, x.typ), packSize(x, destTyp)) + var a = alloc0(allocSize) + pack(x, x.typ, a) + result = unpack(a, destTyp, nil) + dealloc a + proc callForeignFunction*(call: PNode): PNode = InternalAssert call.sons[0].kind == nkPtrLit diff --git a/compiler/evals.nim b/compiler/evals.nim index f0a09fff3..5623f7955 100755 --- a/compiler/evals.nim +++ b/compiler/evals.nim @@ -77,6 +77,7 @@ proc newEvalContext*(module: PSym, filename: string, new(result) result.module = module result.mode = mode + result.features = {allowFFI} initIdNodeTable(result.globals) proc pushStackFrame*(c: PEvalContext, t: PStackFrame) {.inline.} = @@ -90,7 +91,7 @@ proc popStackFrame*(c: PEvalContext) {.inline.} = proc evalMacroCall*(c: PEvalContext, n, nOrig: PNode, sym: PSym): PNode proc evalAux(c: PEvalContext, n: PNode, flags: TEvalFlags): PNode -proc raiseCannotEval(c: PEvalContext, info: TLineInfo): PNode = +proc raiseCannotEval(c: PEvalContext, info: TLineInfo): PNode = result = newNodeI(nkExceptBranch, info) # creating a nkExceptBranch without sons # means that it could not be evaluated @@ -351,7 +352,7 @@ proc evalGlobalVar(c: PEvalContext, s: PSym, flags: TEvalFlags): PNode = result = copyTree(result) else: when hasFFI: - if sfImportc in s.flags: + if sfImportc in s.flags and allowFFI in c.features: result = importcSymbol(s) IdNodeTablePut(c.globals, s, result) return result @@ -398,7 +399,7 @@ proc evalCall(c: PEvalContext, n: PNode): PNode = if n.typ != nil: d.params[0] = getNullValue(n.typ, n.info) when hasFFI: - if sfImportc in prc.sym.flags: + if sfImportc in prc.sym.flags and allowFFI in c.features: var newCall = newNodeI(nkCall, n.info, n.len) newCall.sons[0] = evalGlobalVar(c, prc.sym, {}) for i in 1 .. = 1: stackTrace(c, n, errUnhandledExceptionX, typeToString(result.typ)) else: - stackTrace(c, n, errCannotInterpretNodeX, renderTree(n)) + stackTrace(c, result, errCannotInterpretNodeX, renderTree(n)) proc evalConstExprAux(module: PSym, e: PNode, mode: TEvalMode): PNode = var p = newEvalContext(module, "", mode) @@ -1474,6 +1480,7 @@ proc evalMacroCall(c: PEvalContext, n, nOrig: PNode, sym: PSym): PNode = proc myOpen(module: PSym, filename: string): PPassContext = var c = newEvalContext(module, filename, emRepl) + c.features = {allowCast, allowFFI, allowInfiniteLoops} pushStackFrame(c, newStackFrame()) result = c diff --git a/compiler/main.nim b/compiler/main.nim index a65660380..93def6f09 100755 --- a/compiler/main.nim +++ b/compiler/main.nim @@ -153,7 +153,6 @@ proc CommandCompileToEcmaScript = proc CommandInteractive = msgs.gErrorMax = high(int) # do not stop after first error - incl(gGlobalOptions, optSafeCode) #setTarget(osNimrodVM, cpuNimrodVM) initDefines() DefineSymbol("nimrodvm") diff --git a/lib/system.nim b/lib/system.nim index cbca90a3b..8106df14a 100755 --- a/lib/system.nim +++ b/lib/system.nim @@ -1547,41 +1547,42 @@ when false: # ----------------- GC interface --------------------------------------------- -proc GC_disable*() {.rtl, inl.} - ## disables the GC. If called n-times, n calls to `GC_enable` are needed to - ## reactivate the GC. Note that in most circumstances one should only disable - ## the mark and sweep phase with `GC_disableMarkAndSweep`. +when not defined(nimrodVM): + proc GC_disable*() {.rtl, inl.} + ## disables the GC. If called n-times, n calls to `GC_enable` are needed to + ## reactivate the GC. Note that in most circumstances one should only disable + ## the mark and sweep phase with `GC_disableMarkAndSweep`. -proc GC_enable*() {.rtl, inl.} - ## enables the GC again. + proc GC_enable*() {.rtl, inl.} + ## enables the GC again. -proc GC_fullCollect*() {.rtl.} - ## forces a full garbage collection pass. - ## Ordinary code does not need to call this (and should not). + proc GC_fullCollect*() {.rtl.} + ## forces a full garbage collection pass. + ## Ordinary code does not need to call this (and should not). -type - TGC_Strategy* = enum ## the strategy the GC should use for the application - gcThroughput, ## optimize for throughput - gcResponsiveness, ## optimize for responsiveness (default) - gcOptimizeTime, ## optimize for speed - gcOptimizeSpace ## optimize for memory footprint + type + TGC_Strategy* = enum ## the strategy the GC should use for the application + gcThroughput, ## optimize for throughput + gcResponsiveness, ## optimize for responsiveness (default) + gcOptimizeTime, ## optimize for speed + gcOptimizeSpace ## optimize for memory footprint -proc GC_setStrategy*(strategy: TGC_Strategy) {.rtl, deprecated.} - ## tells the GC the desired strategy for the application. - ## **Deprecated** since version 0.8.14. This has always been a nop. + proc GC_setStrategy*(strategy: TGC_Strategy) {.rtl, deprecated.} + ## tells the GC the desired strategy for the application. + ## **Deprecated** since version 0.8.14. This has always been a nop. -proc GC_enableMarkAndSweep*() {.rtl.} -proc GC_disableMarkAndSweep*() {.rtl.} - ## the current implementation uses a reference counting garbage collector - ## with a seldomly run mark and sweep phase to free cycles. The mark and - ## sweep phase may take a long time and is not needed if the application - ## does not create cycles. Thus the mark and sweep phase can be deactivated - ## and activated separately from the rest of the GC. + proc GC_enableMarkAndSweep*() {.rtl.} + proc GC_disableMarkAndSweep*() {.rtl.} + ## the current implementation uses a reference counting garbage collector + ## with a seldomly run mark and sweep phase to free cycles. The mark and + ## sweep phase may take a long time and is not needed if the application + ## does not create cycles. Thus the mark and sweep phase can be deactivated + ## and activated separately from the rest of the GC. -proc GC_getStatistics*(): string {.rtl.} - ## returns an informative string about the GC's activity. This may be useful - ## for tweaking. - + proc GC_getStatistics*(): string {.rtl.} + ## returns an informative string about the GC's activity. This may be useful + ## for tweaking. + proc GC_ref*[T](x: ref T) {.magic: "GCref".} proc GC_ref*[T](x: seq[T]) {.magic: "GCref".} proc GC_ref*(x: string) {.magic: "GCref".} @@ -1707,28 +1708,29 @@ proc getTypeInfo*[T](x: T): pointer {.magic: "GetTypeInfo".} ## get type information for `x`. Ordinary code should not use this, but ## the `typeinfo` module instead. -when not defined(EcmaScript) and not defined(NimrodVM): +when not defined(EcmaScript): #and not defined(NimrodVM): {.push stack_trace: off, profiler:off.} - proc initGC() - when not defined(boehmgc) and not defined(useMalloc): - proc initAllocator() {.inline.} + when not defined(NimrodVM): + proc initGC() + when not defined(boehmgc) and not defined(useMalloc): + proc initAllocator() {.inline.} - proc initStackBottom() {.inline, compilerproc.} = - # WARNING: This is very fragile! An array size of 8 does not work on my - # Linux 64bit system. -- That's because the stack direction is the other - # way round. - when defined(setStackBottom): - var locals {.volatile.}: pointer - locals = addr(locals) - setStackBottom(locals) + proc initStackBottom() {.inline, compilerproc.} = + # WARNING: This is very fragile! An array size of 8 does not work on my + # Linux 64bit system. -- That's because the stack direction is the other + # way round. + when defined(setStackBottom): + var locals {.volatile.}: pointer + locals = addr(locals) + setStackBottom(locals) - var - strDesc: TNimType + var + strDesc: TNimType - strDesc.size = sizeof(string) - strDesc.kind = tyString - strDesc.flags = {ntfAcyclic} + strDesc.size = sizeof(string) + strDesc.kind = tyString + strDesc.flags = {ntfAcyclic} include "system/ansi_c" @@ -1736,22 +1738,23 @@ when not defined(EcmaScript) and not defined(NimrodVM): result = int(c_strcmp(x, y)) const pccHack = if defined(pcc): "_" else: "" # Hack for PCC - when defined(windows): - # work-around C's sucking abstraction: - # BUGFIX: stdin and stdout should be binary files! - proc setmode(handle, mode: int) {.importc: pccHack & "setmode", - header: "".} - proc fileno(f: C_TextFileStar): int {.importc: pccHack & "fileno", - header: "".} - var - O_BINARY {.importc: pccHack & "O_BINARY", nodecl.}: int + when not defined(NimrodVM): + when defined(windows): + # work-around C's sucking abstraction: + # BUGFIX: stdin and stdout should be binary files! + proc setmode(handle, mode: int) {.importc: pccHack & "setmode", + header: "".} + proc fileno(f: C_TextFileStar): int {.importc: pccHack & "fileno", + header: "".} + var + O_BINARY {.importc: pccHack & "O_BINARY", nodecl.}: int - # we use binary mode in Windows: - setmode(fileno(c_stdin), O_BINARY) - setmode(fileno(c_stdout), O_BINARY) - - when defined(endb): - proc endbStep() + # we use binary mode in Windows: + setmode(fileno(c_stdin), O_BINARY) + setmode(fileno(c_stdout), O_BINARY) + + when defined(endb): + proc endbStep() # ----------------- IO Part ------------------------------------------------ type @@ -1957,13 +1960,14 @@ when not defined(EcmaScript) and not defined(NimrodVM): inc(i) dealloc(a) - proc atomicInc*(memLoc: var int, x: int = 1): int {.inline, discardable.} - ## atomic increment of `memLoc`. Returns the value after the operation. - - proc atomicDec*(memLoc: var int, x: int = 1): int {.inline, discardable.} - ## atomic decrement of `memLoc`. Returns the value after the operation. + when not defined(NimrodVM): + proc atomicInc*(memLoc: var int, x: int = 1): int {.inline, discardable.} + ## atomic increment of `memLoc`. Returns the value after the operation. + + proc atomicDec*(memLoc: var int, x: int = 1): int {.inline, discardable.} + ## atomic decrement of `memLoc`. Returns the value after the operation. - include "system/atomics" + include "system/atomics" type PSafePoint = ptr TSafePoint @@ -1979,71 +1983,76 @@ when not defined(EcmaScript) and not defined(NimrodVM): when hasThreadSupport: include "system/syslocks" include "system/threads" - elif not defined(nogc): + elif not defined(nogc) and not defined(NimrodVM): when not defined(useNimRtl) and not defined(createNimRtl): initStackBottom() initGC() - proc setControlCHook*(hook: proc () {.noconv.}) - ## allows you to override the behaviour of your application when CTRL+C - ## is pressed. Only one such hook is supported. - - proc writeStackTrace*() {.tags: [FWriteIO].} - ## writes the current stack trace to ``stderr``. This is only works - ## for debug builds. - when hostOS != "standalone": - proc getStackTrace*(): string - ## gets the current stack trace. This only works for debug builds. - - proc getStackTrace*(e: ref E_Base): string - ## gets the stack trace associated with `e`, which is the stack that - ## lead to the ``raise`` statement. This only works for debug builds. + when not defined(NimrodVM): + proc setControlCHook*(hook: proc () {.noconv.}) + ## allows you to override the behaviour of your application when CTRL+C + ## is pressed. Only one such hook is supported. - {.push stack_trace: off, profiler:off.} - when hostOS == "standalone": - include "system/embedded" - else: - include "system/excpt" - - # we cannot compile this with stack tracing on - # as it would recurse endlessly! - include "system/arithm" - {.pop.} # stack trace - {.pop.} # stack trace - - when hostOS != "standalone": include "system/dyncalls" - include "system/sets" + proc writeStackTrace*() {.tags: [FWriteIO].} + ## writes the current stack trace to ``stderr``. This is only works + ## for debug builds. + when hostOS != "standalone": + proc getStackTrace*(): string + ## gets the current stack trace. This only works for debug builds. - const - GenericSeqSize = (2 * sizeof(int)) - - proc getDiscriminant(aa: Pointer, n: ptr TNimNode): int = - sysAssert(n.kind == nkCase, "getDiscriminant: node != nkCase") - var d: int - var a = cast[TAddress](aa) - case n.typ.size - of 1: d = ze(cast[ptr int8](a +% n.offset)[]) - of 2: d = ze(cast[ptr int16](a +% n.offset)[]) - of 4: d = int(cast[ptr int32](a +% n.offset)[]) - else: sysAssert(false, "getDiscriminant: invalid n.typ.size") - return d - - proc selectBranch(aa: Pointer, n: ptr TNimNode): ptr TNimNode = - var discr = getDiscriminant(aa, n) - if discr <% n.len: - result = n.sons[discr] - if result == nil: result = n.sons[n.len] - # n.sons[n.len] contains the ``else`` part (but may be nil) + proc getStackTrace*(e: ref E_Base): string + ## gets the stack trace associated with `e`, which is the stack that + ## lead to the ``raise`` statement. This only works for debug builds. + + {.push stack_trace: off, profiler:off.} + when hostOS == "standalone": + include "system/embedded" else: - result = n.sons[n.len] + include "system/excpt" + + # we cannot compile this with stack tracing on + # as it would recurse endlessly! + include "system/arithm" + {.pop.} # stack trace + {.pop.} # stack trace + + when hostOS != "standalone" and not defined(NimrodVM): + include "system/dyncalls" + when not defined(NimrodVM): + include "system/sets" - include "system/mmdisp" - {.push stack_trace: off, profiler:off.} - when hostOS != "standalone": include "system/sysstr" - {.pop.} + const + GenericSeqSize = (2 * sizeof(int)) + + proc getDiscriminant(aa: Pointer, n: ptr TNimNode): int = + sysAssert(n.kind == nkCase, "getDiscriminant: node != nkCase") + var d: int + var a = cast[TAddress](aa) + case n.typ.size + of 1: d = ze(cast[ptr int8](a +% n.offset)[]) + of 2: d = ze(cast[ptr int16](a +% n.offset)[]) + of 4: d = int(cast[ptr int32](a +% n.offset)[]) + else: sysAssert(false, "getDiscriminant: invalid n.typ.size") + return d - include "system/sysio" - when hasThreadSupport: - include "system/channels" + proc selectBranch(aa: Pointer, n: ptr TNimNode): ptr TNimNode = + var discr = getDiscriminant(aa, n) + if discr <% n.len: + result = n.sons[discr] + if result == nil: result = n.sons[n.len] + # n.sons[n.len] contains the ``else`` part (but may be nil) + else: + result = n.sons[n.len] + + include "system/mmdisp" + {.push stack_trace: off, profiler:off.} + when hostOS != "standalone": include "system/sysstr" + {.pop.} + + include "system/sysio" + when hasThreadSupport: + include "system/channels" + else: + include "system/sysio" iterator lines*(filename: string): TaintedString {.tags: [FReadIO].} = ## Iterate over any line in the file named `filename`. @@ -2058,7 +2067,7 @@ when not defined(EcmaScript) and not defined(NimrodVM): var res = TaintedString(newStringOfCap(80)) while f.readLine(res): yield TaintedString(res) - when hostOS != "standalone": + when hostOS != "standalone" and not defined(NimrodVM): include "system/assign" include "system/repr" @@ -2083,42 +2092,43 @@ when not defined(EcmaScript) and not defined(NimrodVM): excHandler.raiseAction = action {.push stack_trace: off, profiler:off.} - when defined(endb): + when defined(endb) and not defined(NimrodVM): include "system/debugger" when defined(profiler) or defined(memProfiler): include "system/profiler" {.pop.} # stacktrace - proc likely*(val: bool): bool {.importc: "likely", nodecl, nosideeffect.} - ## can be used to mark a condition to be likely. This is a hint for the - ## optimizer. - - proc unlikely*(val: bool): bool {.importc: "unlikely", nodecl, nosideeffect.} - ## can be used to mark a condition to be unlikely. This is a hint for the - ## optimizer. + when not defined(NimrodVM): + proc likely*(val: bool): bool {.importc: "likely", nodecl, nosideeffect.} + ## can be used to mark a condition to be likely. This is a hint for the + ## optimizer. - proc rawProc*[T: proc](x: T): pointer {.noSideEffect, inline.} = - ## retrieves the raw proc pointer of the closure `x`. This is - ## useful for interfacing closures with C. - {.emit: """ - `result` = `x`.ClPrc; - """.} + proc unlikely*(val: bool): bool {.importc: "unlikely", nodecl, nosideeffect.} + ## can be used to mark a condition to be unlikely. This is a hint for the + ## optimizer. + + proc rawProc*[T: proc](x: T): pointer {.noSideEffect, inline.} = + ## retrieves the raw proc pointer of the closure `x`. This is + ## useful for interfacing closures with C. + {.emit: """ + `result` = `x`.ClPrc; + """.} - proc rawEnv*[T: proc](x: T): pointer {.noSideEffect, inline.} = - ## retrieves the raw environment pointer of the closure `x`. This is - ## useful for interfacing closures with C. - {.emit: """ - `result` = `x`.ClEnv; - """.} + proc rawEnv*[T: proc](x: T): pointer {.noSideEffect, inline.} = + ## retrieves the raw environment pointer of the closure `x`. This is + ## useful for interfacing closures with C. + {.emit: """ + `result` = `x`.ClEnv; + """.} - proc finished*[T: proc](x: T): bool {.noSideEffect, inline.} = - ## can be used to determine if a first class iterator has finished. - {.emit: """ - `result` = *((NI*) `x`.ClEnv) < 0; - """.} + proc finished*[T: proc](x: T): bool {.noSideEffect, inline.} = + ## can be used to determine if a first class iterator has finished. + {.emit: """ + `result` = *((NI*) `x`.ClEnv) < 0; + """.} -elif defined(ecmaScript) or defined(NimrodVM): +elif defined(ecmaScript): # Stubs: proc nimGCvisit(d: pointer, op: int) {.compilerRtl.} = nil diff --git a/lib/wrappers/sdl/sdl_image.nim b/lib/wrappers/sdl/sdl_image.nim index 7df9aedd4..16e41070b 100755 --- a/lib/wrappers/sdl/sdl_image.nim +++ b/lib/wrappers/sdl/sdl_image.nim @@ -128,19 +128,19 @@ # #****************************************************************************** -import +import sdl -when defined(windows): - const +when defined(windows): + const ImageLibName = "SDL_Image.dll" -elif defined(macosx): - const +elif defined(macosx): + const ImageLibName = "libSDL_image-1.2.0.dylib" -else: - const - ImageLibName = "libSDL_image.so" -const +else: + const + ImageLibName = "libSDL_image(.so|-1.2.so.0)" +const IMAGE_MAJOR_VERSION* = 1 IMAGE_MINOR_VERSION* = 2 IMAGE_PATCHLEVEL* = 5 diff --git a/todo.txt b/todo.txt index caecc9b70..67f4e617f 100755 --- a/todo.txt +++ b/todo.txt @@ -27,6 +27,7 @@ version 0.9.X - optimize genericAssign in the code generator - better support for macros that rewrite procs - macros need access to types and symbols (partially implemented) +- result = result shr 8 for the "system()" wrapper - rethink the syntax/grammar: * parser is not strict enough with newlines From 97d4f2138b34ff1eb3fd63c0ea430a7743df8638 Mon Sep 17 00:00:00 2001 From: Grzegorz Adam Hankiewicz Date: Thu, 3 Jan 2013 15:35:16 +0100 Subject: [PATCH 059/106] Fixes tutorial pdf generation due to hierarchy depth. The hierarchy is now an ascii reprsentation. --- doc/tut2.txt | 64 ++++++++++++++++++++++++++-------------------------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/doc/tut2.txt b/doc/tut2.txt index 9c55b0e35..9f9dbe2db 100755 --- a/doc/tut2.txt +++ b/doc/tut2.txt @@ -439,39 +439,39 @@ Exception hierarchy If you want to create your own exceptions you can inherit from E_Base, but you can also inherit from one of the existing exceptions if they fit your purpose. -The exception tree is: +The exception tree is:: -* E_Base - * EAsynch - * EControlC - * ESynch - * ESystem - * EIO - * EOS - * EInvalidLibrary - * EResourceExhausted - * EOutOfMemory - * EStackOverflow - * EArithmetic - * EDivByZero - * EOverflow - * EAccessViolation - * EAssertionFailed - * EInvalidValue - * EInvalidKey - * EInvalidIndex - * EInvalidField - * EOutOfRange - * ENoExceptionToReraise - * EInvalidObjectAssignment - * EInvalidObjectConversion - * EFloatingPoint - * EFloatInvalidOp - * EFloatDivByZero - * EFloatOverflow - * EFloatUnderflow - * EFloatInexact - * EDeadThread + * E_Base + * EAsynch + * EControlC + * ESynch + * ESystem + * EIO + * EOS + * EInvalidLibrary + * EResourceExhausted + * EOutOfMemory + * EStackOverflow + * EArithmetic + * EDivByZero + * EOverflow + * EAccessViolation + * EAssertionFailed + * EInvalidValue + * EInvalidKey + * EInvalidIndex + * EInvalidField + * EOutOfRange + * ENoExceptionToReraise + * EInvalidObjectAssignment + * EInvalidObjectConversion + * EFloatingPoint + * EFloatInvalidOp + * EFloatDivByZero + * EFloatOverflow + * EFloatUnderflow + * EFloatInexact + * EDeadThread See the `system `_ module for a description of each exception. From bd961fd76c1d61586ccdebc8abafa401ad3428e3 Mon Sep 17 00:00:00 2001 From: Grzegorz Adam Hankiewicz Date: Thu, 13 Dec 2012 19:28:34 +0100 Subject: [PATCH 060/106] Moves toSeq template to public sequtils module. --- lib/pure/collections/sequtils.nim | 6 ++++++ tests/run/ttoseq.nim | 11 +++++------ 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/lib/pure/collections/sequtils.nim b/lib/pure/collections/sequtils.nim index 82679bb7f..0fda5700e 100644 --- a/lib/pure/collections/sequtils.nim +++ b/lib/pure/collections/sequtils.nim @@ -63,3 +63,9 @@ template filterIt*(seq1, pred: expr): expr {.immediate, dirty.} = if pred: result.add(it) result +template toSeq*(iter: expr): expr {.immediate.} = + ## Transforms any iterator into a sequence. + var result {.gensym.}: seq[type(iter)] = @[] + for x in iter: add(result, x) + result + diff --git a/tests/run/ttoseq.nim b/tests/run/ttoseq.nim index ec49489d0..34cc4824b 100755 --- a/tests/run/ttoseq.nim +++ b/tests/run/ttoseq.nim @@ -1,12 +1,11 @@ discard """ - output: "23456" + output: "2345623456" """ -template toSeq*(iter: expr): expr {.immediate.} = - var result: seq[type(iter)] = @[] - for x in iter: add(result, x) - result - +import sequtils + +for x in toSeq(countup(2, 6)): + stdout.write(x) for x in items(toSeq(countup(2, 6))): stdout.write(x) From 43b6daf2d63d81dd44f377e09f684eaa15b5035b Mon Sep 17 00:00:00 2001 From: Araq Date: Tue, 8 Jan 2013 01:19:02 +0100 Subject: [PATCH 061/106] FFI at compiletime improvements --- compiler/astalgo.nim | 2 +- compiler/evalffi.nim | 69 ++++++++++----- compiler/llstream.nim | 17 ++-- lib/system.nim | 202 +++++++++++++++++++++--------------------- todo.txt | 6 +- 5 files changed, 165 insertions(+), 131 deletions(-) diff --git a/compiler/astalgo.nim b/compiler/astalgo.nim index 9a2bebab4..564f262d7 100755 --- a/compiler/astalgo.nim +++ b/compiler/astalgo.nim @@ -429,7 +429,7 @@ proc debugTree(n: PNode, indent: int, maxRecDepth: int): PRope = [istr, makeYamlString($n.kind)]) if maxRecDepth != 0: case n.kind - of nkCharLit..nkInt64Lit: + of nkCharLit..nkUInt64Lit: appf(result, ",$N$1\"intVal\": $2", [istr, toRope(n.intVal)]) of nkFloatLit, nkFloat32Lit, nkFloat64Lit: appf(result, ",$N$1\"floatVal\": $2", diff --git a/compiler/evalffi.nim b/compiler/evalffi.nim index b55f2baad..ba6e7ee8f 100644 --- a/compiler/evalffi.nim +++ b/compiler/evalffi.nim @@ -156,7 +156,10 @@ proc packObject(x: PNode, typ: PType, res: pointer) = pack(it, field.typ, res +! field.offset) else: GlobalError(x.info, "cannot pack unnamed tuple") - + +const maxPackDepth = 20 +var packRecCheck = 0 + proc pack(v: PNode, typ: PType, res: pointer) = template awr(T, v: expr) {.immediate, dirty.} = wr(T, res, v) @@ -186,12 +189,14 @@ proc pack(v: PNode, typ: PType, res: pointer) = of tyFloat32: awr(float32, v.floatVal) of tyFloat64: awr(float64, v.floatVal) - of tyPointer, tyProc: + of tyPointer, tyProc, tyCString, tyString: if v.kind == nkNilLit: # nothing to do since the memory is 0 initialized anyway nil elif v.kind == nkPtrLit: awr(pointer, cast[pointer](v.intVal)) + elif v.kind in {nkStrLit..nkTripleStrLit}: + awr(cstring, cstring(v.strVal)) else: GlobalError(v.info, "cannot map pointer/proc value to FFI") of tyPtr, tyRef, tyVar: @@ -201,15 +206,13 @@ proc pack(v: PNode, typ: PType, res: pointer) = elif v.kind == nkPtrLit: awr(pointer, cast[pointer](v.intVal)) else: + if packRecCheck > maxPackDepth: + packRecCheck = 0 + GlobalError(v.info, "cannot map value to FFI " & typeToString(v.typ)) + inc packRecCheck pack(v.sons[0], typ.sons[0], res +! sizeof(pointer)) + dec packRecCheck awr(pointer, res +! sizeof(pointer)) - of tyCString, tyString: - if v.kind == nkNilLit: - nil - elif v.kind in {nkStrLit..nkTripleStrLit}: - awr(cstring, cstring(v.strVal)) - else: - GlobalError(v.info, "cannot map string value to FFI") of tyArray, tyArrayConstr: let baseSize = typ.sons[1].getSize for i in 0 .. >> " else: "... ", line): add(s.s, line) add(s.s, "\n") - if line.contains("\"\"\""): - inTripleString = not inTripleString - if not continueLine(line, inTripleString): break + inc triples, countTriples(line) + if not continueLine(line, (triples and 1) == 1): break inc(s.lineOffset) result = min(bufLen, len(s.s) - s.rd) if result > 0: diff --git a/lib/system.nim b/lib/system.nim index 8106df14a..3d17b1d6d 100755 --- a/lib/system.nim +++ b/lib/system.nim @@ -1066,86 +1066,87 @@ proc substr*(s: string, first, last: int): string {. ## is used instead: This means ``substr`` can also be used to `cut`:idx: ## or `limit`:idx: a string's length. -proc zeroMem*(p: Pointer, size: int) {.importc, noDecl.} - ## overwrites the contents of the memory at ``p`` with the value 0. - ## Exactly ``size`` bytes will be overwritten. Like any procedure - ## dealing with raw memory this is *unsafe*. +when not defined(nimrodVM): + proc zeroMem*(p: Pointer, size: int) {.importc, noDecl.} + ## overwrites the contents of the memory at ``p`` with the value 0. + ## Exactly ``size`` bytes will be overwritten. Like any procedure + ## dealing with raw memory this is *unsafe*. -proc copyMem*(dest, source: Pointer, size: int) {.importc: "memcpy", noDecl.} - ## copies the contents from the memory at ``source`` to the memory - ## at ``dest``. Exactly ``size`` bytes will be copied. The memory - ## regions may not overlap. Like any procedure dealing with raw - ## memory this is *unsafe*. + proc copyMem*(dest, source: Pointer, size: int) {.importc: "memcpy", noDecl.} + ## copies the contents from the memory at ``source`` to the memory + ## at ``dest``. Exactly ``size`` bytes will be copied. The memory + ## regions may not overlap. Like any procedure dealing with raw + ## memory this is *unsafe*. -proc moveMem*(dest, source: Pointer, size: int) {.importc: "memmove", noDecl.} - ## copies the contents from the memory at ``source`` to the memory - ## at ``dest``. Exactly ``size`` bytes will be copied. The memory - ## regions may overlap, ``moveMem`` handles this case appropriately - ## and is thus somewhat more safe than ``copyMem``. Like any procedure - ## dealing with raw memory this is still *unsafe*, though. + proc moveMem*(dest, source: Pointer, size: int) {.importc: "memmove", noDecl.} + ## copies the contents from the memory at ``source`` to the memory + ## at ``dest``. Exactly ``size`` bytes will be copied. The memory + ## regions may overlap, ``moveMem`` handles this case appropriately + ## and is thus somewhat more safe than ``copyMem``. Like any procedure + ## dealing with raw memory this is still *unsafe*, though. -proc equalMem*(a, b: Pointer, size: int): bool {. - importc: "equalMem", noDecl, noSideEffect.} - ## compares the memory blocks ``a`` and ``b``. ``size`` bytes will - ## be compared. If the blocks are equal, true is returned, false - ## otherwise. Like any procedure dealing with raw memory this is - ## *unsafe*. + proc equalMem*(a, b: Pointer, size: int): bool {. + importc: "equalMem", noDecl, noSideEffect.} + ## compares the memory blocks ``a`` and ``b``. ``size`` bytes will + ## be compared. If the blocks are equal, true is returned, false + ## otherwise. Like any procedure dealing with raw memory this is + ## *unsafe*. -proc alloc*(size: int): pointer {.noconv, rtl, tags: [].} - ## allocates a new memory block with at least ``size`` bytes. The - ## block has to be freed with ``realloc(block, 0)`` or - ## ``dealloc(block)``. The block is not initialized, so reading - ## from it before writing to it is undefined behaviour! - ## The allocated memory belongs to its allocating thread! - ## Use `allocShared` to allocate from a shared heap. -proc alloc0*(size: int): pointer {.noconv, rtl, tags: [].} - ## allocates a new memory block with at least ``size`` bytes. The - ## block has to be freed with ``realloc(block, 0)`` or - ## ``dealloc(block)``. The block is initialized with all bytes - ## containing zero, so it is somewhat safer than ``alloc``. - ## The allocated memory belongs to its allocating thread! - ## Use `allocShared0` to allocate from a shared heap. -proc realloc*(p: Pointer, newsize: int): pointer {.noconv, rtl, tags: [].} - ## grows or shrinks a given memory block. If p is **nil** then a new - ## memory block is returned. In either way the block has at least - ## ``newsize`` bytes. If ``newsize == 0`` and p is not **nil** - ## ``realloc`` calls ``dealloc(p)``. In other cases the block has to - ## be freed with ``dealloc``. - ## The allocated memory belongs to its allocating thread! - ## Use `reallocShared` to reallocate from a shared heap. -proc dealloc*(p: Pointer) {.noconv, rtl, tags: [].} - ## frees the memory allocated with ``alloc``, ``alloc0`` or - ## ``realloc``. This procedure is dangerous! If one forgets to - ## free the memory a leak occurs; if one tries to access freed - ## memory (or just freeing it twice!) a core dump may happen - ## or other memory may be corrupted. - ## The freed memory must belong to its allocating thread! - ## Use `deallocShared` to deallocate from a shared heap. + proc alloc*(size: int): pointer {.noconv, rtl, tags: [].} + ## allocates a new memory block with at least ``size`` bytes. The + ## block has to be freed with ``realloc(block, 0)`` or + ## ``dealloc(block)``. The block is not initialized, so reading + ## from it before writing to it is undefined behaviour! + ## The allocated memory belongs to its allocating thread! + ## Use `allocShared` to allocate from a shared heap. + proc alloc0*(size: int): pointer {.noconv, rtl, tags: [].} + ## allocates a new memory block with at least ``size`` bytes. The + ## block has to be freed with ``realloc(block, 0)`` or + ## ``dealloc(block)``. The block is initialized with all bytes + ## containing zero, so it is somewhat safer than ``alloc``. + ## The allocated memory belongs to its allocating thread! + ## Use `allocShared0` to allocate from a shared heap. + proc realloc*(p: Pointer, newsize: int): pointer {.noconv, rtl, tags: [].} + ## grows or shrinks a given memory block. If p is **nil** then a new + ## memory block is returned. In either way the block has at least + ## ``newsize`` bytes. If ``newsize == 0`` and p is not **nil** + ## ``realloc`` calls ``dealloc(p)``. In other cases the block has to + ## be freed with ``dealloc``. + ## The allocated memory belongs to its allocating thread! + ## Use `reallocShared` to reallocate from a shared heap. + proc dealloc*(p: Pointer) {.noconv, rtl, tags: [].} + ## frees the memory allocated with ``alloc``, ``alloc0`` or + ## ``realloc``. This procedure is dangerous! If one forgets to + ## free the memory a leak occurs; if one tries to access freed + ## memory (or just freeing it twice!) a core dump may happen + ## or other memory may be corrupted. + ## The freed memory must belong to its allocating thread! + ## Use `deallocShared` to deallocate from a shared heap. -proc allocShared*(size: int): pointer {.noconv, rtl.} - ## allocates a new memory block on the shared heap with at - ## least ``size`` bytes. The block has to be freed with - ## ``reallocShared(block, 0)`` or ``deallocShared(block)``. The block - ## is not initialized, so reading from it before writing to it is - ## undefined behaviour! -proc allocShared0*(size: int): pointer {.noconv, rtl.} - ## allocates a new memory block on the shared heap with at - ## least ``size`` bytes. The block has to be freed with - ## ``reallocShared(block, 0)`` or ``deallocShared(block)``. - ## The block is initialized with all bytes - ## containing zero, so it is somewhat safer than ``allocShared``. -proc reallocShared*(p: Pointer, newsize: int): pointer {.noconv, rtl.} - ## grows or shrinks a given memory block on the heap. If p is **nil** - ## then a new memory block is returned. In either way the block has at least - ## ``newsize`` bytes. If ``newsize == 0`` and p is not **nil** - ## ``reallocShared`` calls ``deallocShared(p)``. In other cases the - ## block has to be freed with ``deallocShared``. -proc deallocShared*(p: Pointer) {.noconv, rtl.} - ## frees the memory allocated with ``allocShared``, ``allocShared0`` or - ## ``reallocShared``. This procedure is dangerous! If one forgets to - ## free the memory a leak occurs; if one tries to access freed - ## memory (or just freeing it twice!) a core dump may happen - ## or other memory may be corrupted. + proc allocShared*(size: int): pointer {.noconv, rtl.} + ## allocates a new memory block on the shared heap with at + ## least ``size`` bytes. The block has to be freed with + ## ``reallocShared(block, 0)`` or ``deallocShared(block)``. The block + ## is not initialized, so reading from it before writing to it is + ## undefined behaviour! + proc allocShared0*(size: int): pointer {.noconv, rtl.} + ## allocates a new memory block on the shared heap with at + ## least ``size`` bytes. The block has to be freed with + ## ``reallocShared(block, 0)`` or ``deallocShared(block)``. + ## The block is initialized with all bytes + ## containing zero, so it is somewhat safer than ``allocShared``. + proc reallocShared*(p: Pointer, newsize: int): pointer {.noconv, rtl.} + ## grows or shrinks a given memory block on the heap. If p is **nil** + ## then a new memory block is returned. In either way the block has at least + ## ``newsize`` bytes. If ``newsize == 0`` and p is not **nil** + ## ``reallocShared`` calls ``deallocShared(p)``. In other cases the + ## block has to be freed with ``deallocShared``. + proc deallocShared*(p: Pointer) {.noconv, rtl.} + ## frees the memory allocated with ``allocShared``, ``allocShared0`` or + ## ``reallocShared``. This procedure is dangerous! If one forgets to + ## free the memory a leak occurs; if one tries to access freed + ## memory (or just freeing it twice!) a core dump may happen + ## or other memory may be corrupted. proc swap*[T](a, b: var T) {.magic: "Swap", noSideEffect.} ## swaps the values `a` and `b`. This is often more efficient than @@ -1221,15 +1222,16 @@ const # GC interface: -proc getOccupiedMem*(): int {.rtl.} - ## returns the number of bytes that are owned by the process and hold data. +when not defined(nimrodVM): + proc getOccupiedMem*(): int {.rtl.} + ## returns the number of bytes that are owned by the process and hold data. -proc getFreeMem*(): int {.rtl.} - ## returns the number of bytes that are owned by the process, but do not - ## hold any meaningful data. + proc getFreeMem*(): int {.rtl.} + ## returns the number of bytes that are owned by the process, but do not + ## hold any meaningful data. -proc getTotalMem*(): int {.rtl.} - ## returns the number of bytes that are owned by the process. + proc getTotalMem*(): int {.rtl.} + ## returns the number of bytes that are owned by the process. iterator countdown*[T](a, b: T, step = 1): T {.inline.} = @@ -1942,25 +1944,25 @@ when not defined(EcmaScript): #and not defined(NimrodVM): # ------------------------------------------------------------------------- - proc allocCStringArray*(a: openArray[string]): cstringArray = - ## creates a NULL terminated cstringArray from `a`. The result has to - ## be freed with `deallocCStringArray` after it's not needed anymore. - result = cast[cstringArray](alloc0((a.len+1) * sizeof(cstring))) - for i in 0 .. a.high: - # XXX get rid of this string copy here: - var x = a[i] - result[i] = cast[cstring](alloc0(x.len+1)) - copyMem(result[i], addr(x[0]), x.len) - - proc deallocCStringArray*(a: cstringArray) = - ## frees a NULL terminated cstringArray. - var i = 0 - while a[i] != nil: - dealloc(a[i]) - inc(i) - dealloc(a) - when not defined(NimrodVM): + proc allocCStringArray*(a: openArray[string]): cstringArray = + ## creates a NULL terminated cstringArray from `a`. The result has to + ## be freed with `deallocCStringArray` after it's not needed anymore. + result = cast[cstringArray](alloc0((a.len+1) * sizeof(cstring))) + for i in 0 .. a.high: + # XXX get rid of this string copy here: + var x = a[i] + result[i] = cast[cstring](alloc0(x.len+1)) + copyMem(result[i], addr(x[0]), x.len) + + proc deallocCStringArray*(a: cstringArray) = + ## frees a NULL terminated cstringArray. + var i = 0 + while a[i] != nil: + dealloc(a[i]) + inc(i) + dealloc(a) + proc atomicInc*(memLoc: var int, x: int = 1): int {.inline, discardable.} ## atomic increment of `memLoc`. Returns the value after the operation. diff --git a/todo.txt b/todo.txt index 67f4e617f..0b6457efc 100755 --- a/todo.txt +++ b/todo.txt @@ -2,9 +2,7 @@ version 0.9.2 ============= - FFI: - * make system.nim aware of nimffi * test libffi on windows - * test: SDL with the FFI * test: times.format with the FFI - fix closure bug finally - fix marshal bug @@ -55,7 +53,9 @@ version 0.9.XX - object branch transitions can't work with the current 'reset'; add a 'reset' with an additional parameter --> re-evaluate this issue after constructors have been added -- fix destructors; don't work yet when used as expression +- fix destructors; don't work yet when used as expression; alternative for + version 1: disallow expressions yielding a type with a destructor that are + not in a 'let/var' context (p(a.openFile, b.openFile) makes no sense anyway) - document nimdoc properly finally - make 'clamp' a magic for the range stuff From 3c73654acaff5aef6f5ca60facf2bf43c00a4e36 Mon Sep 17 00:00:00 2001 From: Araq Date: Tue, 8 Jan 2013 07:54:23 +0100 Subject: [PATCH 062/106] Copyright 2012 -> 2013 --- compiler/commands.nim | 4 ++-- compiler/main.nim | 2 +- compiler/nimrod.nim | 2 +- koch.nim | 4 ++-- lib/system/sysio.nim | 2 +- readme.md | 2 +- readme.txt | 2 +- web/news.txt | 2 +- 8 files changed, 10 insertions(+), 10 deletions(-) diff --git a/compiler/commands.nim b/compiler/commands.nim index 8c0e9a784..d999dc065 100755 --- a/compiler/commands.nim +++ b/compiler/commands.nim @@ -1,7 +1,7 @@ # # # The Nimrod Compiler -# (c) Copyright 2012 Andreas Rumpf +# (c) Copyright 2013 Andreas Rumpf # # See the file "copying.txt", included in this # distribution, for details about the copyright. @@ -27,7 +27,7 @@ proc processSwitch*(switch, arg: string, pass: TCmdlinePass, info: TLineInfo) const HelpMessage = "Nimrod Compiler Version $1 (" & compileDate & ") [$2: $3]\n" & - "Copyright (c) 2004-2012 by Andreas Rumpf\n" + "Copyright (c) 2004-2013 by Andreas Rumpf\n" const Usage = slurp"doc/basicopt.txt".replace("//", "") diff --git a/compiler/main.nim b/compiler/main.nim index 93def6f09..517bceb3c 100755 --- a/compiler/main.nim +++ b/compiler/main.nim @@ -1,7 +1,7 @@ # # # The Nimrod Compiler -# (c) Copyright 2012 Andreas Rumpf +# (c) Copyright 2013 Andreas Rumpf # # See the file "copying.txt", included in this # distribution, for details about the copyright. diff --git a/compiler/nimrod.nim b/compiler/nimrod.nim index 1a73bcca0..24b8518d7 100755 --- a/compiler/nimrod.nim +++ b/compiler/nimrod.nim @@ -1,7 +1,7 @@ # # # The Nimrod Compiler -# (c) Copyright 2012 Andreas Rumpf +# (c) Copyright 2013 Andreas Rumpf # # See the file "copying.txt", included in this # distribution, for details about the copyright. diff --git a/koch.nim b/koch.nim index b01de94cb..2a22d7ca5 100755 --- a/koch.nim +++ b/koch.nim @@ -1,7 +1,7 @@ # # # Maintenance program for Nimrod -# (c) Copyright 2012 Andreas Rumpf +# (c) Copyright 2013 Andreas Rumpf # # See the file "copying.txt", included in this # distribution, for details about the copyright. @@ -26,7 +26,7 @@ const +-----------------------------------------------------------------+ | Maintenance program for Nimrod | | Version $1| -| (c) 2012 Andreas Rumpf | +| (c) 2013 Andreas Rumpf | +-----------------------------------------------------------------+ Build time: $2, $3 diff --git a/lib/system/sysio.nim b/lib/system/sysio.nim index 9055cca40..c7accbac0 100755 --- a/lib/system/sysio.nim +++ b/lib/system/sysio.nim @@ -1,7 +1,7 @@ # # # Nimrod's Runtime Library -# (c) Copyright 2012 Andreas Rumpf +# (c) Copyright 2013 Andreas Rumpf # # See the file "copying.txt", included in this # distribution, for details about the copyright. diff --git a/readme.md b/readme.md index d5cb35820..3e39b5f77 100644 --- a/readme.md +++ b/readme.md @@ -62,5 +62,5 @@ developed with Nimrod, allowing you to create commercial applications. Read copying.txt for more details. -Copyright (c) 2004-2012 Andreas Rumpf. +Copyright (c) 2004-2013 Andreas Rumpf. All rights reserved. diff --git a/readme.txt b/readme.txt index d5cb35820..3e39b5f77 100755 --- a/readme.txt +++ b/readme.txt @@ -62,5 +62,5 @@ developed with Nimrod, allowing you to create commercial applications. Read copying.txt for more details. -Copyright (c) 2004-2012 Andreas Rumpf. +Copyright (c) 2004-2013 Andreas Rumpf. All rights reserved. diff --git a/web/news.txt b/web/news.txt index 802825515..824e0c3a3 100755 --- a/web/news.txt +++ b/web/news.txt @@ -2,7 +2,7 @@ News ==== -2012-XX-XX Version 0.9.2 released +2013-XX-XX Version 0.9.2 released ================================= Version 0.9.2 has been released! Get it `here `_. From 1bc5ff6dc912d3bfdd7dbbed88d06bce18804db3 Mon Sep 17 00:00:00 2001 From: Araq Date: Tue, 8 Jan 2013 08:11:21 +0100 Subject: [PATCH 063/106] fixes #292 --- compiler/sempass2.nim | 9 ++++++--- tests/compile/teffects1.nim | 17 +++++++++++++++++ 2 files changed, 23 insertions(+), 3 deletions(-) create mode 100644 tests/compile/teffects1.nim diff --git a/compiler/sempass2.nim b/compiler/sempass2.nim index bd8a3ba02..b2fd0fb04 100644 --- a/compiler/sempass2.nim +++ b/compiler/sempass2.nim @@ -1,7 +1,7 @@ # # # The Nimrod Compiler -# (c) Copyright 2012 Andreas Rumpf +# (c) Copyright 2013 Andreas Rumpf # # See the file "copying.txt", included in this # distribution, for details about the copyright. @@ -146,8 +146,11 @@ proc catches(tracked: PEffects, e: PType) = dec L else: inc i - setLen(tracked.exc.sons, L) - + if not isNil(tracked.exc.sons): + setLen(tracked.exc.sons, L) + else: + assert L == 0 + proc catchesAll(tracked: PEffects) = if not isNil(tracked.exc.sons): setLen(tracked.exc.sons, tracked.bottom) diff --git a/tests/compile/teffects1.nim b/tests/compile/teffects1.nim new file mode 100644 index 000000000..49af28469 --- /dev/null +++ b/tests/compile/teffects1.nim @@ -0,0 +1,17 @@ + +type + PMenu = ref object + PMenuItem = ref object + +proc createMenuItem*(menu: PMenu, label: string, + action: proc (i: PMenuItem, p: pointer) {.cdecl.}) = nil + +var s: PMenu +createMenuItem(s, "Go to definition...", + proc (i: PMenuItem, p: pointer) {.cdecl.} = + try: + echo(i.repr) + except EInvalidValue: + echo("blah") +) + From f280ed1560b3b95bfe7fdea0c74b6e6a012a91dc Mon Sep 17 00:00:00 2001 From: Araq Date: Tue, 8 Jan 2013 16:30:26 +0100 Subject: [PATCH 064/106] implements #295 --- compiler/docgen.nim | 22 +++++++++++++++++++++- compiler/options.nim | 4 +++- compiler/semstmts.nim | 8 ++++---- compiler/sigmatch.nim | 5 +++-- compiler/suggest.nim | 4 +++- compiler/transf.nim | 6 +++--- 6 files changed, 37 insertions(+), 12 deletions(-) diff --git a/compiler/docgen.nim b/compiler/docgen.nim index aff77cc1f..4911f3103 100755 --- a/compiler/docgen.nim +++ b/compiler/docgen.nim @@ -137,7 +137,27 @@ proc genRecComment(d: PDoc, n: PNode): PRope = if result != nil: return else: n.comment = nil - + +proc findDocComment(n: PNode): PNode = + if n == nil: return nil + if not isNil(n.comment) and startsWith(n.comment, "##"): return n + for i in countup(0, safeLen(n)-1): + result = findDocComment(n.sons[i]) + if result != nil: return + +proc extractDocComment*(s: PSym, d: PDoc = nil): string = + let n = findDocComment(s.ast) + result = "" + if not n.isNil: + if not d.isNil: + var dummyHasToc: bool + renderRstToOut(d[], parseRst(n.comment, toFilename(n.info), + toLineNumber(n.info), toColumn(n.info), + dummyHasToc, d.options + {roSkipPounds}), + result) + else: + result = n.comment.substr(2).replace("\n##", "\n").strip + proc isVisible(n: PNode): bool = result = false if n.kind == nkPostfix: diff --git a/compiler/options.nim b/compiler/options.nim index 28382f9ba..0d783cee2 100755 --- a/compiler/options.nim +++ b/compiler/options.nim @@ -1,7 +1,7 @@ # # # The Nimrod Compiler -# (c) Copyright 2012 Andreas Rumpf +# (c) Copyright 2013 Andreas Rumpf # # See the file "copying.txt", included in this # distribution, for details about the copyright. @@ -99,6 +99,8 @@ var gWholeProject*: bool # for 'doc2': output any dependency gListFullPaths*: bool +proc importantComments*(): bool {.inline.} = gCmd in {cmdDoc, cmdIdeTools} + const genSubDir* = "nimcache" NimExt* = "nim" diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index abd064a62..d78112ee5 100755 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -1,7 +1,7 @@ # # # The Nimrod Compiler -# (c) Copyright 2012 Andreas Rumpf +# (c) Copyright 2013 Andreas Rumpf # # See the file "copying.txt", included in this # distribution, for details about the copyright. @@ -247,7 +247,7 @@ proc semVarOrLet(c: PContext, n: PNode, symkind: TSymKind): PNode = if a.kind != nkVarTuple: v.typ = typ b = newNodeI(nkIdentDefs, a.info) - if gCmd == cmdDoc: + if importantComments(): # keep documentation information: b.comment = a.comment addSon(b, newSymNode(v)) @@ -287,7 +287,7 @@ proc semConst(c: PContext, n: PNode): PNode = v.ast = def # no need to copy if sfGenSym notin v.flags: addInterfaceDecl(c, v) var b = newNodeI(nkConstDef, a.info) - if gCmd == cmdDoc: b.comment = a.comment + if importantComments(): b.comment = a.comment addSon(b, newSymNode(v)) addSon(b, ast.emptyNode) # no type description addSon(b, copyTree(def)) @@ -786,7 +786,7 @@ proc semProcAux(c: PContext, n: PNode, kind: TSymKind, n.sons[pragmasPos] = proto.ast.sons[pragmasPos] if n.sons[namePos].kind != nkSym: InternalError(n.info, "semProcAux") n.sons[namePos].sym = proto - if gCmd == cmdDoc and not isNil(proto.ast.comment): + if importantComments() and not isNil(proto.ast.comment): n.comment = proto.ast.comment proto.ast = n # needed for code generation popOwner() diff --git a/compiler/sigmatch.nim b/compiler/sigmatch.nim index 2fecda427..953dcfa74 100755 --- a/compiler/sigmatch.nim +++ b/compiler/sigmatch.nim @@ -1,7 +1,7 @@ # # # The Nimrod Compiler -# (c) Copyright 2012 Andreas Rumpf +# (c) Copyright 2013 Andreas Rumpf # # See the file "copying.txt", included in this # distribution, for details about the copyright. @@ -12,7 +12,8 @@ import intsets, ast, astalgo, semdata, types, msgs, renderer, lookups, semtypinst, - magicsys, condsyms, idents, lexer, options, parampatterns + magicsys, condsyms, idents, lexer, options, parampatterns, strutils, + docgen type TCandidateState* = enum diff --git a/compiler/suggest.nim b/compiler/suggest.nim index 404f1c1bb..130666f4d 100755 --- a/compiler/suggest.nim +++ b/compiler/suggest.nim @@ -1,7 +1,7 @@ # # # The Nimrod Compiler -# (c) Copyright 2012 Andreas Rumpf +# (c) Copyright 2013 Andreas Rumpf # # See the file "copying.txt", included in this # distribution, for details about the copyright. @@ -39,6 +39,8 @@ proc SymToStr(s: PSym, isLocal: bool, section: string, li: TLineInfo): string = result.add($ToLinenumber(li)) result.add(sep) result.add($ToColumn(li)) + result.add(sep) + result.add(s.extractDocComment.escape) proc SymToStr(s: PSym, isLocal: bool, section: string): string = result = SymToStr(s, isLocal, section, s.info) diff --git a/compiler/transf.nim b/compiler/transf.nim index dfa4095b4..6633d7755 100755 --- a/compiler/transf.nim +++ b/compiler/transf.nim @@ -1,7 +1,7 @@ # # # The Nimrod Compiler -# (c) Copyright 2012 Andreas Rumpf +# (c) Copyright 2013 Andreas Rumpf # # See the file "copying.txt", included in this # distribution, for details about the copyright. @@ -150,7 +150,7 @@ proc transformVarSection(c: PTransf, v: PNode): PTransNode = newVar.owner = getCurrOwner(c) IdNodeTablePut(c.transCon.mapping, it.sons[0].sym, newSymNode(newVar)) var defs = newTransNode(nkIdentDefs, it.info, 3) - if gCmd == cmdDoc: + if importantComments(): # keep documentation information: pnode(defs).comment = it.comment defs[0] = newSymNode(newVar).PTransNode @@ -665,7 +665,7 @@ proc transform(c: PTransf, n: PNode): PTransNode = of nkIdentDefs, nkConstDef: result = transformSons(c, n) # XXX comment handling really sucks: - if gCmd == cmdDoc: + if importantComments(): pnode(result).comment = n.comment else: result = transformSons(c, n) From 20a5e37169669702868341ed6e3266372cab9eba Mon Sep 17 00:00:00 2001 From: Araq Date: Tue, 8 Jan 2013 17:23:52 +0100 Subject: [PATCH 065/106] fixes #270 --- compiler/ccgexprs.nim | 2 +- tests/compile/tobject3.nim | 28 ++++++++++++++++++++++++++++ 2 files changed, 29 insertions(+), 1 deletion(-) create mode 100644 tests/compile/tobject3.nim diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index cb0dcfe00..26144c152 100755 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -1648,7 +1648,7 @@ proc downConv(p: BProc, n: PNode, d: var TLoc) = initLocExpr(p, n.sons[0], a) var r = rdLoc(a) if skipTypes(n.sons[0].typ, abstractInst).kind in {tyRef, tyPtr, tyVar} and - n.sons[0].kind notin {nkHiddenAddr, nkAddr}: + n.sons[0].kind notin {nkHiddenAddr, nkAddr, nkObjDownConv}: app(r, "->Sup") for i in countup(2, abs(inheritanceDiff(dest, src))): app(r, ".Sup") r = con("&", r) diff --git a/tests/compile/tobject3.nim b/tests/compile/tobject3.nim new file mode 100644 index 000000000..935e6ca8c --- /dev/null +++ b/tests/compile/tobject3.nim @@ -0,0 +1,28 @@ +type + TFoo = ref object of TObject + Data: int + TBar = ref object of TFoo + nil + TBar2 = ref object of TBar + d2: int + +template super(self: TBar): TFoo = self + +template super(self: TBar2): TBar = self + +proc Foo(self: TFoo) = + echo "TFoo" + +#proc Foo(self: TBar) = +# echo "TBar" +# Foo(super(self)) +# works when this code is uncommented + +proc Foo(self: TBar2) = + echo "TBar2" + Foo(super(self)) + +var b: TBar2 +new(b) + +Foo(b) From e6fc044107c580c056069f1a3638006e83f1360e Mon Sep 17 00:00:00 2001 From: Araq Date: Tue, 8 Jan 2013 18:20:40 +0100 Subject: [PATCH 066/106] fixes #299 --- compiler/ccgcalls.nim | 4 ++-- tests/compile/tclosure4.nim | 13 +++++++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) create mode 100644 tests/compile/tclosure4.nim diff --git a/compiler/ccgcalls.nim b/compiler/ccgcalls.nim index 2673f98b8..53c371b3d 100644 --- a/compiler/ccgcalls.nim +++ b/compiler/ccgcalls.nim @@ -146,7 +146,7 @@ proc genClosureCall(p: BProc, le, ri: PNode, d: var TLoc) = proc addComma(r: PRope): PRope = result = if r == nil: r else: con(r, ", ") - const CallPattern = "$1.ClEnv? $1.ClPrc($3$1.ClEnv) : (($4)($1.ClPrc))($2);$n" + const CallPattern = "$1.ClEnv? $1.ClPrc($3$1.ClEnv) : (($4)($1.ClPrc))($2)" var op: TLoc initLocExpr(p, ri.sons[0], op) var pl: PRope @@ -164,7 +164,7 @@ proc genClosureCall(p: BProc, le, ri: PNode, d: var TLoc) = if i < length - 1: app(pl, ", ") template genCallPattern = - lineF(p, cpsStmts, CallPattern, op.r, pl, pl.addComma, rawProc) + lineF(p, cpsStmts, CallPattern & ";$n", op.r, pl, pl.addComma, rawProc) let rawProc = getRawProcType(p, typ) if typ.sons[0] != nil: diff --git a/tests/compile/tclosure4.nim b/tests/compile/tclosure4.nim new file mode 100644 index 000000000..9584fa98a --- /dev/null +++ b/tests/compile/tclosure4.nim @@ -0,0 +1,13 @@ + +import json, tables + +proc run(json_params: TTable) = + let json_elems = json_params["files"].elems + # These fail compilation. + var files = each(json_elems, proc (x: PJsonNode): string = x.str) + #var files = json_elems.each do (x: PJsonNode) -> string: x.str + echo "Hey!" + +when isMainModule: + let text = """{"files": ["a", "b", "c"]}""" + run(toTable((text.parseJson).fields)) From 3af5c99336c2454aee1d853fa7fcbbf3ce16d568 Mon Sep 17 00:00:00 2001 From: Araq Date: Tue, 8 Jan 2013 20:03:02 +0100 Subject: [PATCH 067/106] fixes #293 --- compiler/ast.nim | 2 +- compiler/evals.nim | 4 ++-- compiler/msgs.nim | 2 +- compiler/renderer.nim | 7 +++---- compiler/semstmts.nim | 25 +++++++++++++++++++++---- compiler/semthreads.nim | 2 +- lib/pure/collections/tables.nim | 2 +- lib/pure/osproc.nim | 8 ++++---- lib/system/ansi_c.nim | 2 +- todo.txt | 6 +++--- 10 files changed, 38 insertions(+), 22 deletions(-) diff --git a/compiler/ast.nim b/compiler/ast.nim index 82a20c312..fc21235bc 100755 --- a/compiler/ast.nim +++ b/compiler/ast.nim @@ -1,7 +1,7 @@ # # # The Nimrod Compiler -# (c) Copyright 2012 Andreas Rumpf +# (c) Copyright 2013 Andreas Rumpf # # See the file "copying.txt", included in this # distribution, for details about the copyright. diff --git a/compiler/evals.nim b/compiler/evals.nim index 5623f7955..4b42f5ada 100755 --- a/compiler/evals.nim +++ b/compiler/evals.nim @@ -1,7 +1,7 @@ # # # The Nimrod Compiler -# (c) Copyright 2012 Andreas Rumpf +# (c) Copyright 2013 Andreas Rumpf # # See the file "copying.txt", included in this # distribution, for details about the copyright. @@ -319,7 +319,7 @@ proc evalVar(c: PEvalContext, n: PNode): PNode = else: if x.kind notin {nkEmpty..nkNilLit}: discardSons(x) - for i in countup(0, sonsLen(result) - 1): addSon(x, result.sons[i]) + for j in countup(0, sonsLen(result) - 1): addSon(x, result.sons[j]) result = emptyNode proc aliasNeeded(n: PNode, flags: TEvalFlags): bool = diff --git a/compiler/msgs.nim b/compiler/msgs.nim index 16603a9a8..9858b1b9e 100755 --- a/compiler/msgs.nim +++ b/compiler/msgs.nim @@ -1,7 +1,7 @@ # # # The Nimrod Compiler -# (c) Copyright 2012 Andreas Rumpf +# (c) Copyright 2013 Andreas Rumpf # # See the file "copying.txt", included in this # distribution, for details about the copyright. diff --git a/compiler/renderer.nim b/compiler/renderer.nim index c8ee66e03..48a190ec1 100755 --- a/compiler/renderer.nim +++ b/compiler/renderer.nim @@ -1,7 +1,7 @@ # # # The Nimrod Compiler -# (c) Copyright 2012 Andreas Rumpf +# (c) Copyright 2013 Andreas Rumpf # # See the file "copying.txt", included in this # distribution, for details about the copyright. @@ -753,8 +753,7 @@ proc doParamsAux(g: var TSrcGen, params: PNode) = proc gsub(g: var TSrcGen, n: PNode, c: TContext) = if isNil(n): return - var - L: int + var a: TContext if n.comment != nil: pushCom(g, n) case n.kind # atoms: @@ -1096,7 +1095,7 @@ proc gsub(g: var TSrcGen, n: PNode, c: TContext) = incl(a.flags, rfInConstExpr) gsection(g, n, a, tkConst, "const") of nkVarSection, nkLetSection: - L = sonsLen(n) + var L = sonsLen(n) if L == 0: return if n.kind == nkVarSection: putWithSpace(g, tkVar, "var") else: putWithSpace(g, tkLet, "let") diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index d78112ee5..c86e3eb91 100755 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -172,11 +172,16 @@ proc fitRemoveHiddenConv(c: PContext, typ: Ptype, n: PNode): PNode = changeType(result, typ) proc findShadowedVar(c: PContext, v: PSym): PSym = - for i in countdown(c.tab.tos - 2, 0): + for i in countdown(c.tab.tos - 2, ModuleTablePos+1): let shadowed = StrTableGet(c.tab.stack[i], v.name) if shadowed != nil and shadowed.kind in skLocalVars: return shadowed +proc identWithin(n: PNode, s: PIdent): bool = + for i in 0 .. n.safeLen-1: + if identWithin(n.sons[i], s): return true + result = n.kind == nkSym and n.sym.name.id == s.id + proc semIdentDef(c: PContext, n: PNode, kind: TSymKind): PSym = if isTopLevel(c): result = semIdentWithPragma(c, kind, n, {sfExported}) @@ -239,7 +244,10 @@ proc semVarOrLet(c: PContext, n: PNode, symkind: TSymKind): PNode = let shadowed = findShadowedVar(c, v) if shadowed != nil: shadowed.flags.incl(sfShadowed) - Message(a.info, warnShadowIdent, v.name.s) + # a shadowed variable is an error unless it appears on the right + # side of the '=': + if warnShadowIdent in gNotes and not identWithin(def, v.name): + Message(a.info, warnShadowIdent, v.name.s) if def != nil and def.kind != nkEmpty: # this is only needed for the evaluation pass: v.ast = def @@ -368,6 +376,15 @@ proc semForFields(c: PContext, n: PNode, m: TMagic): PNode = b.add(ast.emptyNode) stmts.add(b) +proc addForVarDecl(c: PContext, v: PSym) = + if warnShadowIdent in gNotes: + let shadowed = findShadowedVar(c, v) + if shadowed != nil: + # XXX should we do this here? + #shadowed.flags.incl(sfShadowed) + Message(v.info, warnShadowIdent, v.name.s) + addDecl(c, v) + proc semForVars(c: PContext, n: PNode): PNode = result = n var length = sonsLen(n) @@ -383,7 +400,7 @@ proc semForVars(c: PContext, n: PNode): PNode = # for an example: v.typ = n.sons[length-2].typ n.sons[0] = newSymNode(v) - if sfGenSym notin v.flags: addDecl(c, v) + if sfGenSym notin v.flags: addForVarDecl(c, v) else: LocalError(n.info, errWrongNumberOfVariables) elif length-2 != sonsLen(iter): @@ -394,7 +411,7 @@ proc semForVars(c: PContext, n: PNode): PNode = if getCurrOwner().kind == skModule: incl(v.flags, sfGlobal) v.typ = iter.sons[i] n.sons[i] = newSymNode(v) - if sfGenSym notin v.flags: addDecl(c, v) + if sfGenSym notin v.flags: addForVarDecl(c, v) Inc(c.p.nestedLoopCounter) n.sons[length-1] = SemStmt(c, n.sons[length-1]) Dec(c.p.nestedLoopCounter) diff --git a/compiler/semthreads.nim b/compiler/semthreads.nim index 75621be79..9fc6a54d9 100755 --- a/compiler/semthreads.nim +++ b/compiler/semthreads.nim @@ -1,7 +1,7 @@ # # # The Nimrod Compiler -# (c) Copyright 2012 Andreas Rumpf +# (c) Copyright 2013 Andreas Rumpf # # See the file "copying.txt", included in this # distribution, for details about the copyright. diff --git a/lib/pure/collections/tables.nim b/lib/pure/collections/tables.nim index 3b252f1d2..fc394d1f8 100755 --- a/lib/pure/collections/tables.nim +++ b/lib/pure/collections/tables.nim @@ -1,7 +1,7 @@ # # # Nimrod's Runtime Library -# (c) Copyright 2012 Andreas Rumpf +# (c) Copyright 2013 Andreas Rumpf # # See the file "copying.txt", included in this # distribution, for details about the copyright. diff --git a/lib/pure/osproc.nim b/lib/pure/osproc.nim index a8c0ea809..ecf38f9f6 100755 --- a/lib/pure/osproc.nim +++ b/lib/pure/osproc.nim @@ -1,7 +1,7 @@ # # # Nimrod's Runtime Library -# (c) Copyright 2012 Andreas Rumpf +# (c) Copyright 2013 Andreas Rumpf # # See the file "copying.txt", included in this # distribution, for details about the copyright. @@ -201,9 +201,9 @@ proc execProcesses*(cmds: openArray[string], q[r] = startCmd(cmds[i], options=options) inc(i) if i > high(cmds): break - for i in 0..m-1: - if q[i] != nil: close(q[i]) - result = max(waitForExit(q[i]), result) + for j in 0..m-1: + if q[j] != nil: close(q[j]) + result = max(waitForExit(q[j]), result) else: for i in 0..high(cmds): var p = startCmd(cmds[i], options=options) diff --git a/lib/system/ansi_c.nim b/lib/system/ansi_c.nim index 195bc2e60..f89577553 100755 --- a/lib/system/ansi_c.nim +++ b/lib/system/ansi_c.nim @@ -1,7 +1,7 @@ # # # Nimrod's Runtime Library -# (c) Copyright 2012 Andreas Rumpf +# (c) Copyright 2013 Andreas Rumpf # # See the file "copying.txt", included in this # distribution, for details about the copyright. diff --git a/todo.txt b/todo.txt index 0b6457efc..dd3290117 100755 --- a/todo.txt +++ b/todo.txt @@ -1,9 +1,6 @@ version 0.9.2 ============= -- FFI: - * test libffi on windows - * test: times.format with the FFI - fix closure bug finally - fix marshal bug - investigate nimgame bug @@ -12,6 +9,9 @@ version 0.9.2 version 0.9.X ============= +- FFI: + * test libffi on windows + * test: times.format with the FFI - test&finish first class iterators: * nested iterators - implement the missing features wrt inheritance From 5a9637f2f34a2fa93675ad9ded3dc6cd865eba67 Mon Sep 17 00:00:00 2001 From: Araq Date: Tue, 8 Jan 2013 22:51:23 +0100 Subject: [PATCH 068/106] make some tests green --- lib/pure/sockets.nim | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/pure/sockets.nim b/lib/pure/sockets.nim index 86f06324a..f233e53c8 100755 --- a/lib/pure/sockets.nim +++ b/lib/pure/sockets.nim @@ -1,7 +1,7 @@ # # # Nimrod's Runtime Library -# (c) Copyright 2012 Andreas Rumpf +# (c) Copyright 2013 Andreas Rumpf # # See the file "copying.txt", included in this # distribution, for details about the copyright. @@ -1274,6 +1274,7 @@ proc recv*(socket: TSocket): TaintedString {.tags: [FReadIO], deprecated.} = add(result.string, buf) if bytesRead != bufSize-1: break +{.push warning[deprecated]: off.} proc recvTimeout*(socket: TSocket, timeout: int): TaintedString {. tags: [FReadIO], deprecated.} = ## overloaded variant to support a ``timeout`` parameter, the ``timeout`` @@ -1287,6 +1288,7 @@ proc recvTimeout*(socket: TSocket, timeout: int): TaintedString {. raise newException(ETimeout, "Call to recv() timed out.") return socket.recv +{.pop.} proc recvAsync*(socket: TSocket, s: var TaintedString): bool {. tags: [FReadIO], deprecated.} = From 86b19c44eba8a1572feaeb47f039e5da5478720a Mon Sep 17 00:00:00 2001 From: Araq Date: Wed, 9 Jan 2013 00:49:19 +0100 Subject: [PATCH 069/106] allow 'cast' for the JS target --- compiler/ecmasgen.nim | 1 + compiler/main.nim | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/compiler/ecmasgen.nim b/compiler/ecmasgen.nim index 770e6a4a7..25d147ebc 100755 --- a/compiler/ecmasgen.nim +++ b/compiler/ecmasgen.nim @@ -1529,6 +1529,7 @@ proc gen(p: var TProc, n: PNode, r: var TCompRes) = of nkCheckedFieldExpr: genCheckedFieldAccess(p, n, r) of nkObjDownConv: gen(p, n.sons[0], r) of nkObjUpConv: upConv(p, n, r) + of nkCast: gen(p, n.sons[1], r) of nkChckRangeF: genRangeChck(p, n, r, "chckRangeF") of nkChckRange64: genRangeChck(p, n, r, "chckRange64") of nkChckRange: genRangeChck(p, n, r, "chckRange") diff --git a/compiler/main.nim b/compiler/main.nim index 517bceb3c..0a76b967a 100755 --- a/compiler/main.nim +++ b/compiler/main.nim @@ -142,7 +142,7 @@ when has_LLVM_Backend: compileProject() proc CommandCompileToEcmaScript = - incl(gGlobalOptions, optSafeCode) + #incl(gGlobalOptions, optSafeCode) setTarget(osEcmaScript, cpuEcmaScript) #initDefines() DefineSymbol("nimrod") # 'nimrod' is always defined From c9690864d4cf9945cb5eb9497f7621e56f9bebd0 Mon Sep 17 00:00:00 2001 From: Araq Date: Wed, 9 Jan 2013 08:44:20 +0100 Subject: [PATCH 070/106] optimized ropes work now but no speedup --- compiler/ropes.nim | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/compiler/ropes.nim b/compiler/ropes.nim index f27bf70f7..95dce8278 100755 --- a/compiler/ropes.nim +++ b/compiler/ropes.nim @@ -1,7 +1,7 @@ # # # The Nimrod Compiler -# (c) Copyright 2012 Andreas Rumpf +# (c) Copyright 2013 Andreas Rumpf # # See the file "copying.txt", included in this # distribution, for details about the copyright. @@ -68,7 +68,7 @@ type # though it is not necessary) when true: - # working version: + # old version: type PRope* = ref TRope TRope*{.acyclic.} = object of TObject # the empty rope is represented @@ -330,7 +330,7 @@ when true: result = false else: - # optimized but broken version: + # optimized version with 'unsafeNew': type PRope* = ref TRope @@ -480,7 +480,9 @@ else: it = it.left assert(it != nil) assert(it.L < 0) - write(f, cstring(it.d), -it.L) + let L = -it.L + if writeBuffer(f, cstring(it.d), L) != L: + InternalError("cannot store file") proc WriteRope*(head: PRope, filename: string, useWarning = false) = var f: tfile From 6e65f5b49e7483bb7c74f70985022927aee61bdf Mon Sep 17 00:00:00 2001 From: Grzegorz Adam Hankiewicz Date: Sun, 6 Jan 2013 13:31:19 +0100 Subject: [PATCH 071/106] Hyperlinks each proc and explains it is like map. --- lib/pure/collections/sequtils.nim | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/pure/collections/sequtils.nim b/lib/pure/collections/sequtils.nim index 0fda5700e..f4feedac7 100644 --- a/lib/pure/collections/sequtils.nim +++ b/lib/pure/collections/sequtils.nim @@ -9,8 +9,11 @@ ## :Author: Alex Mitchell ## -## This module implements operations for the built-in `seq`:idx: type -## which were inspired by functional programming languages. +## This module implements operations for the built-in `seq`:idx: type which +## were inspired by functional programming languages. If you are looking for +## the typical `map` function which applies a function to every element in a +## sequence, it already exists as the `each` proc in the `system +## `_ module in both mutable and immutable styles. ## ## **Note**: This interface will change as soon as the compiler supports ## closures and proper coroutines. From 0b078e81d01f7ee3ab5d886a84d70c242228ef9b Mon Sep 17 00:00:00 2001 From: Grzegorz Adam Hankiewicz Date: Sun, 6 Jan 2013 13:31:46 +0100 Subject: [PATCH 072/106] Adds each's map alias to index. --- lib/system.nim | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/system.nim b/lib/system.nim index 3d17b1d6d..26e5ac228 100755 --- a/lib/system.nim +++ b/lib/system.nim @@ -1461,14 +1461,14 @@ proc pop*[T](s: var seq[T]): T {.inline, noSideEffect.} = setLen(s, L) proc each*[T, S](data: openArray[T], op: proc (x: T): S {.closure.}): seq[S] = - ## The well-known ``map`` operation from functional programming. Applies + ## The well-known `map`:idx: operation from functional programming. Applies ## `op` to every item in `data` and returns the result as a sequence. newSeq(result, data.len) for i in 0..data.len-1: result[i] = op(data[i]) proc each*[T](data: var openArray[T], op: proc (x: var T) {.closure.}) = - ## The well-known ``map`` operation from functional programming. Applies - ## `op` to every item in `data`. + ## The well-known `map`:idx: operation from functional programming. Applies + ## `op` to every item in `data` modifying it directly. for i in 0..data.len-1: op(data[i]) iterator fields*[T: tuple](x: T): TObject {. From 1757baecaff7b1fbe445cf47a2615e36aa0c4a0c Mon Sep 17 00:00:00 2001 From: Grzegorz Adam Hankiewicz Date: Mon, 7 Jan 2013 00:01:01 +0100 Subject: [PATCH 073/106] Adds documentation examples to sequtils. --- lib/pure/collections/sequtils.nim | 153 ++++++++++++++++++++++++++++-- 1 file changed, 145 insertions(+), 8 deletions(-) diff --git a/lib/pure/collections/sequtils.nim b/lib/pure/collections/sequtils.nim index f4feedac7..34858b548 100644 --- a/lib/pure/collections/sequtils.nim +++ b/lib/pure/collections/sequtils.nim @@ -15,6 +15,11 @@ ## sequence, it already exists as the `each` proc in the `system ## `_ module in both mutable and immutable styles. ## +## Also, for functional style programming you may want to pass `anonymous procs +## `_ to procs like ``filter`` to reduce typing. +## Anonymous procs can use `the special do notation `_ +## which is more convenient in certain situations. +## ## **Note**: This interface will change as soon as the compiler supports ## closures and proper coroutines. @@ -22,7 +27,17 @@ when not defined(nimhygiene): {.pragma: dirty.} proc concat*[T](seqs: varargs[seq[T]]): seq[T] = - ## Takes several sequences' items and returns them inside of one sequence. + ## Takes several sequences' items and returns them inside a new sequence. + ## + ## Example: + ## + ## .. code-block:: nimrod + ## let + ## s1 = @[1, 2, 3] + ## s2 = @[4, 5] + ## s3 = @[6, 7] + ## total = concat(s1, s2, s3) + ## assert total == @[1, 2, 3, 4, 5, 6, 7] var L = 0 for seqitm in items(seqs): inc(L, len(seqitm)) newSeq(result, L) @@ -33,14 +48,38 @@ proc concat*[T](seqs: varargs[seq[T]]): seq[T] = inc(i) proc distnct*[T](seq1: seq[T]): seq[T] = - ## Removes duplicates from a sequence and returns it. + ## Returns a new sequence without duplicates. Example: + ## + ## .. code-block:: nimrod + ## let + ## dup1 = @[1, 1, 3, 4, 2, 2, 8, 1, 4] + ## dup2 = @["a", "a", "c", "d", "d"] + ## unique1 = distnct(dup1) + ## unique2 = distnct(dup2) + ## assert unique1 == @[1, 3, 4, 2, 8] + ## assert unique2 == @["a", "c", "d"] result = @[] for itm in items(seq1): if not result.contains(itm): result.add(itm) proc zip*[S, T](seq1: seq[S], seq2: seq[T]): seq[tuple[a: S, b: T]] = - ## Combines two sequences. If one sequence is too short, - ## the remaining items in the longer sequence are discarded. + ## Returns a new sequence with a combination of the two input sequences. + ## + ## For convenience you can access the returned tuples through the named + ## fields `a` and `b`. If one sequence is shorter, the remaining items in the + ## longer sequence are discarded. Example: + ## + ## .. code-block:: nimrod + ## let + ## short = @[1, 2, 3] + ## long = @[6, 5, 4, 3, 2, 1] + ## words = @["one", "two", "three"] + ## zip1 = zip(short, long) + ## zip2 = zip(short, words) + ## assert zip1 == @[(1, 6), (2, 5), (3, 4)] + ## assert zip2 == @[(1, "one"), (2, "two"), (3, "three")] + ## assert zip1[2].b == 4 + ## assert zip2[2].b == "three" var m = min(seq1.len, seq2.len) newSeq(result, m) for i in 0 .. m-1: result[i] = (seq1[i], seq2[i]) @@ -48,18 +87,44 @@ proc zip*[S, T](seq1: seq[S], seq2: seq[T]): seq[tuple[a: S, b: T]] = iterator filter*[T](seq1: seq[T], pred: proc(item: T): bool {.closure.}): T = ## Iterates through a sequence and yields every item that fulfills the ## predicate. + ## + ## Example: + ## + ## .. code-block:: nimrod + ## let numbers = @[1, 4, 5, 8, 9, 7, 4] + ## for n in filter(numbers, proc (x: int): bool = x mod 2 == 0): + ## echo($n) + ## # echoes 4, 8, 4 in separate lines for i in countup(0, len(seq1) -1): var item = seq1[i] if pred(item): yield seq1[i] proc filter*[T](seq1: seq[T], pred: proc(item: T): bool {.closure.}): seq[T] = - ## Returns all items in a sequence that fulfilled the predicate. + ## Returns a new sequence with all the items that fulfilled the predicate. + ## + ## Example: + ## + ## .. code-block:: nimrod + ## let + ## colors = @["red", "yellow", "black"] + ## f1 = filter(colors, proc(x: string): bool = x.len < 6) + ## f2 = filter(colors) do (x: string) -> bool : x.len > 5 + ## assert f1 == @["red", "black"] + ## assert f2 == @["yellow"] accumulateResult(filter(seq1, pred)) template filterIt*(seq1, pred: expr): expr {.immediate, dirty.} = - ## Finds a specific item in a sequence as long as the - ## predicate returns true. The predicate needs to be an expression - ## containing ``it``: ``filterIt("abcxyz", it == 'x')``. + ## Returns a new sequence with all the items that fulfilled the predicate. + ## + ## Unlike the `proc` version, the predicate needs to be an expression using + ## the ``it`` variable for testing, like: ``filterIt("abcxyz", it == 'x')``. + ## Example: + ## + ## .. code-block:: nimrod + ## let + ## temperatures = @[-272.15, -2.0, 24.5, 44.31, 99.9, -113.44] + ## acceptable = filterIt(temperatures, it < 50 and it > -10) + ## assert acceptable == @[-2.0, 24.5, 44.31] block: var result: type(seq1) = @[] for it in items(seq1): @@ -68,7 +133,79 @@ template filterIt*(seq1, pred: expr): expr {.immediate, dirty.} = template toSeq*(iter: expr): expr {.immediate.} = ## Transforms any iterator into a sequence. + ## + ## Example: + ## + ## .. code-block:: nimrod + ## let + ## numeric = @[1, 2, 3, 4, 5, 6, 7, 8, 9] + ## odd_numbers = toSeq(filter(numeric) do (x: int) -> bool: + ## if x mod 2 == 1: + ## result = true) + ## assert odd_numbers == @[1, 3, 5, 7, 9] + ## var result {.gensym.}: seq[type(iter)] = @[] for x in iter: add(result, x) result +when isMainModule: + import strutils + proc toStr(x: int): string {.procvar.} = $x + # concat test + let + s1 = @[1, 2, 3] + s2 = @[4, 5] + s3 = @[6, 7] + total = concat(s1, s2, s3) + assert total == @[1, 2, 3, 4, 5, 6, 7] + + # duplicates test + let + dup1 = @[1, 1, 3, 4, 2, 2, 8, 1, 4] + dup2 = @["a", "a", "c", "d", "d"] + unique1 = distnct(dup1) + unique2 = distnct(dup2) + assert unique1 == @[1, 3, 4, 2, 8] + assert unique2 == @["a", "c", "d"] + + # zip test + let + short = @[1, 2, 3] + long = @[6, 5, 4, 3, 2, 1] + words = @["one", "two", "three"] + zip1 = zip(short, long) + zip2 = zip(short, words) + assert zip1 == @[(1, 6), (2, 5), (3, 4)] + assert zip2 == @[(1, "one"), (2, "two"), (3, "three")] + assert zip1[2].b == 4 + assert zip2[2].b == "three" + + # filter proc test + let + colors = @["red", "yellow", "black"] + f1 = filter(colors, proc(x: string): bool = x.len < 6) + f2 = filter(colors) do (x: string) -> bool : x.len > 5 + assert f1 == @["red", "black"] + assert f2 == @["yellow"] + + # filter iterator test + let numbers = @[1, 4, 5, 8, 9, 7, 4] + for n in filter(numbers, proc (x: int): bool = x mod 2 == 0): + echo($n) + # echoes 4, 8, 4 in separate lines + + # filterIt test + let + temperatures = @[-272.15, -2.0, 24.5, 44.31, 99.9, -113.44] + acceptable = filterIt(temperatures, it < 50 and it > -10) + assert acceptable == @[-2.0, 24.5, 44.31] + + # toSeq test + let + numeric = @[1, 2, 3, 4, 5, 6, 7, 8, 9] + odd_numbers = toSeq(filter(numeric) do (x: int) -> bool: + if x mod 2 == 1: + result = true) + assert odd_numbers == @[1, 3, 5, 7, 9] + + echo "Finished doc tests" From abca30cc067348cc0d4cf6e62e8fe2a70de0a9ae Mon Sep 17 00:00:00 2001 From: Grzegorz Adam Hankiewicz Date: Wed, 9 Jan 2013 17:54:01 +0100 Subject: [PATCH 074/106] Adds note about distnct being misspelled on purpose. --- lib/pure/collections/sequtils.nim | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/pure/collections/sequtils.nim b/lib/pure/collections/sequtils.nim index 34858b548..e4ea22830 100644 --- a/lib/pure/collections/sequtils.nim +++ b/lib/pure/collections/sequtils.nim @@ -48,7 +48,11 @@ proc concat*[T](seqs: varargs[seq[T]]): seq[T] = inc(i) proc distnct*[T](seq1: seq[T]): seq[T] = - ## Returns a new sequence without duplicates. Example: + ## Returns a new sequence without duplicates. + ## + ## This proc is `misspelled` on purpose to avoid a clash with the keyword + ## ``distinct`` used to `define a derived type incompatible with its base + ## type `_. Example: ## ## .. code-block:: nimrod ## let From 4670445a935e2555f4c5c41f810b606fe5abaa0d Mon Sep 17 00:00:00 2001 From: Araq Date: Fri, 11 Jan 2013 01:08:37 +0100 Subject: [PATCH 075/106] 'importcpp' for the JS target to generate an infix call --- compiler/ecmasgen.nim | 44 +++++++++++++++++++++++++-------- lib/pure/collections/tables.nim | 2 +- 2 files changed, 35 insertions(+), 11 deletions(-) diff --git a/compiler/ecmasgen.nim b/compiler/ecmasgen.nim index 25d147ebc..8899c904e 100755 --- a/compiler/ecmasgen.nim +++ b/compiler/ecmasgen.nim @@ -1,7 +1,7 @@ # # # The Nimrod Compiler -# (c) Copyright 2012 Andreas Rumpf +# (c) Copyright 2013 Andreas Rumpf # # See the file "copying.txt", included in this # distribution, for details about the copyright. @@ -964,24 +964,45 @@ proc genDeref(p: var TProc, n: PNode, r: var TCompRes) = if a.kind != etyBaseIndex: InternalError(n.info, "genDeref") r.res = ropef("$1[$2]", [a.com, a.res]) +proc genArg(p: var TProc, n: PNode, r: var TCompRes) = + var a: TCompRes + gen(p, n, a) + if a.kind == etyBaseIndex: + app(r.res, a.com) + app(r.res, ", ") + app(r.res, a.res) + else: + app(r.res, mergeExpr(a)) + proc genArgs(p: var TProc, n: PNode, r: var TCompRes) = app(r.res, "(") for i in countup(1, sonsLen(n) - 1): if i > 1: app(r.res, ", ") - var a: TCompRes - gen(p, n.sons[i], a) - if a.kind == etyBaseIndex: - app(r.res, a.com) - app(r.res, ", ") - app(r.res, a.res) - else: - app(r.res, mergeExpr(a)) + genArg(p, n.sons[i], r) app(r.res, ")") proc genCall(p: var TProc, n: PNode, r: var TCompRes) = gen(p, n.sons[0], r) genArgs(p, n, r) +proc genInfixCall(p: var TProc, n: PNode, r: var TCompRes) = + gen(p, n.sons[1], r) + if r.kind == etyBaseIndex: + if r.com == nil: + GlobalError(n.info, "cannot invoke with infix syntax") + r.res = ropef("$1[0]", [r.res, r.com]) + r.com = nil + app(r.res, ".") + var op: TCompRes + gen(p, n.sons[0], op) + app(r.res, mergeExpr(op)) + + app(r.res, "(") + for i in countup(2, sonsLen(n) - 1): + if i > 2: app(r.res, ", ") + genArg(p, n.sons[i], r) + app(r.res, ")") + proc genEcho(p: var TProc, n: PNode, r: var TCompRes) = useMagic(p, "rawEcho") app(r.res, "rawEcho") @@ -1513,9 +1534,12 @@ proc gen(p: var TProc, n: PNode, r: var TCompRes) = else: r.res = toRope(f.ToStrMaxPrecision) of nkBlockExpr: genBlock(p, n, r) of nkIfExpr: genIfExpr(p, n, r) - of nkCallKinds: + of nkCallKinds: if (n.sons[0].kind == nkSym) and (n.sons[0].sym.magic != mNone): genMagic(p, n, r) + elif n.sons[0].kind == nkSym and sfInfixCall in n.sons[0].sym.flags and + n.len >= 2: + genInfixCall(p, n, r) else: genCall(p, n, r) of nkCurly: genSetConstr(p, n, r) diff --git a/lib/pure/collections/tables.nim b/lib/pure/collections/tables.nim index fc394d1f8..3b252f1d2 100755 --- a/lib/pure/collections/tables.nim +++ b/lib/pure/collections/tables.nim @@ -1,7 +1,7 @@ # # # Nimrod's Runtime Library -# (c) Copyright 2013 Andreas Rumpf +# (c) Copyright 2012 Andreas Rumpf # # See the file "copying.txt", included in this # distribution, for details about the copyright. From 1115c8bdf8204cdf58fd9a488e8b559b6f4be16b Mon Sep 17 00:00:00 2001 From: Araq Date: Sat, 12 Jan 2013 17:23:16 +0100 Subject: [PATCH 076/106] old version of ropes.nim to prevent merge conflicts --- compiler/ropes.nim | 742 ++++++++++++++------------------------------- 1 file changed, 236 insertions(+), 506 deletions(-) diff --git a/compiler/ropes.nim b/compiler/ropes.nim index f27bf70f7..50c89e4d9 100755 --- a/compiler/ropes.nim +++ b/compiler/ropes.nim @@ -58,536 +58,266 @@ import msgs, strutils, platform, hashes, crc, options -const - PayloadSize = 64_000 # dummy size for range checking - type TFormatStr* = string # later we may change it to CString for better # performance of the code generator (assignments # copy the format strings # though it is not necessary) + PRope* = ref TRope + TRope*{.acyclic.} = object of TObject # the empty rope is represented + # by nil to safe space + left*, right*: PRope + length*: int + data*: string # != nil if a leaf + + TRopeSeq* = seq[PRope] -when true: - # working version: - type - PRope* = ref TRope - TRope*{.acyclic.} = object of TObject # the empty rope is represented - # by nil to safe space - left*, right*: PRope - length*: int - data*: string # != nil if a leaf - - TRopeSeq* = seq[PRope] +proc con*(a, b: PRope): PRope +proc con*(a: PRope, b: string): PRope +proc con*(a: string, b: PRope): PRope +proc con*(a: varargs[PRope]): PRope +proc app*(a: var PRope, b: PRope) +proc app*(a: var PRope, b: string) +proc prepend*(a: var PRope, b: PRope) +proc toRope*(s: string): PRope +proc toRope*(i: BiggestInt): PRope +proc ropeLen*(a: PRope): int +proc writeRopeIfNotEqual*(r: PRope, filename: string): bool +proc ropeToStr*(p: PRope): string +proc ropef*(frmt: TFormatStr, args: varargs[PRope]): PRope +proc appf*(c: var PRope, frmt: TFormatStr, args: varargs[PRope]) +proc RopeEqualsFile*(r: PRope, f: string): bool + # returns true if the rope r is the same as the contents of file f +proc RopeInvariant*(r: PRope): bool + # exported for debugging +# implementation - proc con*(a, b: PRope): PRope - proc con*(a: PRope, b: string): PRope - proc con*(a: string, b: PRope): PRope - proc con*(a: varargs[PRope]): PRope - proc app*(a: var PRope, b: PRope) - proc app*(a: var PRope, b: string) - proc prepend*(a: var PRope, b: PRope) - proc toRope*(s: string): PRope - proc toRope*(i: BiggestInt): PRope - proc ropeLen*(a: PRope): int - proc writeRopeIfNotEqual*(r: PRope, filename: string): bool - proc ropeToStr*(p: PRope): string - proc ropef*(frmt: TFormatStr, args: varargs[PRope]): PRope - proc appf*(c: var PRope, frmt: TFormatStr, args: varargs[PRope]) - proc RopeEqualsFile*(r: PRope, f: string): bool - # returns true if the rope r is the same as the contents of file f - proc RopeInvariant*(r: PRope): bool - # exported for debugging - # implementation +proc ropeLen(a: PRope): int = + if a == nil: result = 0 + else: result = a.length + +proc newRope(data: string = nil): PRope = + new(result) + if data != nil: + result.length = len(data) + result.data = data - proc ropeLen(a: PRope): int = - if a == nil: result = 0 - else: result = a.length - - proc newRope(data: string = nil): PRope = - new(result) - if data != nil: - result.length = len(data) - result.data = data +proc newMutableRope*(capacity = 30): PRope = + ## creates a new rope that supports direct modifications of the rope's + ## 'data' and 'length' fields. + new(result) + result.data = newStringOfCap(capacity) - proc newMutableRope*(capacity = 30): PRope = - ## creates a new rope that supports direct modifications of the rope's - ## 'data' and 'length' fields. - new(result) - result.data = newStringOfCap(capacity) +proc freezeMutableRope*(r: PRope) {.inline.} = + r.length = r.data.len - proc freezeMutableRope*(r: PRope) {.inline.} = - r.length = r.data.len +var + cache: array[0..2048*2 -1, PRope] - var - cache: array[0..2048*2 -1, PRope] +proc RopeInvariant(r: PRope): bool = + if r == nil: + result = true + else: + result = true # + # if r.data <> snil then + # result := true + # else begin + # result := (r.left <> nil) and (r.right <> nil); + # if result then result := ropeInvariant(r.left); + # if result then result := ropeInvariant(r.right); + # end - proc RopeInvariant(r: PRope): bool = - if r == nil: - result = true - else: - result = true # - # if r.data <> snil then - # result := true - # else begin - # result := (r.left <> nil) and (r.right <> nil); - # if result then result := ropeInvariant(r.left); - # if result then result := ropeInvariant(r.right); - # end - - proc insertInCache(s: string): PRope = - var h = hash(s) and high(cache) - result = cache[h] - if isNil(result) or result.data != s: - result = newRope(s) - cache[h] = result - - proc toRope(s: string): PRope = - if s.len == 0: - result = nil - else: - result = insertInCache(s) - assert(RopeInvariant(result)) - - proc RopeSeqInsert(rs: var TRopeSeq, r: PRope, at: Natural) = - var length = len(rs) - if at > length: - setlen(rs, at + 1) - else: - setlen(rs, length + 1) # move old rope elements: - for i in countdown(length, at + 1): - rs[i] = rs[i - 1] # this is correct, I used pen and paper to validate it - rs[at] = r - - proc newRecRopeToStr(result: var string, resultLen: var int, r: PRope) = - var stack = @[r] - while len(stack) > 0: - var it = pop(stack) - while it.data == nil: - add(stack, it.right) - it = it.left - assert(it.data != nil) - CopyMem(addr(result[resultLen]), addr(it.data[0]), it.length) - Inc(resultLen, it.length) - assert(resultLen <= len(result)) - - proc ropeToStr(p: PRope): string = - if p == nil: - result = "" - else: - result = newString(p.length) - var resultLen = 0 - newRecRopeToStr(result, resultLen, p) - - proc con(a, b: PRope): PRope = - if a == nil: result = b - elif b == nil: result = a - else: - result = newRope() - result.length = a.length + b.length - result.left = a - result.right = b - - proc con(a: PRope, b: string): PRope = result = con(a, toRope(b)) - proc con(a: string, b: PRope): PRope = result = con(toRope(a), b) - - proc con(a: varargs[PRope]): PRope = - for i in countup(0, high(a)): result = con(result, a[i]) - - proc toRope(i: BiggestInt): PRope = result = toRope($i) - - proc app(a: var PRope, b: PRope) = a = con(a, b) - proc app(a: var PRope, b: string) = a = con(a, b) - proc prepend(a: var PRope, b: PRope) = a = con(b, a) - - proc writeRope*(f: TFile, c: PRope) = - var stack = @[c] - while len(stack) > 0: - var it = pop(stack) - while it.data == nil: - add(stack, it.right) - it = it.left - assert(it != nil) - assert(it.data != nil) - write(f, it.data) - - proc WriteRope*(head: PRope, filename: string, useWarning = false) = - var f: tfile - if open(f, filename, fmWrite): - if head != nil: WriteRope(f, head) - close(f) - else: - rawMessage(if useWarning: warnCannotOpenFile else: errCannotOpenFile, - filename) - - proc ropef(frmt: TFormatStr, args: varargs[PRope]): PRope = - var i = 0 - var length = len(frmt) +proc insertInCache(s: string): PRope = + var h = hash(s) and high(cache) + result = cache[h] + if isNil(result) or result.data != s: + result = newRope(s) + cache[h] = result + +proc toRope(s: string): PRope = + if s.len == 0: result = nil - var num = 0 - while i <= length - 1: - if frmt[i] == '$': - inc(i) # skip '$' - case frmt[i] - of '$': - app(result, "$") - inc(i) - of '#': - inc(i) - app(result, args[num]) - inc(num) - of '0'..'9': - var j = 0 - while true: - j = (j * 10) + Ord(frmt[i]) - ord('0') - inc(i) - if (i > length + 0 - 1) or not (frmt[i] in {'0'..'9'}): break - num = j - if j > high(args) + 1: - internalError("ropes: invalid format string $" & $(j)) - else: - app(result, args[j - 1]) - of 'n': - if optLineDir notin gOptions: app(result, tnl) - inc i - of 'N': - app(result, tnl) - inc(i) - else: InternalError("ropes: invalid format string $" & frmt[i]) - var start = i - while i < length: - if frmt[i] != '$': inc(i) - else: break - if i - 1 >= start: - app(result, substr(frmt, start, i - 1)) - assert(RopeInvariant(result)) + else: + result = insertInCache(s) + assert(RopeInvariant(result)) - proc appf(c: var PRope, frmt: TFormatStr, args: varargs[PRope]) = - app(c, ropef(frmt, args)) +proc RopeSeqInsert(rs: var TRopeSeq, r: PRope, at: Natural) = + var length = len(rs) + if at > length: + setlen(rs, at + 1) + else: + setlen(rs, length + 1) # move old rope elements: + for i in countdown(length, at + 1): + rs[i] = rs[i - 1] # this is correct, I used pen and paper to validate it + rs[at] = r - const - bufSize = 1024 # 1 KB is reasonable +proc newRecRopeToStr(result: var string, resultLen: var int, r: PRope) = + var stack = @[r] + while len(stack) > 0: + var it = pop(stack) + while it.data == nil: + add(stack, it.right) + it = it.left + assert(it.data != nil) + CopyMem(addr(result[resultLen]), addr(it.data[0]), it.length) + Inc(resultLen, it.length) + assert(resultLen <= len(result)) - proc auxRopeEqualsFile(r: PRope, bin: var tfile, buf: Pointer): bool = - if r.data != nil: - if r.length > bufSize: - internalError("ropes: token too long") - return - var readBytes = readBuffer(bin, buf, r.length) - result = readBytes == r.length and - equalMem(buf, addr(r.data[0]), r.length) # BUGFIX - else: - result = auxRopeEqualsFile(r.left, bin, buf) - if result: result = auxRopeEqualsFile(r.right, bin, buf) - - proc RopeEqualsFile(r: PRope, f: string): bool = - var bin: tfile - result = open(bin, f) - if not result: - return # not equal if file does not exist - var buf = alloc(BufSize) - result = auxRopeEqualsFile(r, bin, buf) - if result: - result = readBuffer(bin, buf, bufSize) == 0 # really at the end of file? - dealloc(buf) - close(bin) +proc ropeToStr(p: PRope): string = + if p == nil: + result = "" + else: + result = newString(p.length) + var resultLen = 0 + newRecRopeToStr(result, resultLen, p) - proc crcFromRopeAux(r: PRope, startVal: TCrc32): TCrc32 = - if r.data != nil: - result = startVal - for i in countup(0, len(r.data) - 1): - result = updateCrc32(r.data[i], result) - else: - result = crcFromRopeAux(r.left, startVal) - result = crcFromRopeAux(r.right, result) +proc con(a, b: PRope): PRope = + if a == nil: result = b + elif b == nil: result = a + else: + result = newRope() + result.length = a.length + b.length + result.left = a + result.right = b - proc newCrcFromRopeAux(r: PRope, startVal: TCrc32): TCrc32 = - # XXX profiling shows this is actually expensive - var stack: TRopeSeq = @[r] - result = startVal - while len(stack) > 0: - var it = pop(stack) - while it.data == nil: - add(stack, it.right) - it = it.left - assert(it.data != nil) - var i = 0 - var L = len(it.data) - while i < L: - result = updateCrc32(it.data[i], result) +proc con(a: PRope, b: string): PRope = result = con(a, toRope(b)) +proc con(a: string, b: PRope): PRope = result = con(toRope(a), b) + +proc con(a: varargs[PRope]): PRope = + for i in countup(0, high(a)): result = con(result, a[i]) + +proc toRope(i: BiggestInt): PRope = result = toRope($i) + +proc app(a: var PRope, b: PRope) = a = con(a, b) +proc app(a: var PRope, b: string) = a = con(a, b) +proc prepend(a: var PRope, b: PRope) = a = con(b, a) + +proc writeRope*(f: TFile, c: PRope) = + var stack = @[c] + while len(stack) > 0: + var it = pop(stack) + while it.data == nil: + add(stack, it.right) + it = it.left + assert(it != nil) + assert(it.data != nil) + write(f, it.data) + +proc WriteRope*(head: PRope, filename: string, useWarning = false) = + var f: tfile + if open(f, filename, fmWrite): + if head != nil: WriteRope(f, head) + close(f) + else: + rawMessage(if useWarning: warnCannotOpenFile else: errCannotOpenFile, + filename) + +proc ropef(frmt: TFormatStr, args: varargs[PRope]): PRope = + var i = 0 + var length = len(frmt) + result = nil + var num = 0 + while i <= length - 1: + if frmt[i] == '$': + inc(i) # skip '$' + case frmt[i] + of '$': + app(result, "$") inc(i) + of '#': + inc(i) + app(result, args[num]) + inc(num) + of '0'..'9': + var j = 0 + while true: + j = (j * 10) + Ord(frmt[i]) - ord('0') + inc(i) + if (i > length + 0 - 1) or not (frmt[i] in {'0'..'9'}): break + num = j + if j > high(args) + 1: + internalError("ropes: invalid format string $" & $(j)) + else: + app(result, args[j - 1]) + of 'n': + if optLineDir notin gOptions: app(result, tnl) + inc i + of 'N': + app(result, tnl) + inc(i) + else: InternalError("ropes: invalid format string $" & frmt[i]) + var start = i + while i < length: + if frmt[i] != '$': inc(i) + else: break + if i - 1 >= start: + app(result, substr(frmt, start, i - 1)) + assert(RopeInvariant(result)) - proc crcFromRope(r: PRope): TCrc32 = - result = newCrcFromRopeAux(r, initCrc32) +proc appf(c: var PRope, frmt: TFormatStr, args: varargs[PRope]) = + app(c, ropef(frmt, args)) - proc writeRopeIfNotEqual(r: PRope, filename: string): bool = - # returns true if overwritten - var c: TCrc32 - c = crcFromFile(filename) - if c != crcFromRope(r): - writeRope(r, filename) - result = true - else: - result = false +const + bufSize = 1024 # 1 KB is reasonable -else: - # optimized but broken version: +proc auxRopeEqualsFile(r: PRope, bin: var tfile, buf: Pointer): bool = + if r.data != nil: + if r.length > bufSize: + internalError("ropes: token too long") + return + var readBytes = readBuffer(bin, buf, r.length) + result = readBytes == r.length and + equalMem(buf, addr(r.data[0]), r.length) # BUGFIX + else: + result = auxRopeEqualsFile(r.left, bin, buf) + if result: result = auxRopeEqualsFile(r.right, bin, buf) + +proc RopeEqualsFile(r: PRope, f: string): bool = + var bin: tfile + result = open(bin, f) + if not result: + return # not equal if file does not exist + var buf = alloc(BufSize) + result = auxRopeEqualsFile(r, bin, buf) + if result: + result = readBuffer(bin, buf, bufSize) == 0 # really at the end of file? + dealloc(buf) + close(bin) - type - PRope* = ref TRope - TRope*{.acyclic.} = object of TObject # the empty rope is represented - # by nil to safe space - left, right: PRope - L: int # < 0 if a leaf - d: array [0..PayloadSize, char] # != nil if a leaf - - TRopeSeq* = seq[PRope] +proc crcFromRopeAux(r: PRope, startVal: TCrc32): TCrc32 = + if r.data != nil: + result = startVal + for i in countup(0, len(r.data) - 1): + result = updateCrc32(r.data[i], result) + else: + result = crcFromRopeAux(r.left, startVal) + result = crcFromRopeAux(r.right, result) - proc con*(a, b: PRope): PRope - proc con*(a: PRope, b: string): PRope - proc con*(a: string, b: PRope): PRope - proc con*(a: varargs[PRope]): PRope - proc app*(a: var PRope, b: PRope) - proc app*(a: var PRope, b: string) - proc prepend*(a: var PRope, b: PRope) - proc toRope*(s: string): PRope - proc toRope*(i: BiggestInt): PRope - proc ropeLen*(a: PRope): int - proc writeRopeIfNotEqual*(r: PRope, filename: string): bool - proc ropeToStr*(p: PRope): string - proc ropef*(frmt: TFormatStr, args: varargs[PRope]): PRope - proc appf*(c: var PRope, frmt: TFormatStr, args: varargs[PRope]) - proc RopeEqualsFile*(r: PRope, f: string): bool - # returns true if the rope r is the same as the contents of file f - proc RopeInvariant*(r: PRope): bool - # exported for debugging - # implementation - - proc ropeLen(a: PRope): int = - if a == nil: result = 0 - else: result = a.L.abs - - proc newRope(data: string = nil): PRope = - if data != nil: - unsafeNew(result, sizeof(TRope)-PayloadSize+len(data)+1) - result.L = -len(data) - # copy including '\0': - copyMem(addr result.d, cstring(data), len(data)) - else: - unsafeNew(result, sizeof(TRope)-PayloadSize+1) - - proc eqContent(r: PRope, s: string): bool = - assert r.L < 0 - if -r.L == s.len: - result = equalMem(addr(r.d), cstring(s), s.len) - - when false: - proc newMutableRope*(capacity = 30): PRope = - ## creates a new rope that supports direct modifications of the rope's - ## 'data' and 'length' fields. - new(result) - result.data = newStringOfCap(capacity) - - proc freezeMutableRope*(r: PRope) {.inline.} = - r.length = r.data.len - - var - cache: array[0..2048*2 -1, PRope] - - proc RopeInvariant(r: PRope): bool = - if r == nil: - result = true - else: - result = true # - # if r.data <> snil then - # result := true - # else begin - # result := (r.left <> nil) and (r.right <> nil); - # if result then result := ropeInvariant(r.left); - # if result then result := ropeInvariant(r.right); - # end - - proc insertInCache(s: string): PRope = - var h = hash(s) and high(cache) - result = cache[h] - if isNil(result) or not eqContent(result, s): - result = newRope(s) - cache[h] = result - - proc toRope(s: string): PRope = - if s.len == 0: - result = nil - else: - result = insertInCache(s) - assert(RopeInvariant(result)) - - proc RopeSeqInsert(rs: var TRopeSeq, r: PRope, at: Natural) = - var length = len(rs) - if at > length: - setlen(rs, at + 1) - else: - setlen(rs, length + 1) # move old rope elements: - for i in countdown(length, at + 1): - rs[i] = rs[i - 1] # this is correct, I used pen and paper to validate it - rs[at] = r - - proc newRecRopeToStr(result: var string, resultLen: var int, r: PRope) = - var stack = @[r] - while len(stack) > 0: - var it = pop(stack) - while it.L >= 0: - add(stack, it.right) - it = it.left - assert(it.L < 0) - CopyMem(addr(result[resultLen]), addr(it.d[0]), -it.L) - Inc(resultLen, -it.L) - assert(resultLen <= len(result)) - - proc ropeToStr(p: PRope): string = - if p == nil: - result = "" - else: - result = newString(p.L.abs) - var resultLen = 0 - newRecRopeToStr(result, resultLen, p) - - proc con(a, b: PRope): PRope = - if a == nil: result = b - elif b == nil: result = a - else: - result = newRope() - result.L = a.L.abs + b.L.abs - result.left = a - result.right = b - - proc con(a: PRope, b: string): PRope = result = con(a, toRope(b)) - proc con(a: string, b: PRope): PRope = result = con(toRope(a), b) - - proc con(a: varargs[PRope]): PRope = - for i in countup(0, high(a)): result = con(result, a[i]) - - proc toRope(i: BiggestInt): PRope = result = toRope($i) - - proc app(a: var PRope, b: PRope) = a = con(a, b) - proc app(a: var PRope, b: string) = a = con(a, b) - proc prepend(a: var PRope, b: PRope) = a = con(b, a) - - proc writeRope*(f: TFile, c: PRope) = - var stack = @[c] - while len(stack) > 0: - var it = pop(stack) - while it.L >= 0: - add(stack, it.right) - it = it.left - assert(it != nil) - assert(it.L < 0) - write(f, cstring(it.d), -it.L) - - proc WriteRope*(head: PRope, filename: string, useWarning = false) = - var f: tfile - if open(f, filename, fmWrite): - if head != nil: WriteRope(f, head) - close(f) - else: - rawMessage(if useWarning: warnCannotOpenFile else: errCannotOpenFile, - filename) - - proc ropef(frmt: TFormatStr, args: varargs[PRope]): PRope = +proc newCrcFromRopeAux(r: PRope, startVal: TCrc32): TCrc32 = + # XXX profiling shows this is actually expensive + var stack: TRopeSeq = @[r] + result = startVal + while len(stack) > 0: + var it = pop(stack) + while it.data == nil: + add(stack, it.right) + it = it.left + assert(it.data != nil) var i = 0 - var length = len(frmt) - result = nil - var num = 0 - while i <= length - 1: - if frmt[i] == '$': - inc(i) # skip '$' - case frmt[i] - of '$': - app(result, "$") - inc(i) - of '#': - inc(i) - app(result, args[num]) - inc(num) - of '0'..'9': - var j = 0 - while true: - j = (j * 10) + Ord(frmt[i]) - ord('0') - inc(i) - if (i > length + 0 - 1) or not (frmt[i] in {'0'..'9'}): break - num = j - if j > high(args) + 1: - internalError("ropes: invalid format string $" & $(j)) - else: - app(result, args[j - 1]) - of 'n': - if optLineDir notin gOptions: app(result, tnl) - inc i - of 'N': - app(result, tnl) - inc(i) - else: InternalError("ropes: invalid format string $" & frmt[i]) - var start = i - while i < length: - if frmt[i] != '$': inc(i) - else: break - if i - 1 >= start: - app(result, substr(frmt, start, i - 1)) - assert(RopeInvariant(result)) + var L = len(it.data) + while i < L: + result = updateCrc32(it.data[i], result) + inc(i) - proc appf(c: var PRope, frmt: TFormatStr, args: varargs[PRope]) = - app(c, ropef(frmt, args)) +proc crcFromRope(r: PRope): TCrc32 = + result = newCrcFromRopeAux(r, initCrc32) - const - bufSize = 1024 # 1 KB is reasonable - - proc auxRopeEqualsFile(r: PRope, bin: var tfile, buf: Pointer): bool = - if r.L < 0: - if -r.L > bufSize: - internalError("ropes: token too long") - return - var readBytes = readBuffer(bin, buf, -r.L) - result = readBytes == -r.L and - equalMem(buf, addr(r.d[0]), readBytes) - else: - result = auxRopeEqualsFile(r.left, bin, buf) - if result: result = auxRopeEqualsFile(r.right, bin, buf) - - proc RopeEqualsFile(r: PRope, f: string): bool = - var bin: tfile - result = open(bin, f) - if not result: - return # not equal if file does not exist - var buf = alloc(BufSize) - result = auxRopeEqualsFile(r, bin, buf) - if result: - result = readBuffer(bin, buf, bufSize) == 0 # really at the end of file? - dealloc(buf) - close(bin) - - proc newCrcFromRopeAux(r: PRope, startVal: TCrc32): TCrc32 = - # XXX profiling shows this is actually expensive - var stack: TRopeSeq = @[r] - result = startVal - while len(stack) > 0: - var it = pop(stack) - while it.L >= 0: - add(stack, it.right) - it = it.left - assert(it.L < 0) - var i = 0 - var L = -it.L - while i < L: - result = updateCrc32(it.d[i], result) - inc(i) - - proc crcFromRope(r: PRope): TCrc32 = - result = newCrcFromRopeAux(r, initCrc32) - - proc writeRopeIfNotEqual(r: PRope, filename: string): bool = - # returns true if overwritten - var c: TCrc32 - c = crcFromFile(filename) - if c != crcFromRope(r): - writeRope(r, filename) - result = true - else: - result = false +proc writeRopeIfNotEqual(r: PRope, filename: string): bool = + # returns true if overwritten + var c: TCrc32 + c = crcFromFile(filename) + if c != crcFromRope(r): + writeRope(r, filename) + result = true + else: + result = false From b3e70febb476a4e6512ee499aba130008c3fa1c8 Mon Sep 17 00:00:00 2001 From: Araq Date: Sat, 12 Jan 2013 23:59:29 +0100 Subject: [PATCH 077/106] 'sort' for ordered tables --- lib/pure/collections/tables.nim | 48 ++++++++++++++++++++++++++++++++- tests/run/ttables.nim | 39 +++++++++++++++++++++++++++ 2 files changed, 86 insertions(+), 1 deletion(-) diff --git a/lib/pure/collections/tables.nim b/lib/pure/collections/tables.nim index 3b252f1d2..4290af08a 100755 --- a/lib/pure/collections/tables.nim +++ b/lib/pure/collections/tables.nim @@ -1,7 +1,7 @@ # # # Nimrod's Runtime Library -# (c) Copyright 2012 Andreas Rumpf +# (c) Copyright 2013 Andreas Rumpf # # See the file "copying.txt", included in this # distribution, for details about the copyright. @@ -308,6 +308,52 @@ proc `$`*[A, B](t: TOrderedTable[A, B]): string = ## The `$` operator for ordered hash tables. dollarImpl() +proc sort*[A, B](t: var TOrderedTable[A, B], + cmp: proc (x,y: tuple[key: A, val: B]): int) = + ## sorts `t` according to `cmp`. This modifies the internal list + ## that kept the insertion order, so insertion order is lost after this + ## call but key lookup and insertions remain possible after `sort` (in + ## contrast to the `sort` for count tables). + var list = t.first + var + p, q, e, tail, oldhead: int + nmerges, psize, qsize, i: int + if t.counter == 0: return + var insize = 1 + while true: + p = list; oldhead = list + list = -1; tail = -1; nmerges = 0 + while p >= 0: + inc(nmerges) + q = p + psize = 0 + i = 0 + while i < insize: + inc(psize) + q = t.data[q].next + if q < 0: break + inc(i) + qsize = insize + while psize > 0 or (qsize > 0 and q >= 0): + if psize == 0: + e = q; q = t.data[q].next; dec(qsize) + elif qsize == 0 or q < 0: + e = p; p = t.data[p].next; dec(psize) + elif cmp((t.data[p].key, t.data[p].val), + (t.data[q].key, t.data[q].val)) <= 0: + e = p; p = t.data[p].next; dec(psize) + else: + e = q; q = t.data[q].next; dec(qsize) + if tail >= 0: t.data[tail].next = e + else: list = e + tail = e + p = q + t.data[tail].next = -1 + if nmerges <= 1: break + insize = insize * 2 + t.first = list + t.last = tail + # ------------------------------ count tables ------------------------------- type diff --git a/tests/run/ttables.nim b/tests/run/ttables.nim index 3eb17a803..681ff5424 100755 --- a/tests/run/ttables.nim +++ b/tests/run/ttables.nim @@ -19,6 +19,25 @@ const "50": 344490, "60": 344491, "70": 344492, "80": 344497} + sorteddata = { + "---00": 346677844, + "0": 34404, + "1": 344004, + "10": 34484, + "11": 34474, + "12": 789, + "19": 34464, + "2": 344774, "20": 34454, + "3": 342244, "30": 34141244, + "34": 123456, + "4": 3412344, "40": 344114, + "5": 341232144, "50": 344490, + "6": 34214544, "60": 344491, + "7": 3434544, "70": 344492, + "8": 344544, "80": 344497, + "9": 34435644, + "90": 343} + block tableTest1: var t = initTable[tuple[x, y: int], string]() t[(0,0)] = "00" @@ -86,5 +105,25 @@ block countTableTest1: block SyntaxTest: var x = toTable[int, string]({:}) +proc orderedTableSortTest() = + var t = initOrderedTable[string, int](2) + for key, val in items(data): t[key] = val + for key, val in items(data): assert t[key] == val + t.sort(proc (x, y: tuple[key: string, val: int]): int = cmp(x.key, y.key)) + var i = 0 + # `pairs` needs to yield in sorted order: + for key, val in pairs(t): + doAssert key == sorteddata[i][0] + doAssert val == sorteddata[i][1] + inc(i) + + # check that lookup still works: + for key, val in pairs(t): + doAssert val == t[key] + # check that insert still works: + t["newKeyHere"] = 80 + + +orderedTableSortTest() echo "true" From 2f14d0e10890ccbdf4e2cdb2d553ad38ecb0322d Mon Sep 17 00:00:00 2001 From: Araq Date: Tue, 15 Jan 2013 01:09:15 +0100 Subject: [PATCH 078/106] fixes a performance regression for json handling --- lib/pure/json.nim | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/lib/pure/json.nim b/lib/pure/json.nim index cee135f08..cca1e4a70 100755 --- a/lib/pure/json.nim +++ b/lib/pure/json.nim @@ -541,6 +541,11 @@ proc newJString*(s: String): PJsonNode = result.kind = JString result.str = s +proc newJStringMove(s: String): PJsonNode = + new(result) + result.kind = JString + shallowCopy(result.str, s) + proc newJInt*(n: biggestInt): PJsonNode = ## Creates a new `JInt PJsonNode`. new(result) @@ -809,7 +814,9 @@ proc parseJson(p: var TJsonParser): PJsonNode = ## Parses JSON from a JSON Parser `p`. case p.tok of tkString: - result = newJString(p.a) + # we capture 'p.a' here, so we need to give it a fresh buffer afterwards: + result = newJStringMove(p.a) + p.a = "" discard getTok(p) of tkInt: result = newJInt(parseBiggestInt(p.a)) From 10d995c2908e2c4a2e8faeb7bf92e9f38987c85a Mon Sep 17 00:00:00 2001 From: Araq Date: Tue, 15 Jan 2013 01:09:38 +0100 Subject: [PATCH 079/106] support for LuaJIT --- lib/wrappers/lua/lua.nim | 39 ++++++++++++++++++++++++++------------- 1 file changed, 26 insertions(+), 13 deletions(-) diff --git a/lib/wrappers/lua/lua.nim b/lib/wrappers/lua/lua.nim index 000e09993..0346c4285 100755 --- a/lib/wrappers/lua/lua.nim +++ b/lib/wrappers/lua/lua.nim @@ -35,19 +35,32 @@ #** In french or in english # -when defined(MACOSX): - const - NAME* = "liblua(|5.2|5.1|5.0).dylib" - LIB_NAME* = "liblua(|5.2|5.1|5.0).dylib" -elif defined(UNIX): - const - NAME* = "liblua(|5.2|5.1|5.0).so(|.0)" - LIB_NAME* = "liblua(|5.2|5.1|5.0).so(|.0)" -else: - const - NAME* = "lua(|5.2|5.1|5.0).dll" - LIB_NAME* = "lua(|5.2|5.1|5.0).dll" - +when defined(useLuajit): + when defined(MACOSX): + const + NAME* = "libluajit.dylib" + LIB_NAME* = "libluajit.dylib" + elif defined(UNIX): + const + NAME* = "libluajit.so(|.0)" + LIB_NAME* = "libluajit.so(|.0)" + else: + const + NAME* = "luajit.dll" + LIB_NAME* = "luajit.dll" +else: + when defined(MACOSX): + const + NAME* = "liblua(|5.2|5.1|5.0).dylib" + LIB_NAME* = "liblua(|5.2|5.1|5.0).dylib" + elif defined(UNIX): + const + NAME* = "liblua(|5.2|5.1|5.0).so(|.0)" + LIB_NAME* = "liblua(|5.2|5.1|5.0).so(|.0)" + else: + const + NAME* = "lua(|5.2|5.1|5.0).dll" + LIB_NAME* = "lua(|5.2|5.1|5.0).dll" const VERSION* = "Lua 5.1" From c43697b59a08459fdfe41129b9ed0f96cf0150fa Mon Sep 17 00:00:00 2001 From: Araq Date: Wed, 16 Jan 2013 08:42:30 +0100 Subject: [PATCH 080/106] implemented generic multi methods --- compiler/cgmeth.nim | 19 +++++++++---------- compiler/sem.nim | 3 ++- compiler/semcall.nim | 20 +++++++++++++++++--- compiler/seminst.nim | 1 + compiler/semstmts.nim | 29 +++++++++++++++-------------- compiler/types.nim | 8 ++++++-- tests/run/tmultim6.nim | 30 ++++++++++++++++++++++++++++++ todo.txt | 1 - web/news.txt | 1 + 9 files changed, 81 insertions(+), 31 deletions(-) create mode 100644 tests/run/tmultim6.nim diff --git a/compiler/cgmeth.nim b/compiler/cgmeth.nim index 687653aaf..e7bd54ef0 100755 --- a/compiler/cgmeth.nim +++ b/compiler/cgmeth.nim @@ -1,7 +1,7 @@ # # # The Nimrod Compiler -# (c) Copyright 2012 Andreas Rumpf +# (c) Copyright 2013 Andreas Rumpf # # See the file "copying.txt", included in this # distribution, for details about the copyright. @@ -65,10 +65,10 @@ proc sameMethodBucket(a, b: PSym): bool = break if sameType(aa, bb) or (aa.kind == tyObject) and (bb.kind == tyObject) and - (inheritanceDiff(bb, aa) < 0): + (inheritanceDiff(bb, aa) < 0): nil - else: - return + else: + return result = true proc attachDispatcher(s: PSym, dispatcher: PNode) = @@ -106,17 +106,16 @@ proc methodDef*(s: PSym, fromCache: bool) = # attach to itself to prevent bugs: attachDispatcher(disp, newSymNode(disp)) -proc relevantCol(methods: TSymSeq, col: int): bool = +proc relevantCol(methods: TSymSeq, col: int): bool = # returns true iff the position is relevant var t = methods[0].typ.sons[col] - result = false - if skipTypes(t, skipPtrs).kind == tyObject: - for i in countup(1, high(methods)): - if not SameType(methods[i].typ.sons[col], t): + if skipTypes(t, skipPtrs).kind == tyObject: + for i in countup(1, high(methods)): + let t2 = skipTypes(methods[i].typ.sons[col], skipPtrs) + if not SameType(t2, t): return true proc cmpSignatures(a, b: PSym, relevantCols: TIntSet): int = - result = 0 for col in countup(1, sonsLen(a.typ) - 1): if Contains(relevantCols, col): var aa = skipTypes(a.typ.sons[col], skipPtrs) diff --git a/compiler/sem.nim b/compiler/sem.nim index c70cbf61e..77e93a066 100755 --- a/compiler/sem.nim +++ b/compiler/sem.nim @@ -1,7 +1,7 @@ # # # The Nimrod Compiler -# (c) Copyright 2012 Andreas Rumpf +# (c) Copyright 2013 Andreas Rumpf # # See the file "copying.txt", included in this # distribution, for details about the copyright. @@ -45,6 +45,7 @@ proc tryExpr(c: PContext, n: PNode, flags: TExprFlags = {}): PNode proc fixImmediateParams(n: PNode): PNode proc activate(c: PContext, n: PNode) proc semQuoteAst(c: PContext, n: PNode): PNode +proc finishMethod(c: PContext, s: PSym) proc IndexTypesMatch(c: PContext, f, a: PType, arg: PNode): PNode diff --git a/compiler/semcall.nim b/compiler/semcall.nim index 0a5f19822..67d157261 100755 --- a/compiler/semcall.nim +++ b/compiler/semcall.nim @@ -1,7 +1,7 @@ # # # The Nimrod Compiler -# (c) Copyright 2012 Andreas Rumpf +# (c) Copyright 2013 Andreas Rumpf # # See the file "copying.txt", included in this # distribution, for details about the copyright. @@ -15,8 +15,22 @@ proc sameMethodDispatcher(a, b: PSym): bool = if a.kind == skMethod and b.kind == skMethod: var aa = lastSon(a.ast) var bb = lastSon(b.ast) - if aa.kind == nkSym and bb.kind == nkSym and aa.sym == bb.sym: - result = true + if aa.kind == nkSym and bb.kind == nkSym: + if aa.sym == bb.sym: + result = true + else: + nil + # generics have no dispatcher yet, so we need to compare the method + # names; however, the names are equal anyway because otherwise we + # wouldn't even consider them to be overloaded. But even this does + # not work reliably! See tmultim6 for an example: + # method collide[T](a: TThing, b: TUnit[T]) is instantiated and not + # method collide[T](a: TUnit[T], b: TThing)! This means we need to + # *instantiate* every candidate! However, we don't keep more than 2-3 + # candidated around so we cannot implement that for now. So in order + # to avoid subtle problems, the call remains ambiguous and needs to + # be disambiguated by the programmer; this way the right generic is + # instantiated. proc resolveOverloads(c: PContext, n, orig: PNode, filter: TSymKinds): TCandidate = diff --git a/compiler/seminst.nim b/compiler/seminst.nim index 0a1a17f72..b54170435 100755 --- a/compiler/seminst.nim +++ b/compiler/seminst.nim @@ -186,6 +186,7 @@ proc generateInstance(c: PContext, fn: PSym, pt: TIdTable, popOwner() c.friendModule = oldFriend dec(c.InstCounter) + if result.kind == skMethod: finishMethod(c, result) proc instGenericContainer(c: PContext, n: PNode, header: PType): PType = var cl: TReplTypeVars diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index c86e3eb91..363cff89a 100755 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -867,25 +867,26 @@ proc semIterator(c: PContext, n: PNode): PNode = proc semProc(c: PContext, n: PNode): PNode = result = semProcAux(c, n, skProc, procPragmas) +proc hasObjParam(s: PSym): bool = + var t = s.typ + for col in countup(1, sonsLen(t)-1): + if skipTypes(t.sons[col], skipPtrs).kind == tyObject: + return true + +proc finishMethod(c: PContext, s: PSym) = + if hasObjParam(s): + methodDef(s, false) + proc semMethod(c: PContext, n: PNode): PNode = if not isTopLevel(c): LocalError(n.info, errXOnlyAtModuleScope, "method") result = semProcAux(c, n, skMethod, methodPragmas) var s = result.sons[namePos].sym - var t = s.typ - var hasObjParam = false - - for col in countup(1, sonsLen(t)-1): - if skipTypes(t.sons[col], skipPtrs).kind == tyObject: - hasObjParam = true - break - - # XXX this not really correct way to do it: Perhaps it should be done after - # generic instantiation. Well it's good enough for now: - if hasObjParam: - methodDef(s, false) - else: - LocalError(n.info, errXNeedsParamObjectType, "method") + if not isGenericRoutine(s): + if hasObjParam(s): + methodDef(s, false) + else: + LocalError(n.info, errXNeedsParamObjectType, "method") proc semConverterDef(c: PContext, n: PNode): PNode = if not isTopLevel(c): LocalError(n.info, errXOnlyAtModuleScope, "converter") diff --git a/compiler/types.nim b/compiler/types.nim index 23d202fdf..998ba43d2 100755 --- a/compiler/types.nim +++ b/compiler/types.nim @@ -1,7 +1,7 @@ # # # The Nimrod Compiler -# (c) Copyright 2012 Andreas Rumpf +# (c) Copyright 2013 Andreas Rumpf # # See the file "copying.txt", included in this # distribution, for details about the copyright. @@ -876,20 +876,24 @@ proc inheritanceDiff*(a, b: PType): int = # | returns: -x iff `a` is the x'th direct superclass of `b` # | returns: +x iff `a` is the x'th direct subclass of `b` # | returns: `maxint` iff `a` and `b` are not compatible at all + assert a.kind == tyObject + assert b.kind == tyObject var x = a result = 0 while x != nil: + x = skipTypes(x, skipPtrs) if sameObjectTypes(x, b): return x = x.sons[0] dec(result) var y = b result = 0 while y != nil: + y = skipTypes(y, skipPtrs) if sameObjectTypes(y, a): return y = y.sons[0] inc(result) result = high(int) - + proc typeAllowedAux(marker: var TIntSet, typ: PType, kind: TSymKind): bool proc typeAllowedNode(marker: var TIntSet, n: PNode, kind: TSymKind): bool = result = true diff --git a/tests/run/tmultim6.nim b/tests/run/tmultim6.nim new file mode 100644 index 000000000..a55b69e37 --- /dev/null +++ b/tests/run/tmultim6.nim @@ -0,0 +1,30 @@ +discard """ + output: "collide: unit, thing | collide: unit, thing | collide: thing, unit" +""" +# Test multi methods + +type + TThing = object {.inheritable.} + TUnit[T] = object of TThing + x: T + TParticle = object of TThing + a, b: int + +method collide(a, b: TThing) {.inline.} = + quit "to override!" + +method collide[T](a: TThing, b: TUnit[T]) {.inline.} = + write stdout, "collide: thing, unit | " + +method collide[T](a: TUnit[T], b: TThing) {.inline.} = + write stdout, "collide: unit, thing | " + +proc test(a, b: TThing) {.inline.} = + collide(a, b) + +var + a: TThing + b, c: TUnit[string] +collide(b, TThing(c)) +test(b, c) +collide(a, b) diff --git a/todo.txt b/todo.txt index dd3290117..ab7c4f2f7 100755 --- a/todo.txt +++ b/todo.txt @@ -15,7 +15,6 @@ version 0.9.X - test&finish first class iterators: * nested iterators - implement the missing features wrt inheritance -- implement generic methods - improve the compiler as a service - ``=`` should be overloadable; requires specialization for ``=`` - implement constructors + full 'not nil' checking diff --git a/web/news.txt b/web/news.txt index 824e0c3a3..e4e9108e0 100755 --- a/web/news.txt +++ b/web/news.txt @@ -57,6 +57,7 @@ Language Additions symbol forwarding so client modules don't have to import a module's dependencies explicitly. - Overloading based on ASTs has been implemented. +- Generics are now supported for multi methods. 2012-09-23 Version 0.9.0 released From e353737e34eeba6056737fa9e895f69bd56fb3af Mon Sep 17 00:00:00 2001 From: Araq Date: Wed, 16 Jan 2013 19:57:29 +0100 Subject: [PATCH 081/106] todo.txt changes --- todo.txt | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/todo.txt b/todo.txt index ab7c4f2f7..56a27e0fa 100755 --- a/todo.txt +++ b/todo.txt @@ -1,11 +1,21 @@ version 0.9.2 ============= +- implement constructors + full 'not nil' checking +- ``restrict`` pragma + backend support - fix closure bug finally - fix marshal bug - investigate nimgame bug +version 0.9.4 +============= + +- make 'bind' default for templates and introduce 'mixin'; + special rule for ``[]=`` +- implicit deref for parameter matching; overloading based on 'var T' + + version 0.9.X ============= @@ -17,15 +27,10 @@ version 0.9.X - implement the missing features wrt inheritance - improve the compiler as a service - ``=`` should be overloadable; requires specialization for ``=`` -- implement constructors + full 'not nil' checking -- make 'bind' default for templates and introduce 'mixin'; - special rule for ``[]=`` -- implicit deref for parameter matching; overloading based on 'var T' - optimize genericAssign in the code generator - better support for macros that rewrite procs - macros need access to types and symbols (partially implemented) - result = result shr 8 for the "system()" wrapper - - rethink the syntax/grammar: * parser is not strict enough with newlines * change comment handling in the AST From 9257c29ffd68fffd61c68061318ca52bb8324b4c Mon Sep 17 00:00:00 2001 From: Araq Date: Sat, 19 Jan 2013 02:09:08 +0100 Subject: [PATCH 082/106] small bugfix for lambdalifting; preparations for a better testing framework --- compiler/lambdalifting.nim | 5 +- lib/pure/json.nim | 2 +- tests/compile/tforwardgeneric.nim | 1 + tests/specials.nim | 2 +- tests/tester.nim | 114 ++++++++++++++++++++---------- 5 files changed, 82 insertions(+), 42 deletions(-) diff --git a/compiler/lambdalifting.nim b/compiler/lambdalifting.nim index 309e01c6f..bca5b8c1d 100644 --- a/compiler/lambdalifting.nim +++ b/compiler/lambdalifting.nim @@ -257,6 +257,8 @@ proc captureVar(o: POuterContext, i: PInnerContext, local: PSym, # Currently captures are restricted to a single level of nesting: LocalError(info, errIllegalCaptureX, local.name.s) i.fn.typ.callConv = ccClosure + #echo "captureVar ", i.fn.name.s, i.fn.id, " ", local.name.s, local.id + incl(i.fn.typ.flags, tfCapturesEnv) # we need to remember which inner most closure belongs to this lambda: @@ -286,9 +288,10 @@ proc interestingVar(s: PSym): bool {.inline.} = sfGlobal notin s.flags proc semCaptureSym*(s, owner: PSym) = - if interestingVar(s) and owner.id != s.owner.id: + if interestingVar(s) and owner.id != s.owner.id and s.kind != skResult: if owner.typ != nil and not isGenericRoutine(owner): owner.typ.callConv = ccClosure + #echo "semCaptureSym ", owner.name.s, owner.id, " ", s.name.s, s.id # since the analysis is not entirely correct, we don't set 'tfCapturesEnv' # here diff --git a/lib/pure/json.nim b/lib/pure/json.nim index cca1e4a70..d2516adfb 100755 --- a/lib/pure/json.nim +++ b/lib/pure/json.nim @@ -636,7 +636,7 @@ proc `[]`*(node: PJsonNode, name: String): PJsonNode = return item return nil -proc `[]`*(node: PJsonNode, index: Int): PJsonNode {.raises: [EInvalidIndex].} = +proc `[]`*(node: PJsonNode, index: Int): PJsonNode = ## Gets the node at `index` in an Array. assert(node.kind == JArray) return node.elems[index] diff --git a/tests/compile/tforwardgeneric.nim b/tests/compile/tforwardgeneric.nim index 84bef15cc..ef263d733 100644 --- a/tests/compile/tforwardgeneric.nim +++ b/tests/compile/tforwardgeneric.nim @@ -1,5 +1,6 @@ discard """ output: "1.0000000000000000e+00 10" + ccodecheck: "!@'ClEnv'" """ proc p[T](a, b: T): T diff --git a/tests/specials.nim b/tests/specials.nim index ba46e67b4..08013ea7d 100644 --- a/tests/specials.nim +++ b/tests/specials.nim @@ -1,7 +1,7 @@ # # # Nimrod Tester -# (c) Copyright 2012 Andreas Rumpf +# (c) Copyright 2013 Andreas Rumpf # # See the file "copying.txt", included in this # distribution, for details about the copyright. diff --git a/tests/tester.nim b/tests/tester.nim index 8156dab7f..60c84403e 100755 --- a/tests/tester.nim +++ b/tests/tester.nim @@ -1,7 +1,7 @@ # # # Nimrod Tester -# (c) Copyright 2012 Andreas Rumpf +# (c) Copyright 2013 Andreas Rumpf # # See the file "copying.txt", included in this # distribution, for details about the copyright. @@ -25,22 +25,35 @@ const type TTestAction = enum actionCompile, actionRun, actionReject - TSpec {.pure.} = object + TResultEnum = enum + reNimrodcCrash, # nimrod compiler seems to have crashed + reMsgsDiffer, # error messages differ + reFilesDiffer, # expected and given filenames differ + reLinesDiffer, # expected and given line numbers differ + reOutputsDiffer, + reExitcodesDiffer, + reInvalidPeg, + reCodegenFailure, + reCodeNotFound, + reExeNotFound, + reIgnored, # test is ignored + reSuccess # test was successful + TTarget = enum + targetC, targetCpp, targetObjC, targetJS + + TSpec = object action: TTestAction file, cmd: string outp: string line, exitCode: int msg: string - err: bool - disabled: bool + ccodeCheck: string + err: TResultEnum substr: bool - TResults {.pure.} = object + TResults = object total, passed, skipped: int data: string - TResultEnum = enum - reFailure, reIgnored, reSuccess - # ----------------------- Spec parser ---------------------------------------- when not defined(parseCfgBool): @@ -82,9 +95,9 @@ template parseSpecAux(fillResult: stmt) {.immediate.} = proc parseSpec(filename: string): TSpec = result.file = filename - result.err = true result.msg = "" result.outp = "" + result.ccodeCheck = "" result.cmd = cmdTemplate parseSpecAux: case normalize(e.key) @@ -103,8 +116,10 @@ proc parseSpec(filename: string): TSpec = of "exitcode": discard parseInt(e.value, result.exitCode) of "errormsg", "msg": result.msg = e.value - of "disabled": result.disabled = parseCfgBool(e.value) + of "disabled": + if parseCfgBool(e.value): result.err = reIgnored of "cmd": result.cmd = e.value + of "ccodecheck": result.ccodeCheck = e.value else: echo ignoreMsg(p, e) # ---------------------------------------------------------------------------- @@ -134,7 +149,6 @@ proc callCompiler(cmdTemplate, filename, options: string): TSpec = result.msg = "" result.file = "" result.outp = "" - result.err = true result.line = -1 if err =~ pegLineError: result.file = extractFilename(matches[0]) @@ -143,7 +157,7 @@ proc callCompiler(cmdTemplate, filename, options: string): TSpec = elif err =~ pegOtherError: result.msg = matches[0] elif suc =~ pegSuccess: - result.err = false + result.err = reSuccess proc initResults: TResults = result.total = 0 @@ -164,9 +178,9 @@ proc `$`(x: TResults): string = proc colorResult(r: TResultEnum): string = case r - of reFailure: result = "no" of reIgnored: result = "ignored" of reSuccess: result = "yes" + else: result = "no" const TableHeader4 = "" & @@ -217,12 +231,12 @@ proc listResults(reject, compile, run: TResults) = proc cmpMsgs(r: var TResults, expected, given: TSpec, test: string) = if strip(expected.msg) notin strip(given.msg): - r.addResult(test, expected.msg, given.msg, reFailure) + r.addResult(test, expected.msg, given.msg, reMsgsDiffer) elif extractFilename(expected.file) != extractFilename(given.file) and "internal error:" notin expected.msg: - r.addResult(test, expected.file, given.file, reFailure) + r.addResult(test, expected.file, given.file, reFilesDiffer) elif expected.line != given.line and expected.line != 0: - r.addResult(test, $expected.line, $given.line, reFailure) + r.addResult(test, $expected.line, $given.line, reLinesDiffer) else: r.addResult(test, expected.msg, given.msg, reSuccess) inc(r.passed) @@ -233,7 +247,7 @@ proc rejectSingleTest(r: var TResults, test, options: string) = inc(r.total) echo t var expected = parseSpec(test) - if expected.disabled: + if expected.err == reIgnored: r.addResult(t, "", "", reIgnored) inc(r.skipped) else: @@ -244,31 +258,47 @@ proc reject(r: var TResults, dir, options: string) = ## handle all the tests that the compiler should reject for test in os.walkFiles(dir / "t*.nim"): rejectSingleTest(r, test, options) +proc codegenCheck(test, check, ext: string, given: var TSpec) = + if check.len > 0: + try: + let (path, name, ext2) = test.splitFile + echo path / "nimcache" / name.changeFileExt(ext) + let contents = readFile(path / "nimcache" / name.changeFileExt(ext)).string + if contents.find(check.peg) < 0: + given.err = reCodegenFailure + except EInvalidValue: + given.err = reInvalidPeg + except EIO: + given.err = reCodeNotFound + +proc codegenChecks(test: string, expected: TSpec, given: var TSpec) = + codegenCheck(test, expected.ccodeCheck, ".c", given) + proc compile(r: var TResults, pattern, options: string) = for test in os.walkFiles(pattern): let t = extractFilename(test) echo t inc(r.total) let expected = parseSpec(test) - if expected.disabled: + if expected.err == reIgnored: r.addResult(t, "", reIgnored) inc(r.skipped) else: var given = callCompiler(expected.cmd, test, options) - r.addResult(t, given.msg, if given.err: reFailure else: reSuccess) - if not given.err: inc(r.passed) + if given.err == reSuccess: + codegenChecks(test, expected, given) + r.addResult(t, given.msg, given.err) + if given.err == reSuccess: inc(r.passed) proc compileSingleTest(r: var TResults, test, options: string) = + # does not extract the spec because the file is not supposed to have any let test = test.addFileExt(".nim") let t = extractFilename(test) inc(r.total) echo t let given = callCompiler(cmdTemplate, test, options) - r.addResult(t, given.msg, if given.err: reFailure else: reSuccess) - if not given.err: inc(r.passed) - -type - TTarget = enum targetC, targetJS + r.addResult(t, given.msg, given.err) + if given.err == reSuccess: inc(r.passed) proc runSingleTest(r: var TResults, test, options: string, target: TTarget) = var test = test.addFileExt(".nim") @@ -276,13 +306,13 @@ proc runSingleTest(r: var TResults, test, options: string, target: TTarget) = echo t inc(r.total) var expected = parseSpec(test) - if expected.disabled: + if expected.err == reIgnored: r.addResult(t, "", "", reIgnored) inc(r.skipped) else: var given = callCompiler(expected.cmd, test, options) - if given.err: - r.addResult(t, "", given.msg, reFailure) + if given.err != reSuccess: + r.addResult(t, "", given.msg, given.err) else: var exeFile: string if target == targetC: @@ -296,25 +326,26 @@ proc runSingleTest(r: var TResults, test, options: string, target: TTarget) = (if target==targetJS: "node " else: "") & exeFile) if exitCode != expected.ExitCode: r.addResult(t, "exitcode: " & $expected.ExitCode, - "exitcode: " & $exitCode, reFailure) + "exitcode: " & $exitCode, reExitCodesDiffer) else: - var success = strip(buf.string) == strip(expected.outp) - if expected.substr and not success: - success = expected.outp in buf.string - if success: inc(r.passed) - r.addResult(t, expected.outp, - buf.string, if success: reSuccess else: reFailure) + if strip(buf.string) != strip(expected.outp): + if not (expected.substr and expected.outp in buf.string): + given.err = reOutputsDiffer + if given.err == reSuccess: + codeGenChecks(test, expected, given) + if given.err == reSuccess: inc(r.passed) + r.addResult(t, expected.outp, buf.string, given.err) else: - r.addResult(t, expected.outp, "executable not found", reFailure) + r.addResult(t, expected.outp, "executable not found", reExeNotFound) proc runSingleTest(r: var TResults, test, options: string) = runSingleTest(r, test, options, targetC) - + proc run(r: var TResults, dir, options: string) = for test in os.walkFiles(dir / "t*.nim"): runSingleTest(r, test, options) include specials - + proc compileExample(r: var TResults, pattern, options: string) = for test in os.walkFiles(pattern): compileSingleTest(r, test, options) @@ -392,7 +423,12 @@ proc main() = runDLLTests r, p.cmdLineRest.string of "gc": runGCTests(r, p.cmdLineRest.string) - of "test", "comp", "rej": + of "test": + if p.kind != cmdArgument: quit usage + var testFile = p.key.string + p.next() + runSingleTest(r, testFile, p.cmdLineRest.string) + of "comp", "rej": if p.kind != cmdArgument: quit usage var testFile = p.key.string p.next() From 7b5b9656f5cc92f76fbaba763385970169e3a7f6 Mon Sep 17 00:00:00 2001 From: Araq Date: Sat, 19 Jan 2013 12:33:07 +0100 Subject: [PATCH 083/106] fixes sequtils.filterIt --- lib/pure/collections/sequtils.nim | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/lib/pure/collections/sequtils.nim b/lib/pure/collections/sequtils.nim index e4ea22830..73713eec9 100644 --- a/lib/pure/collections/sequtils.nim +++ b/lib/pure/collections/sequtils.nim @@ -129,11 +129,10 @@ template filterIt*(seq1, pred: expr): expr {.immediate, dirty.} = ## temperatures = @[-272.15, -2.0, 24.5, 44.31, 99.9, -113.44] ## acceptable = filterIt(temperatures, it < 50 and it > -10) ## assert acceptable == @[-2.0, 24.5, 44.31] - block: - var result: type(seq1) = @[] - for it in items(seq1): - if pred: result.add(it) - result + var result {.gensym.}: type(seq1) = @[] + for it in items(seq1): + if pred: result.add(it) + result template toSeq*(iter: expr): expr {.immediate.} = ## Transforms any iterator into a sequence. From b9d75f894401afcbc98081167555715a95e67d43 Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Sun, 20 Jan 2013 12:47:25 +0000 Subject: [PATCH 084/106] Fixed a bug with the 'do notation' consuming statements after it. --- compiler/parser.nim | 20 ++++++++------------ tests/run/tdomulttest.nim | 17 +++++++++++++++++ 2 files changed, 25 insertions(+), 12 deletions(-) create mode 100644 tests/run/tdomulttest.nim diff --git a/compiler/parser.nim b/compiler/parser.nim index bfc6b3b65..e785a7e62 100755 --- a/compiler/parser.nim +++ b/compiler/parser.nim @@ -389,7 +389,7 @@ proc parseGStrLit(p: var TParser, a: PNode): PNode = else: result = a -proc identOrLiteral(p: var TParser): PNode = +proc identOrLiteral(p: var TParser): PNode = case p.tok.tokType of tkSymbol: result = newIdentNodeP(p.tok.ident, p) @@ -494,12 +494,6 @@ proc primarySuffix(p: var TParser, r: PNode): PNode = result = newNodeP(nkCall, p) addSon(result, a) exprColonEqExprListAux(p, nkExprEqExpr, tkParRi, tkEquals, result) - parseDoBlocks(p, result) - of tkDo: - var a = result - result = newNodeP(nkCall, p) - addSon(result, a) - parseDoBlocks(p, result) of tkDot: result = dotExpr(p, result) result = parseGStrLit(p, result) @@ -736,7 +730,7 @@ proc parseProcExpr(p: var TParser, isExpr: bool): PNode = proc isExprStart(p: TParser): bool = case p.tok.tokType of tkSymbol, tkAccent, tkOpr, tkNot, tkNil, tkCast, tkIf, - tkProc, tkIterator, tkBind, + tkProc, tkDo, tkIterator, tkBind, tkParLe, tkBracketLe, tkCurlyLe, tkIntLit..tkCharLit, tkVar, tkRef, tkPtr, tkTuple, tkObject, tkType, tkWhen, tkCase, tkShared: result = true @@ -832,6 +826,8 @@ proc primary(p: var TParser, mode: TPrimaryMode): PNode = getTok(p) optInd(p, result) addSon(result, primary(p, pmNormal)) + of tkDo: + result = parseDoBlock(p) else: result = identOrLiteral(p) if mode != pmSkipSuffix: @@ -856,7 +852,7 @@ proc parseExprStmt(p: var TParser): PNode = var call = if a.kind == nkCall: a else: newNode(nkCommand, a.info, @[a]) while true: - if not isExprStart(p): break + if not isExprStart(p): break var e = parseExpr(p) addSon(call, e) if p.tok.tokType != tkComma: break @@ -864,7 +860,7 @@ proc parseExprStmt(p: var TParser): PNode = optInd(p, a) if p.tok.tokType == tkDo: parseDoBlocks(p, call) - return + return call result = if call.sonsLen <= 1: a else: call if p.tok.tokType == tkColon: @@ -1552,14 +1548,14 @@ proc parseAll(p: var TParser): PNode = proc parseTopLevelStmt(p: var TParser): PNode = result = ast.emptyNode - while true: + while true: case p.tok.tokType of tkSad, tkSemicolon: getTok(p) of tkDed, tkInd: parMessage(p, errInvalidIndentation) getTok(p) of tkEof: break - else: + else: result = complexOrSimpleStmt(p) if result.kind == nkEmpty: parMessage(p, errExprExpected, p.tok) break diff --git a/tests/run/tdomulttest.nim b/tests/run/tdomulttest.nim new file mode 100644 index 000000000..76ab0963c --- /dev/null +++ b/tests/run/tdomulttest.nim @@ -0,0 +1,17 @@ +discard """ + file: "tasynciossl.nim" + cmd: "nimrod cc --hints:on --define:ssl $# $#" + output: "555\ntest\nmulti lines\n99999999\nend" +""" +proc foo(bar, baz: proc (x: int): int) = + echo bar(555) + echo baz(99999999) + +foo do (x: int) -> int: + return x +do (x: int) -> int: + echo("test") + echo("multi lines") + return x + +echo("end") \ No newline at end of file From be51e8b61b1afbedb1e041b087236787f6e4df81 Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Sun, 20 Jan 2013 17:34:43 +0000 Subject: [PATCH 085/106] Reverted my 'do notation' fixes. --- compiler/parser.nim | 20 ++++++++++++-------- tests/run/tdomulttest.nim | 3 +-- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/compiler/parser.nim b/compiler/parser.nim index e785a7e62..bfc6b3b65 100755 --- a/compiler/parser.nim +++ b/compiler/parser.nim @@ -389,7 +389,7 @@ proc parseGStrLit(p: var TParser, a: PNode): PNode = else: result = a -proc identOrLiteral(p: var TParser): PNode = +proc identOrLiteral(p: var TParser): PNode = case p.tok.tokType of tkSymbol: result = newIdentNodeP(p.tok.ident, p) @@ -494,6 +494,12 @@ proc primarySuffix(p: var TParser, r: PNode): PNode = result = newNodeP(nkCall, p) addSon(result, a) exprColonEqExprListAux(p, nkExprEqExpr, tkParRi, tkEquals, result) + parseDoBlocks(p, result) + of tkDo: + var a = result + result = newNodeP(nkCall, p) + addSon(result, a) + parseDoBlocks(p, result) of tkDot: result = dotExpr(p, result) result = parseGStrLit(p, result) @@ -730,7 +736,7 @@ proc parseProcExpr(p: var TParser, isExpr: bool): PNode = proc isExprStart(p: TParser): bool = case p.tok.tokType of tkSymbol, tkAccent, tkOpr, tkNot, tkNil, tkCast, tkIf, - tkProc, tkDo, tkIterator, tkBind, + tkProc, tkIterator, tkBind, tkParLe, tkBracketLe, tkCurlyLe, tkIntLit..tkCharLit, tkVar, tkRef, tkPtr, tkTuple, tkObject, tkType, tkWhen, tkCase, tkShared: result = true @@ -826,8 +832,6 @@ proc primary(p: var TParser, mode: TPrimaryMode): PNode = getTok(p) optInd(p, result) addSon(result, primary(p, pmNormal)) - of tkDo: - result = parseDoBlock(p) else: result = identOrLiteral(p) if mode != pmSkipSuffix: @@ -852,7 +856,7 @@ proc parseExprStmt(p: var TParser): PNode = var call = if a.kind == nkCall: a else: newNode(nkCommand, a.info, @[a]) while true: - if not isExprStart(p): break + if not isExprStart(p): break var e = parseExpr(p) addSon(call, e) if p.tok.tokType != tkComma: break @@ -860,7 +864,7 @@ proc parseExprStmt(p: var TParser): PNode = optInd(p, a) if p.tok.tokType == tkDo: parseDoBlocks(p, call) - return call + return result = if call.sonsLen <= 1: a else: call if p.tok.tokType == tkColon: @@ -1548,14 +1552,14 @@ proc parseAll(p: var TParser): PNode = proc parseTopLevelStmt(p: var TParser): PNode = result = ast.emptyNode - while true: + while true: case p.tok.tokType of tkSad, tkSemicolon: getTok(p) of tkDed, tkInd: parMessage(p, errInvalidIndentation) getTok(p) of tkEof: break - else: + else: result = complexOrSimpleStmt(p) if result.kind == nkEmpty: parMessage(p, errExprExpected, p.tok) break diff --git a/tests/run/tdomulttest.nim b/tests/run/tdomulttest.nim index 76ab0963c..6842d38ed 100644 --- a/tests/run/tdomulttest.nim +++ b/tests/run/tdomulttest.nim @@ -1,6 +1,5 @@ discard """ - file: "tasynciossl.nim" - cmd: "nimrod cc --hints:on --define:ssl $# $#" + file: "tdomulttest.nim" output: "555\ntest\nmulti lines\n99999999\nend" """ proc foo(bar, baz: proc (x: int): int) = From 0c13512db954e5224dcfcc218ae4bd4f3df40f7a Mon Sep 17 00:00:00 2001 From: Araq Date: Mon, 21 Jan 2013 01:10:23 +0100 Subject: [PATCH 086/106] rst parser: show the beginning of the quote in the error message --- packages/docutils/rst.nim | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/packages/docutils/rst.nim b/packages/docutils/rst.nim index c752e5065..b22bdf6ce 100755 --- a/packages/docutils/rst.nim +++ b/packages/docutils/rst.nim @@ -313,6 +313,10 @@ proc rstMessage(p: TRstParser, msgKind: TMsgKind, arg: string) = p.s.msgHandler(p.filename, p.line + p.tok[p.idx].line, p.col + p.tok[p.idx].col, msgKind, arg) +proc rstMessage(p: TRstParser, msgKind: TMsgKind, arg: string, line, col: int) = + p.s.msgHandler(p.filename, p.line + line, + p.col + col, msgKind, arg) + proc rstMessage(p: TRstParser, msgKind: TMsgKind) = p.s.msgHandler(p.filename, p.line + p.tok[p.idx].line, p.col + p.tok[p.idx].col, msgKind, @@ -684,6 +688,9 @@ when false: proc parseUntil(p: var TRstParser, father: PRstNode, postfix: string, interpretBackslash: bool) = + let + line = p.tok[p.idx].line + col = p.tok[p.idx].col while true: case p.tok[p.idx].kind of tkPunct: @@ -707,7 +714,7 @@ proc parseUntil(p: var TRstParser, father: PRstNode, postfix: string, of tkWhite: add(father, newRstNode(rnLeaf, " ")) inc(p.idx) - else: rstMessage(p, meExpected, postfix) + else: rstMessage(p, meExpected, postfix, line, col) proc parseMarkdownCodeblock(p: var TRstParser): PRstNode = var args = newRstNode(rnDirArg) From 9aecfcd8c8bf96a3aaa77c1bb7c1c242aabd960f Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Mon, 21 Jan 2013 14:20:54 +0000 Subject: [PATCH 087/106] Fixed os.sleep not sleeping for long enough with values below 1000ms. --- lib/pure/os.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/pure/os.nim b/lib/pure/os.nim index 9513dbffb..01daf5ad6 100755 --- a/lib/pure/os.nim +++ b/lib/pure/os.nim @@ -1449,7 +1449,7 @@ proc sleep*(milsecs: int) {.rtl, extern: "nos$1", tags: [FTime].} = else: var a, b: Ttimespec a.tv_sec = TTime(milsecs div 1000) - a.tv_nsec = (milsecs mod 1000) * 1000 + a.tv_nsec = (milsecs mod 1000) * 1000 * 1000 discard posix.nanosleep(a, b) proc getFileSize*(file: string): biggestInt {.rtl, extern: "nos$1", From ddc8913ebbaba375010d415a5ff89fd1ee7fba96 Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Mon, 21 Jan 2013 14:44:23 +0000 Subject: [PATCH 088/106] Added a timeout parameter to irc.reconnect. --- lib/pure/irc.nim | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/lib/pure/irc.nim b/lib/pure/irc.nim index 8916681f7..aa4e2d557 100644 --- a/lib/pure/irc.nim +++ b/lib/pure/irc.nim @@ -32,7 +32,7 @@ ## **Warning:** The API of this module is unstable, and therefore is subject ## to change. -import sockets, strutils, parseutils, times, asyncio +import sockets, strutils, parseutils, times, asyncio, os type TIRC* = object of TObject @@ -56,6 +56,7 @@ type channelsToJoin: seq[string] msgLimit: bool messageBuffer: seq[tuple[timeToSend: float, m: string]] + lastReconnect: float PIRC* = ref TIRC @@ -235,12 +236,19 @@ proc connect*(irc: PIRC) = irc.send("NICK " & irc.nick, true) irc.send("USER $1 * 0 :$2" % [irc.user, irc.realname], true) -proc reconnect*(irc: PIRC) = +proc reconnect*(irc: PIRC, timeout = 5000) = ## Reconnects to an IRC server. ## + ## ``Timeout`` specifies the time to wait in miliseconds between multiple + ## consecutive reconnections. + ## ## This should be used when an ``EvDisconnected`` event occurs. + let secSinceReconnect = int(epochTime() - irc.lastReconnect) + if secSinceReconnect < timeout: + sleep(timeout - secSinceReconnect) irc.sock = socket() irc.connect() + irc.lastReconnect = epochTime() proc irc*(address: string, port: TPort = 6667.TPort, nick = "NimrodBot", @@ -409,15 +417,22 @@ proc connect*(irc: PAsyncIRC) = irc.asyncSock.connect(irc.address, irc.port) -proc reconnect*(irc: PAsyncIRC) = +proc reconnect*(irc: PAsyncIRC, timeout = 5000) = ## Reconnects to an IRC server. ## + ## ``Timeout`` specifies the time to wait in miliseconds between multiple + ## consecutive reconnections. + ## ## This should be used when an ``EvDisconnected`` event occurs. ## ## When successfully reconnected an ``EvConnected`` event will occur. + let secSinceReconnect = int(epochTime() - irc.lastReconnect) + if secSinceReconnect < timeout: + sleep(timeout - secSinceReconnect) irc.asyncSock = AsyncSocket() irc.myDispatcher.register(irc) irc.connect() + irc.lastReconnect = epochTime() proc asyncIRC*(address: string, port: TPort = 6667.TPort, nick = "NimrodBot", From 958f20aa1ab5c511de73eb397be5fa8d0f0ff085 Mon Sep 17 00:00:00 2001 From: Grzegorz Adam Hankiewicz Date: Mon, 21 Jan 2013 18:25:45 +0100 Subject: [PATCH 089/106] Documents two-variable for loop with sequences. --- doc/tut1.txt | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/doc/tut1.txt b/doc/tut1.txt index 2ae7e9540..aead6e36e 100755 --- a/doc/tut1.txt +++ b/doc/tut1.txt @@ -349,6 +349,7 @@ provides. The example uses the built-in ``countup`` iterator: echo("Counting to ten: ") for i in countup(1, 10): echo($i) + # --> Outputs 1 2 3 4 5 6 7 8 9 10 on different lines The built-in ``$`` operator turns an integer (``int``) and many other types into a string. The variable ``i`` is implicitly declared by the ``for`` loop @@ -362,6 +363,7 @@ the same: while i <= 10: echo($i) inc(i) # increment i by 1 + # --> Outputs 1 2 3 4 5 6 7 8 9 10 on different lines Counting down can be achieved as easily (but is less often needed): @@ -369,6 +371,7 @@ Counting down can be achieved as easily (but is less often needed): echo("Counting down from 10 to 1: ") for i in countdown(10, 1): echo($i) + # --> Outputs 10 9 8 7 6 5 4 3 2 1 on different lines Since counting up occurs so often in programs, Nimrod also has a ``..`` iterator that does the same: @@ -1202,6 +1205,28 @@ raised) for performance reasons. Thus one should use empty sequences ``@[]`` rather than ``nil`` as the *empty* value. But ``@[]`` creates a sequence object on the heap, so there is a trade-off to be made here. +The ``for`` statement can be used with one or two variables when used with a +sequence. When you use the one variable form, the variable will hold the value +provided by the sequence. The ``for`` statement is looping over the results +from the ``items()`` iterator from the `system `_ module. But if +you use the two variable form, the first variable will hold the index position +and the second variable will hold the value. Here the ``for`` statement is +looping over the results from the ``pairs()`` iterator from the `system +`_ module. Examples: + +.. code-block:: nimrod + for i in @[3, 4, 5]: + echo($i) + # --> 3 + # --> 4 + # --> 5 + + for i, value in @[3, 4, 5]: + echo("index: ", $i, ", value:", $value) + # --> index: 0, value:3 + # --> index: 1, value:4 + # --> index: 2, value:5 + Open arrays ----------- From acc394ca239b1a1ae864017c22aae8660b655053 Mon Sep 17 00:00:00 2001 From: Grzegorz Adam Hankiewicz Date: Mon, 21 Jan 2013 21:43:34 +0100 Subject: [PATCH 090/106] Adds some documentation related to exceptions. --- doc/manual.txt | 17 +++++++++++++++++ doc/tut2.txt | 14 ++++++++++++++ 2 files changed, 31 insertions(+) diff --git a/doc/manual.txt b/doc/manual.txt index 2a79c0f99..80a5642c5 100755 --- a/doc/manual.txt +++ b/doc/manual.txt @@ -2843,6 +2843,23 @@ in an implicit try block: finally: close(f) ... +The ``except`` statement has a limitation in this form: you can't specify the +type of the exception, you have to catch everything. Also, if you want to use +both ``finally`` and ``except`` you need to reverse the usual sequence of the +statements. Example: + +.. code-block:: nimrod + proc test() = + raise newException(E_base, "Hey ho") + + proc tester() = + finally: echo "3. Finally block" + except: echo "2. Except block" + echo "1. Pre exception" + test() + echo "4. Post exception" + # --> 1, 2, 3 is printed, 4 is never reached + Raise statement --------------- diff --git a/doc/tut2.txt b/doc/tut2.txt index 9f9dbe2db..4a4ef1757 100755 --- a/doc/tut2.txt +++ b/doc/tut2.txt @@ -433,6 +433,20 @@ handled, it is propagated through the call stack. This means that often the rest of the procedure - that is not within a ``finally`` clause - is not executed (if an exception occurs). +If you need to *access* the actual exception object or message inside an +``except`` branch you can use the getCurrentException() and +getCurrentExceptionMsg() procs from the `system `_ module. +Example: + +.. code-block:: nimrod + try: + doSomethingHere() + except: + let + e = getCurrentException() + msg = getCurrentExceptionMsg() + echo "Got exception ", repr(e), " with message ", msg + Exception hierarchy ------------------- From 0d978c4a9a2a6e476231d7bd70a3932af1e4c9c6 Mon Sep 17 00:00:00 2001 From: Araq Date: Tue, 22 Jan 2013 00:29:54 +0100 Subject: [PATCH 091/106] fixed some closure related bugs --- compiler/ccgexprs.nim | 2 +- compiler/lambdalifting.nim | 2 +- compiler/main.nim | 4 +- compiler/semexprs.nim | 4 +- compiler/semstmts.nim | 38 ++++--- tests/compile/tclosurebug2.nim | 194 +++++++++++++++++++++++++++++++++ tests/run/tdomulttest.nim | 1 + tests/run/tmultim6.nim | 2 +- todo.txt | 3 +- 9 files changed, 226 insertions(+), 24 deletions(-) create mode 100644 tests/compile/tclosurebug2.nim diff --git a/compiler/ccgexprs.nim b/compiler/ccgexprs.nim index 26144c152..7781040d9 100755 --- a/compiler/ccgexprs.nim +++ b/compiler/ccgexprs.nim @@ -1,7 +1,7 @@ # # # The Nimrod Compiler -# (c) Copyright 2012 Andreas Rumpf +# (c) Copyright 2013 Andreas Rumpf # # See the file "copying.txt", included in this # distribution, for details about the copyright. diff --git a/compiler/lambdalifting.nim b/compiler/lambdalifting.nim index bca5b8c1d..0c3eea3be 100644 --- a/compiler/lambdalifting.nim +++ b/compiler/lambdalifting.nim @@ -1,7 +1,7 @@ # # # The Nimrod Compiler -# (c) Copyright 2012 Andreas Rumpf +# (c) Copyright 2013 Andreas Rumpf # # See the file "copying.txt", included in this # distribution, for details about the copyright. diff --git a/compiler/main.nim b/compiler/main.nim index 0a76b967a..fd934e195 100755 --- a/compiler/main.nim +++ b/compiler/main.nim @@ -312,7 +312,9 @@ proc MainCommand = # no need to write rod files and would slow down things: #registerPass(rodwrite.rodwritePass()) discard CompileModule(options.libpath / "system", {sfSystemModule}) - service.serve(proc () = + # I'm lazy and misused this piece of code as a testcase so don't remove + # the invocation with a named parameter: + service.serve(action = proc () = let projectFile = mainCommandArg() discard CompileModule(projectFile, {sfMainModule}) ) diff --git a/compiler/semexprs.nim b/compiler/semexprs.nim index d0fbcab6f..4b4cb5bdf 100755 --- a/compiler/semexprs.nim +++ b/compiler/semexprs.nim @@ -1,7 +1,7 @@ # # # The Nimrod Compiler -# (c) Copyright 2012 Andreas Rumpf +# (c) Copyright 2013 Andreas Rumpf # # See the file "copying.txt", included in this # distribution, for details about the copyright. @@ -318,7 +318,7 @@ proc semIs(c: PContext, n: PNode): PNode = if not containsGenericType(t1): result = evalIsOp(n) proc semOpAux(c: PContext, n: PNode) = - let flags = {efDetermineType} + const flags = {efDetermineType} for i in countup(1, n.sonsLen- 1): var a = n.sons[i] if a.kind == nkExprEqExpr and sonsLen(a) == 2: diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index 363cff89a..5bd8462f3 100755 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -686,17 +686,18 @@ proc semLambda(c: PContext, n: PNode, flags: TExprFlags): PNode = if n.sons[pragmasPos].kind != nkEmpty: pragma(c, s, n.sons[pragmasPos], lambdaPragmas) s.options = gOptions - if n.sons[bodyPos].kind != nkEmpty: - if sfImportc in s.flags: + if n.sons[bodyPos].kind != nkEmpty: + if sfImportc in s.flags: LocalError(n.sons[bodyPos].info, errImplOfXNotAllowed, s.name.s) - if efDetermineType notin flags: - pushProcCon(c, s) - addResult(c, s.typ.sons[0], n.info, skProc) - let semBody = hloBody(c, semProcBody(c, n.sons[bodyPos])) - n.sons[bodyPos] = transformBody(c.module, semBody, s) - addResultNode(c, n) - popProcCon(c) - sideEffectsCheck(c, s) + #if efDetermineType notin flags: + # XXX not good enough + pushProcCon(c, s) + addResult(c, s.typ.sons[0], n.info, skProc) + let semBody = hloBody(c, semProcBody(c, n.sons[bodyPos])) + n.sons[bodyPos] = transformBody(c.module, semBody, s) + addResultNode(c, n) + popProcCon(c) + sideEffectsCheck(c, s) else: LocalError(n.info, errImplOfXexpected, s.name.s) closeScope(c.tab) # close scope for parameters @@ -706,13 +707,16 @@ proc semLambda(c: PContext, n: PNode, flags: TExprFlags): PNode = proc activate(c: PContext, n: PNode) = # XXX: This proc is part of my plan for getting rid of # forward declarations. stay tuned. - case n.kind - of nkLambdaKinds: - discard semLambda(c, n, {}) - of nkCallKinds: - for i in 1 .. counter) + result = (length * 2 < counter * 3) or (length - counter < 4) + +proc nextTry(h, maxHash: THash): THash {.inline.} = + result = ((5 * h) + 1) and maxHash + +template rawGetImpl() {.dirty.} = + var h: THash = hash(key) and high(t.data) # start with real hash value + while t.data[h].slot != seEmpty: + if t.data[h].key == key and t.data[h].slot == seFilled: + return h + h = nextTry(h, high(t.data)) + result = -1 + +template rawInsertImpl() {.dirty.} = + var h: THash = hash(key) and high(data) + while data[h].slot == seFilled: + h = nextTry(h, high(data)) + data[h].key = key + data[h].val = val + data[h].slot = seFilled + +template AddImpl() {.dirty.} = + if mustRehash(len(t.data), t.counter): Enlarge(t) + RawInsert(t, t.data, key, val) + inc(t.counter) + +template PutImpl() {.dirty.} = + var index = RawGet(t, key) + if index >= 0: + t.data[index].val = val + else: + AddImpl() + +proc len*[A, B](t: TOrderedTable[A, B]): int {.inline.} = + ## returns the number of keys in `t`. + result = t.counter + +template forAllOrderedPairs(yieldStmt: stmt) {.dirty, immediate.} = + var h = t.first + while h >= 0: + var nxt = t.data[h].next + if t.data[h].slot == seFilled: yieldStmt + h = nxt + +iterator pairs*[A, B](t: TOrderedTable[A, B]): tuple[key: A, val: B] = + ## iterates over any (key, value) pair in the table `t` in insertion + ## order. + forAllOrderedPairs: + yield (t.data[h].key, t.data[h].val) + +iterator mpairs*[A, B](t: var TOrderedTable[A, B]): tuple[key: A, val: var B] = + ## iterates over any (key, value) pair in the table `t` in insertion + ## order. The values can be modified. + forAllOrderedPairs: + yield (t.data[h].key, t.data[h].val) + +iterator keys*[A, B](t: TOrderedTable[A, B]): A = + ## iterates over any key in the table `t` in insertion order. + forAllOrderedPairs: + yield t.data[h].key + +iterator values*[A, B](t: TOrderedTable[A, B]): B = + ## iterates over any value in the table `t` in insertion order. + forAllOrderedPairs: + yield t.data[h].val + +iterator mvalues*[A, B](t: var TOrderedTable[A, B]): var B = + ## iterates over any value in the table `t` in insertion order. The values + ## can be modified. + forAllOrderedPairs: + yield t.data[h].val + +proc RawGet[A, B](t: TOrderedTable[A, B], key: A): int = + rawGetImpl() + +proc `[]`*[A, B](t: TOrderedTable[A, B], key: A): B = + ## retrieves the value at ``t[key]``. If `key` is not in `t`, + ## default empty value for the type `B` is returned + ## and no exception is raised. One can check with ``hasKey`` whether the key + ## exists. + var index = RawGet(t, key) + if index >= 0: result = t.data[index].val + +proc mget*[A, B](t: var TOrderedTable[A, B], key: A): var B = + ## retrieves the value at ``t[key]``. The value can be modified. + ## If `key` is not in `t`, the ``EInvalidKey`` exception is raised. + var index = RawGet(t, key) + if index >= 0: result = t.data[index].val + else: raise newException(EInvalidKey, "key not found: " & $key) + +proc hasKey*[A, B](t: TOrderedTable[A, B], key: A): bool = + ## returns true iff `key` is in the table `t`. + result = rawGet(t, key) >= 0 + +proc RawInsert[A, B](t: var TOrderedTable[A, B], + data: var TOrderedKeyValuePairSeq[A, B], + key: A, val: B) = + rawInsertImpl() + data[h].next = -1 + if t.first < 0: t.first = h + if t.last >= 0: data[t.last].next = h + t.last = h + +proc Enlarge[A, B](t: var TOrderedTable[A, B]) = + var n: TOrderedKeyValuePairSeq[A, B] + newSeq(n, len(t.data) * growthFactor) + var h = t.first + t.first = -1 + t.last = -1 + while h >= 0: + var nxt = t.data[h].next + if t.data[h].slot == seFilled: + RawInsert(t, n, t.data[h].key, t.data[h].val) + h = nxt + swap(t.data, n) + +proc `[]=`*[A, B](t: var TOrderedTable[A, B], key: A, val: B) = + ## puts a (key, value)-pair into `t`. + putImpl() + +proc add*[A, B](t: var TOrderedTable[A, B], key: A, val: B) = + ## puts a new (key, value)-pair into `t` even if ``t[key]`` already exists. + AddImpl() + +proc initOrderedTable*[A, B](initialSize=64): TOrderedTable[A, B] = + ## creates a new ordered hash table that is empty. `initialSize` needs to be + ## a power of two. + assert isPowerOfTwo(initialSize) + result.counter = 0 + result.first = -1 + result.last = -1 + newSeq(result.data, initialSize) + +proc toOrderedTable*[A, B](pairs: openarray[tuple[key: A, + val: B]]): TOrderedTable[A, B] = + ## creates a new ordered hash table that contains the given `pairs`. + result = initOrderedTable[A, B](nextPowerOfTwo(pairs.len+10)) + for key, val in items(pairs): result[key] = val + +proc sort*[A, B](t: var TOrderedTable[A,B], + cmp: proc (x, y: tuple[key: A, val: B]): int {.closure.}) = + ## sorts the ordered table so that the entry with the highest counter comes + ## first. This is destructive (with the advantage of being efficient)! + ## You must not modify `t` afterwards! + ## You can use the iterators `pairs`, `keys`, and `values` to iterate over + ## `t` in the sorted order. + + # we use shellsort here; fast enough and simple + var h = 1 + while true: + h = 3 * h + 1 + if h >= high(t.data): break + while true: + h = h div 3 + for i in countup(h, high(t.data)): + var j = i + #echo(t.data.len, " ", j, " - ", h) + #echo(repr(t.data[j-h])) + proc rawCmp(x, y: TOrderedKeyValuePair[A, B]): int = + if x.slot in {seEmpty, seDeleted} and y.slot in {seEmpty, seDeleted}: + return 0 + elif x.slot in {seEmpty, seDeleted}: + return -1 + elif y.slot in {seEmpty, seDeleted}: + return 1 + else: + let item1 = (x.key, x.val) + let item2 = (y.key, y.val) + return cmp(item1, item2) + + while rawCmp(t.data[j-h], t.data[j]) <= 0: + swap(t.data[j], t.data[j-h]) + j = j-h + if j < h: break + if h == 1: break diff --git a/tests/run/tdomulttest.nim b/tests/run/tdomulttest.nim index 6842d38ed..4ee6de128 100644 --- a/tests/run/tdomulttest.nim +++ b/tests/run/tdomulttest.nim @@ -1,6 +1,7 @@ discard """ file: "tdomulttest.nim" output: "555\ntest\nmulti lines\n99999999\nend" + disabled: true """ proc foo(bar, baz: proc (x: int): int) = echo bar(555) diff --git a/tests/run/tmultim6.nim b/tests/run/tmultim6.nim index a55b69e37..5f45f572a 100644 --- a/tests/run/tmultim6.nim +++ b/tests/run/tmultim6.nim @@ -1,5 +1,5 @@ discard """ - output: "collide: unit, thing | collide: unit, thing | collide: thing, unit" + output: "collide: unit, thing | collide: unit, thing | collide: thing, unit |" """ # Test multi methods diff --git a/todo.txt b/todo.txt index 56a27e0fa..c9510d9ca 100755 --- a/todo.txt +++ b/todo.txt @@ -3,8 +3,8 @@ version 0.9.2 - implement constructors + full 'not nil' checking - ``restrict`` pragma + backend support -- fix closure bug finally - fix marshal bug +- fix: 'result' is not properly cleaned for NRVO - investigate nimgame bug @@ -62,6 +62,7 @@ version 0.9.XX not in a 'let/var' context (p(a.openFile, b.openFile) makes no sense anyway) - document nimdoc properly finally - make 'clamp' a magic for the range stuff +- better type syntax for functions and tuples: tuple(int, int); (int,int)->int Not essential for 1.0.0 From 7d256d011e17a3046466cc9e74c1cdc2a9ca6ae0 Mon Sep 17 00:00:00 2001 From: Grzegorz Adam Hankiewicz Date: Tue, 22 Jan 2013 00:29:43 +0100 Subject: [PATCH 092/106] Fixes rst format for enumerated list in html output. --- doc/manual.txt | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/doc/manual.txt b/doc/manual.txt index 2a79c0f99..f4a1b5098 100755 --- a/doc/manual.txt +++ b/doc/manual.txt @@ -2682,16 +2682,14 @@ In contrast to that, a `closure iterator`:idx: can be passed around: invoke(count0) invoke(count2) - Closure iterators have other restrictions than inline iterators: -1.) ``yield`` in a closure iterator can not occur in a ``try`` statement. -2.) For now, a closure iterator cannot be evaluated at compile time. -3.) ``return`` is allowed in a closure iterator (but rarely useful). -4.) Since closure iterators can be used as a collaborative tasking - system, ``void`` is a valid return type for them. -5.) Both inline and closure iterators cannot be recursive. - +1. ``yield`` in a closure iterator can not occur in a ``try`` statement. +2. For now, a closure iterator cannot be evaluated at compile time. +3. ``return`` is allowed in a closure iterator (but rarely useful). +4. Since closure iterators can be used as a collaborative tasking + system, ``void`` is a valid return type for them. +5. Both inline and closure iterators cannot be recursive. Iterators that are neither marked ``{.closure.}`` nor ``{.inline.}`` explicitly default to being inline, but that this may change in future versions of the From 09e4b194f5cde00c8c23cf02292463b7c20939c7 Mon Sep 17 00:00:00 2001 From: Araq Date: Tue, 22 Jan 2013 00:37:33 +0100 Subject: [PATCH 093/106] bootstrapping works again --- compiler/main.nim | 4 +--- compiler/semstmts.nim | 2 +- tests/compile/tnamedparamanonproc.nim | 14 ++++++++++++++ 3 files changed, 16 insertions(+), 4 deletions(-) create mode 100644 tests/compile/tnamedparamanonproc.nim diff --git a/compiler/main.nim b/compiler/main.nim index fd934e195..0a76b967a 100755 --- a/compiler/main.nim +++ b/compiler/main.nim @@ -312,9 +312,7 @@ proc MainCommand = # no need to write rod files and would slow down things: #registerPass(rodwrite.rodwritePass()) discard CompileModule(options.libpath / "system", {sfSystemModule}) - # I'm lazy and misused this piece of code as a testcase so don't remove - # the invocation with a named parameter: - service.serve(action = proc () = + service.serve(proc () = let projectFile = mainCommandArg() discard CompileModule(projectFile, {sfMainModule}) ) diff --git a/compiler/semstmts.nim b/compiler/semstmts.nim index 5bd8462f3..e3956b918 100755 --- a/compiler/semstmts.nim +++ b/compiler/semstmts.nim @@ -690,7 +690,7 @@ proc semLambda(c: PContext, n: PNode, flags: TExprFlags): PNode = if sfImportc in s.flags: LocalError(n.sons[bodyPos].info, errImplOfXNotAllowed, s.name.s) #if efDetermineType notin flags: - # XXX not good enough + # XXX not good enough; see tnamedparamanonproc.nim pushProcCon(c, s) addResult(c, s.typ.sons[0], n.info, skProc) let semBody = hloBody(c, semProcBody(c, n.sons[bodyPos])) diff --git a/tests/compile/tnamedparamanonproc.nim b/tests/compile/tnamedparamanonproc.nim new file mode 100644 index 000000000..272b84e91 --- /dev/null +++ b/tests/compile/tnamedparamanonproc.nim @@ -0,0 +1,14 @@ + +type + PButton = ref object + TButtonClicked = proc(button: PButton) {.nimcall.} + +proc newButton*(onClick: TButtonClicked) = + nil + +proc main() = + newButton(onClick = proc(b: PButton) = + var requestomat = 12 + ) + +main() From 899193268aef7638c3cd14fb286ee59e1fae6e17 Mon Sep 17 00:00:00 2001 From: Araq Date: Tue, 22 Jan 2013 00:59:34 +0100 Subject: [PATCH 094/106] bugfix: ftpclient compiles again --- lib/pure/ftpclient.nim | 2 +- todo.txt | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/lib/pure/ftpclient.nim b/lib/pure/ftpclient.nim index e656d001e..b61793866 100644 --- a/lib/pure/ftpclient.nim +++ b/lib/pure/ftpclient.nim @@ -114,7 +114,7 @@ proc getDSock(ftp: PFTPClient): TSocket = proc getCSock(ftp: PFTPClient): TSocket = if ftp.isAsync: return ftp.asyncCSock else: return ftp.csock -template blockingOperation(sock: TSocket, body: stmt) = +template blockingOperation(sock: TSocket, body: stmt) {.immediate.} = if ftp.isAsync: sock.setBlocking(true) body diff --git a/todo.txt b/todo.txt index c9510d9ca..14f892b4e 100755 --- a/todo.txt +++ b/todo.txt @@ -5,7 +5,6 @@ version 0.9.2 - ``restrict`` pragma + backend support - fix marshal bug - fix: 'result' is not properly cleaned for NRVO -- investigate nimgame bug version 0.9.4 From 91700f29e718e41026b536d72b9ff010a4574bf2 Mon Sep 17 00:00:00 2001 From: Grzegorz Adam Hankiewicz Date: Tue, 22 Jan 2013 20:29:07 +0100 Subject: [PATCH 095/106] Renames each proc to map, each is left deprecated. --- doc/tut2.txt | 2 +- examples/maximum.nim | 2 +- lib/pure/collections/sequtils.nim | 4 +-- lib/system.nim | 44 ++++++++++++++++++++++++++++--- tests/compile/tclosure4.nim | 4 +-- tests/compile/toverprc.nim | 2 +- web/index.txt | 2 +- 7 files changed, 48 insertions(+), 12 deletions(-) diff --git a/doc/tut2.txt b/doc/tut2.txt index 9f9dbe2db..11b1b1dd5 100755 --- a/doc/tut2.txt +++ b/doc/tut2.txt @@ -218,7 +218,7 @@ So "pure object oriented" code is easy to write: import strutils stdout.writeln("Give a list of numbers (separated by spaces): ") - stdout.write(stdin.readLine.split.each(parseInt).max.`$`) + stdout.write(stdin.readLine.split.map(parseInt).max.`$`) stdout.writeln(" is the maximum!") diff --git a/examples/maximum.nim b/examples/maximum.nim index 1e26ee1a7..ac6160f76 100755 --- a/examples/maximum.nim +++ b/examples/maximum.nim @@ -3,4 +3,4 @@ import strutils echo "Give a list of numbers (separated by spaces): " -stdin.readLine.split.each(parseInt).max.`$`.echo(" is the maximum!") +stdin.readLine.split.map(parseInt).max.`$`.echo(" is the maximum!") diff --git a/lib/pure/collections/sequtils.nim b/lib/pure/collections/sequtils.nim index 73713eec9..298e7f27e 100644 --- a/lib/pure/collections/sequtils.nim +++ b/lib/pure/collections/sequtils.nim @@ -12,8 +12,8 @@ ## This module implements operations for the built-in `seq`:idx: type which ## were inspired by functional programming languages. If you are looking for ## the typical `map` function which applies a function to every element in a -## sequence, it already exists as the `each` proc in the `system -## `_ module in both mutable and immutable styles. +## sequence, it already exists in the `system `_ module in both +## mutable and immutable styles. ## ## Also, for functional style programming you may want to pass `anonymous procs ## `_ to procs like ``filter`` to reduce typing. diff --git a/lib/system.nim b/lib/system.nim index 26e5ac228..0b6c494a4 100755 --- a/lib/system.nim +++ b/lib/system.nim @@ -1460,15 +1460,51 @@ proc pop*[T](s: var seq[T]): T {.inline, noSideEffect.} = result = s[L] setLen(s, L) -proc each*[T, S](data: openArray[T], op: proc (x: T): S {.closure.}): seq[S] = - ## The well-known `map`:idx: operation from functional programming. Applies +proc each*[T, S](data: openArray[T], op: proc (x: T): S {.closure.}): seq[S] {. + deprecated.} = + ## The well-known ``map`` operation from functional programming. Applies ## `op` to every item in `data` and returns the result as a sequence. + ## + ## **Deprecated since version 0.9:** Use the ``map`` proc instead. newSeq(result, data.len) for i in 0..data.len-1: result[i] = op(data[i]) -proc each*[T](data: var openArray[T], op: proc (x: var T) {.closure.}) = - ## The well-known `map`:idx: operation from functional programming. Applies +proc each*[T](data: var openArray[T], op: proc (x: var T) {.closure.}) {. + deprecated.} = + ## The well-known ``map`` operation from functional programming. Applies ## `op` to every item in `data` modifying it directly. + ## + ## **Deprecated since version 0.9:** Use the ``map`` proc instead. + for i in 0..data.len-1: op(data[i]) + +proc map*[T, S](data: openArray[T], op: proc (x: T): S {.closure.}): seq[S] = + ## Returns a new sequence with the results of `op` applied to every item in + ## `data`. + ## + ## Since the input is not modified you can use this version of ``map`` to + ## transform the type of the elements in the input sequence. Example: + ## + ## .. code-block:: nimrod + ## let + ## a = @[1, 2, 3, 4] + ## b = map(a, proc(x: int): string = $x) + ## assert b == @["1", "2", "3", "4"] + newSeq(result, data.len) + for i in 0..data.len-1: result[i] = op(data[i]) + +proc map*[T](data: var openArray[T], op: proc (x: var T) {.closure.}) = + ## Applies `op` to every item in `data` modifying it directly. + ## + ## Note that this version of ``map`` requires your input and output types to + ## be the same, since they are modified in-place. Example: + ## + ## .. code-block:: nimrod + ## var a = @["1", "2", "3", "4"] + ## echo repr(a) + ## # --> ["1", "2", "3", "4"] + ## map(a, proc(x: var string) = x &= "42") + ## echo repr(a) + ## # --> ["142", "242", "342", "442"] for i in 0..data.len-1: op(data[i]) iterator fields*[T: tuple](x: T): TObject {. diff --git a/tests/compile/tclosure4.nim b/tests/compile/tclosure4.nim index 9584fa98a..8e08376b6 100644 --- a/tests/compile/tclosure4.nim +++ b/tests/compile/tclosure4.nim @@ -4,8 +4,8 @@ import json, tables proc run(json_params: TTable) = let json_elems = json_params["files"].elems # These fail compilation. - var files = each(json_elems, proc (x: PJsonNode): string = x.str) - #var files = json_elems.each do (x: PJsonNode) -> string: x.str + var files = map(json_elems, proc (x: PJsonNode): string = x.str) + #var files = json_elems.map do (x: PJsonNode) -> string: x.str echo "Hey!" when isMainModule: diff --git a/tests/compile/toverprc.nim b/tests/compile/toverprc.nim index 43271b684..f3aa66b80 100755 --- a/tests/compile/toverprc.nim +++ b/tests/compile/toverprc.nim @@ -21,7 +21,7 @@ proc takeParseInt(x: proc (y: string): int {.noSideEffect.}): int = result = x("123") echo "Give a list of numbers (separated by spaces): " -var x = stdin.readline.split.each(parseInt).max +var x = stdin.readline.split.map(parseInt).max echo x, " is the maximum!" echo "another number: ", takeParseInt(parseInt) diff --git a/web/index.txt b/web/index.txt index 207e425ae..99beb3743 100755 --- a/web/index.txt +++ b/web/index.txt @@ -33,7 +33,7 @@ from that model. # Prints the maximum integer from a list of integers # delimited by whitespace read from stdin. let tokens = stdin.readLine.split - echo tokens.each(parseInt).max, " is the maximum." + echo tokens.map(parseInt).max, " is the maximum." Nimrod is efficient From 351e66b76766bd5119b0af91b769fcc7768af097 Mon Sep 17 00:00:00 2001 From: Grzegorz Adam Hankiewicz Date: Tue, 22 Jan 2013 16:34:53 +0100 Subject: [PATCH 096/106] Moves addr out of tutorial into manual, indexing it too. --- doc/manual.txt | 27 +++++++++++++++++++++------ doc/tut1.txt | 16 ++++++---------- lib/system.nim | 3 ++- 3 files changed, 29 insertions(+), 17 deletions(-) diff --git a/doc/manual.txt b/doc/manual.txt index 2a79c0f99..6f76a1183 100755 --- a/doc/manual.txt +++ b/doc/manual.txt @@ -2232,7 +2232,6 @@ Type casts ---------- Example: - .. code-block:: nimrod cast[int](x) @@ -2243,11 +2242,27 @@ only needed for low-level programming and are inherently unsafe. The addr operator ----------------- -The `addr` operator returns the address of an l-value. If the -type of the location is ``T``, the `addr` operator result is -of the type ``ptr T``. Taking the address of an object that resides -on the stack is **unsafe**, as the pointer may live longer than the -object on the stack and can thus reference a non-existing object. +The `addr`:idx: operator returns the address of an l-value. If the type of the +location is ``T``, the `addr` operator result is of the type ``ptr T``. An +address is always an untraced reference. Taking the address of an object that +resides on the stack is **unsafe**, as the pointer may live longer than the +object on the stack and can thus reference a non-existing object. You can get +the address of variables, but you can't use it on variables declared through +``let`` statements: + +.. code-block:: nimrod + + let t1 = "Hello" + var + t2 = t1 + t3 : pointer = addr(t2) + echo repr(addr(t2)) + # --> ref 0x7fff6b71b670 --> 0x10bb81050"Hello" + echo cast[ptr string](t3)[] + # --> Hello + # The following line doesn't compile: + echo repr(addr(t1)) + # Error: expression has no address Procedures diff --git a/doc/tut1.txt b/doc/tut1.txt index 2ae7e9540..8edd5c33c 100755 --- a/doc/tut1.txt +++ b/doc/tut1.txt @@ -1303,14 +1303,10 @@ untraced references are *unsafe*. However for certain low-level operations Traced references are declared with the **ref** keyword, untraced references are declared with the **ptr** keyword. -The empty ``[]`` subscript notation can be used to *derefer* a reference, -meaning to retrieve the item the reference points to. The ``addr`` operator -returns the address of an item. An address is always an untraced reference: -``addr`` is an *unsafe* feature. - -The ``.`` (access a tuple/object field operator) -and ``[]`` (array/string/sequence index operator) operators perform implicit -dereferencing operations for reference types: +The empty ``[]`` subscript notation can be used to *derefer* a reference, +meaning to retrieve the item the reference points to. The ``.`` (access a +tuple/object field operator) and ``[]`` (array/string/sequence index operator) +operators perform implicit dereferencing operations for reference types: .. code-block:: nimrod @@ -1327,8 +1323,8 @@ dereferencing operations for reference types: To allocate a new traced object, the built-in procedure ``new`` has to be used. To deal with untraced memory, the procedures ``alloc``, ``dealloc`` and -``realloc`` can be used. The documentation of the system module contains -further information. +``realloc`` can be used. The documentation of the `system `_ +module contains further information. If a reference points to *nothing*, it has the value ``nil``. diff --git a/lib/system.nim b/lib/system.nim index 26e5ac228..130b8532f 100755 --- a/lib/system.nim +++ b/lib/system.nim @@ -38,7 +38,8 @@ type char* {.magic: Char.} ## built-in 8 bit character type (unsigned) string* {.magic: String.} ## built-in string type cstring* {.magic: Cstring.} ## built-in cstring (*compatible string*) type - pointer* {.magic: Pointer.} ## built-in pointer type + pointer* {.magic: Pointer.} ## built-in pointer type, use the ``addr`` + ## operator to get a pointer to a variable const on* = true ## alias for ``true`` From 5aced9186d2edb6010d7d2bd0fdb1dfa317d73b8 Mon Sep 17 00:00:00 2001 From: Grzegorz Adam Hankiewicz Date: Tue, 15 Jan 2013 20:54:35 +0100 Subject: [PATCH 097/106] Adds randomize(seed) for repeatable pseudo random numbers. Also fixes srand48() type to clong. --- lib/pure/math.nim | 31 +++++++++++++++++++++++++++---- 1 file changed, 27 insertions(+), 4 deletions(-) diff --git a/lib/pure/math.nim b/lib/pure/math.nim index 53594db62..f9ab6d0f8 100755 --- a/lib/pure/math.nim +++ b/lib/pure/math.nim @@ -141,6 +141,11 @@ proc randomize*() ## number, i.e. a tickcount. Note: Does nothing for the ECMAScript target, ## as ECMAScript does not support this. +proc randomize*(seed: int) + ## initializes the random number generator with a specific seed. + ## Note: Does nothing for the ECMAScript target, + ## as ECMAScript does not support this. + when not defined(ECMAScript): proc sqrt*(x: float): float {.importc: "sqrt", header: "".} ## computes the square root of `x`. @@ -190,15 +195,17 @@ when not defined(ECMAScript): proc rand(): cint {.importc: "rand", nodecl.} when not defined(windows): - proc srand48(seed: cint) {.importc: "srand48", nodecl.} + proc srand48(seed: clong) {.importc: "srand48", nodecl.} proc drand48(): float {.importc: "drand48", nodecl.} proc random(max: float): float = result = drand48() * max proc randomize() = - let x = gettime(nil) - srand(x) - when defined(srand48): srand48(x) + randomize(gettime(nil)) + + proc randomize(seed: int) = + srand(cint(seed)) + when defined(srand48): srand48(seed) proc random(max: int): int = result = int(rand()) mod max @@ -217,6 +224,7 @@ else: proc random(max: float): float = result = float(mathrandom() * float(max)) proc randomize() = nil + proc randomize(seed: int) = nil proc sqrt*(x: float): float {.importc: "Math.sqrt", nodecl.} proc ln*(x: float): float {.importc: "Math.log", nodecl.} @@ -301,3 +309,18 @@ proc standardDeviation*(s: TRunningStat): float = {.pop.} {.pop.} + +when isMainModule and not defined(ECMAScript): + # Verifies random seed initialization. + let seed = gettime(nil) + randomize(seed) + const SIZE = 10 + var buf : array[0..SIZE, int] + # Fill the buffer with random values + for i in 0..SIZE-1: + buf[i] = random(high(int)) + # Check that the second random calls are the same for each position. + randomize(seed) + for i in 0..SIZE-1: + assert buf[i] == random(high(int)), "non deterministic random seeding" + echo "random values equal after reseeding" From 0e5e852f5cca7c4f03b4294120b6c0bb551e3211 Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Fri, 25 Jan 2013 21:43:20 +0000 Subject: [PATCH 098/106] Added some tooltip functions to the gtk wrapper. --- lib/wrappers/gtk/gtk2.nim | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/lib/wrappers/gtk/gtk2.nim b/lib/wrappers/gtk/gtk2.nim index 7094bfd5d..04a5bbffb 100755 --- a/lib/wrappers/gtk/gtk2.nim +++ b/lib/wrappers/gtk/gtk2.nim @@ -16598,6 +16598,7 @@ proc message_dialog_new*(parent: PWindow, flags: TDialogFlags, cdecl, importc: "gtk_message_dialog_new", dynlib: lib.} proc set_markup*(msgDialog: PMessageDialog, str: cstring) {.cdecl, importc: "gtk_message_dialog_set_markup", dynlib: lib.} + proc signal_new*(name: cstring, signal_flags: TSignalRunType, object_type: TType, function_offset: guint, marshaller: TSignalMarshaller, return_val: TType, n_args: guint): guint{. @@ -16895,6 +16896,15 @@ type proc set_tooltip_text*(w: PWidget, t: cstring){.cdecl, dynlib: lib, importc: "gtk_widget_set_tooltip_text".} +proc get_tooltip_text*(w: PWidget): cstring{.cdecl, + dynlib: lib, importc: "gtk_widget_get_tooltip_text".} + +proc set_tooltip_markup*(w: PWidget, m: cstring) {.cdecl, dynlib: lib, + importc: "gtk_widget_set_tooltip_markup".} + +proc get_tooltip_markup*(w: PWidget): cstring {.cdecl, dynlib: lib, + importc: "gtk_widget_get_tooltip_markup".} + proc set_tooltip_column*(w: PTreeview, column: gint){.cdecl, dynlib: lib, importc: "gtk_tree_view_set_tooltip_column".} @@ -16907,6 +16917,9 @@ proc trigger_tooltip_query*(widg: PTooltip){.cdecl, dynlib: lib, proc set_has_tooltip*(widget: PWidget, b: gboolean){.cdecl, dynlib: lib, importc: "gtk_widget_set_has_tooltip".} +proc get_has_tooltip*(widget: PWidget): gboolean{.cdecl, dynlib: lib, + importc: "gtk_widget_get_has_tooltip".} + proc set_markup*(tp: PTooltip, mk: cstring){.cdecl, dynlib: lib, importc: "gtk_tooltip_set_markup".} From c4743805d9a45cf4bd2fe1535e29a1b953cfe876 Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Fri, 25 Jan 2013 21:43:54 +0000 Subject: [PATCH 099/106] Added strutils.unescape and fixed issue with strutils.escape. --- lib/pure/strutils.nim | 40 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/lib/pure/strutils.nim b/lib/pure/strutils.nim index 8a5061037..8b64434d8 100755 --- a/lib/pure/strutils.nim +++ b/lib/pure/strutils.nim @@ -850,13 +850,51 @@ proc escape*(s: string, prefix = "\"", suffix = "\""): string {.noSideEffect, for c in items(s): case c of '\0'..'\31', '\128'..'\255': - add(result, '\\') + add(result, "\\x") add(result, toHex(ord(c), 2)) of '\\': add(result, "\\\\") of '\'': add(result, "\\'") of '\"': add(result, "\\\"") else: add(result, c) add(result, suffix) + +proc unescape*(s: string, prefix = "\"", suffix = "\""): string {.noSideEffect, + rtl, extern: "nsuUnescape".} = + ## Unescapes a string `s`. This complements ``escape`` as it performs the + ## opposite operations. + ## + ## If `s` does not begin with ``prefix`` and end with ``suffix`` a EInvalidValue + ## exception will be raised. + result = newStringOfCap(s.len) + var i = 0 + if s[0 .. prefix.len-1] != prefix: + raise newException(EInvalidValue, + "String does not start with a prefix of: " & prefix) + i.inc() + while True: + if i == s.len-suffix.len: break + case s[i] + of '\\': + case s[i+1]: + of 'x': + let j = parseHexInt(s[i+2 .. i+3]) + result.add(chr(j)) + inc(i, 2) + of '\\': + result.add('\\') + of '\'': + result.add('\'') + of '\"': + result.add('\"') + else: result.add("\\" & s[i+1]) + inc(i) + of '\0': break + else: + result.add(s[i]) + i.inc() + if s[i .. -1] != suffix: + raise newException(EInvalidValue, + "String does not end with a suffix of: " & suffix) proc validIdentifier*(s: string): bool {.noSideEffect, rtl, extern: "nsuValidIdentifier".} = From bb38420ac77b4913f1810a98b3b4d48e7f411158 Mon Sep 17 00:00:00 2001 From: Dominik Picheta Date: Sat, 26 Jan 2013 22:19:48 +0000 Subject: [PATCH 100/106] Added gtk_window_is_active to gtk wrapper. --- lib/wrappers/gtk/gtk2.nim | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/wrappers/gtk/gtk2.nim b/lib/wrappers/gtk/gtk2.nim index 04a5bbffb..6b418024e 100755 --- a/lib/wrappers/gtk/gtk2.nim +++ b/lib/wrappers/gtk/gtk2.nim @@ -17050,6 +17050,10 @@ proc remove*(combo_box: PComboBoxText; position: gint){.cdecl, importc: "gtk_combo_box_text_remove", dynlib: lib.} proc get_active_text*(combo_box: PComboBoxText): cstring{.cdecl, importc: "gtk_combo_box_text_get_active_text", dynlib: lib.} +proc is_active*(win: PWindow): gboolean{.cdecl, + importc: "gtk_window_is_active", dynlib: lib.} +proc has_toplevel_focus*(win: PWindow): gboolean{.cdecl, + importc: "gtk_window_has_toplevel_focus", dynlib: lib.} proc nimrod_init*() = var From 0ba7da62ddf431800221edd3611d1673724cd8d2 Mon Sep 17 00:00:00 2001 From: Simon Hafner Date: Sat, 26 Jan 2013 16:44:49 -0600 Subject: [PATCH 101/106] no symbol shall be generated for imported procs --- compiler/ecmasgen.nim | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compiler/ecmasgen.nim b/compiler/ecmasgen.nim index 8899c904e..773729681 100755 --- a/compiler/ecmasgen.nim +++ b/compiler/ecmasgen.nim @@ -940,7 +940,7 @@ proc genSym(p: var TProc, n: PNode, r: var TCompRes) = of skProc, skConverter, skMethod: discard mangleName(s) r.res = s.loc.r - if lfNoDecl in s.loc.flags or s.magic != mNone or isGenericRoutine(s): nil + if lfNoDecl in s.loc.flags or s.magic != mNone or isGenericRoutine(s) or {sfImportc, sfInfixCall} * s.flags != {}: nil elif s.kind == skMethod and s.getBody.kind == nkEmpty: # we cannot produce code for the dispatcher yet: nil From c77ae9e77d7f0bc5ab5202e9ca1327ddd2a954c0 Mon Sep 17 00:00:00 2001 From: Araq Date: Sun, 27 Jan 2013 00:40:34 +0100 Subject: [PATCH 102/106] fixes #319 --- compiler/ecmasgen.nim | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/compiler/ecmasgen.nim b/compiler/ecmasgen.nim index 8899c904e..f3f716a01 100755 --- a/compiler/ecmasgen.nim +++ b/compiler/ecmasgen.nim @@ -209,6 +209,24 @@ proc genObjectInfo(p: var TProc, typ: PType, name: PRope) = appf(p.g.typeInfo, "$1.base = $2;$n", [name, genTypeInfo(p, typ.sons[0])]) +proc genTupleFields(p: var TProc, typ: PType): PRope = + var s: PRope = nil + for i in 0 .. 0: app(s, ", " & tnl) + s.appf("{kind: 1, offset: \"Field$1\", len: 0, " & + "typ: $2, name: \"Field$1\", sons: null}", + [i.toRope, genTypeInfo(p, typ.sons[i])]) + result = ropef("{kind: 2, len: $1, offset: 0, " & + "typ: null, name: null, sons: [$2]}", [toRope(typ.len), s]) + +proc genTupleInfo(p: var TProc, typ: PType, name: PRope) = + var s = ropef("var $1 = {size: 0, kind: $2, base: null, node: null, " & + "finalizer: null};$n", [name, toRope(ord(typ.kind))]) + prepend(p.g.typeInfo, s) + appf(p.g.typeInfo, "var NNI$1 = $2;$n", + [toRope(typ.id), genTupleFields(p, typ)]) + appf(p.g.typeInfo, "$1.node = NNI$2;$n", [name, toRope(typ.id)]) + proc genEnumInfo(p: var TProc, typ: PType, name: PRope) = var s, n: PRope @@ -261,7 +279,8 @@ proc genTypeInfo(p: var TProc, typ: PType): PRope = appf(p.g.typeInfo, "$1.base = $2;$n", [result, genTypeInfo(p, typ.sons[1])]) of tyEnum: genEnumInfo(p, t, result) - of tyObject, tyTuple: genObjectInfo(p, t, result) + of tyObject: genObjectInfo(p, t, result) + of tyTuple: genTupleInfo(p, t, result) else: InternalError("genTypeInfo(" & $t.kind & ')') proc gen(p: var TProc, n: PNode, r: var TCompRes) From 541ba485c265c3b7b5f9908f0def0c38550f6b23 Mon Sep 17 00:00:00 2001 From: Araq Date: Sun, 27 Jan 2013 00:42:14 +0100 Subject: [PATCH 103/106] incompleteStruct pragma for C's FILE --- lib/system.nim | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/system.nim b/lib/system.nim index 26e5ac228..f32b575e9 100755 --- a/lib/system.nim +++ b/lib/system.nim @@ -1,7 +1,7 @@ # # # Nimrod's Runtime Library -# (c) Copyright 2012 Andreas Rumpf +# (c) Copyright 2013 Andreas Rumpf # # See the file "copying.txt", included in this # distribution, for details about the copyright. @@ -1760,8 +1760,7 @@ when not defined(EcmaScript): #and not defined(NimrodVM): # ----------------- IO Part ------------------------------------------------ type - CFile {.importc: "FILE", nodecl, final.} = object # empty record for - # data hiding + CFile {.importc: "FILE", nodecl, final, incompletestruct.} = object TFile* = ptr CFile ## The type representing a file handle. TFileMode* = enum ## The file mode when opening a file. From 18534b7dde8686f6aacf74a33b24facb0517d705 Mon Sep 17 00:00:00 2001 From: Araq Date: Sun, 27 Jan 2013 00:42:43 +0100 Subject: [PATCH 104/106] incompleteStruct pragma for C's FILE --- lib/system/ansi_c.nim | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/system/ansi_c.nim b/lib/system/ansi_c.nim index f89577553..33e1ea982 100755 --- a/lib/system/ansi_c.nim +++ b/lib/system/ansi_c.nim @@ -21,9 +21,8 @@ proc c_strlen(a: CString): int {.nodecl, noSideEffect, importc: "strlen".} proc c_memset(p: pointer, value: cint, size: int) {.nodecl, importc: "memset".} type - C_TextFile {.importc: "FILE", nodecl, final.} = object # empty record for - # data hiding - C_BinaryFile {.importc: "FILE", nodecl, final.} = object + C_TextFile {.importc: "FILE", nodecl, final, incompleteStruct.} = object + C_BinaryFile {.importc: "FILE", nodecl, final, incompleteStruct.} = object C_TextFileStar = ptr CTextFile C_BinaryFileStar = ptr CBinaryFile From 0e1b67cfff7f9e7352439888e06e2c255599e193 Mon Sep 17 00:00:00 2001 From: Araq Date: Sun, 27 Jan 2013 01:38:55 +0100 Subject: [PATCH 105/106] implemented $/repr for enums for the JS target --- compiler/ecmasgen.nim | 34 ++++++++++++++++++---------------- 1 file changed, 18 insertions(+), 16 deletions(-) diff --git a/compiler/ecmasgen.nim b/compiler/ecmasgen.nim index 6ecda9a7c..9883acf6d 100755 --- a/compiler/ecmasgen.nim +++ b/compiler/ecmasgen.nim @@ -227,27 +227,24 @@ proc genTupleInfo(p: var TProc, typ: PType, name: PRope) = [toRope(typ.id), genTupleFields(p, typ)]) appf(p.g.typeInfo, "$1.node = NNI$2;$n", [name, toRope(typ.id)]) -proc genEnumInfo(p: var TProc, typ: PType, name: PRope) = - var - s, n: PRope - length: int - field: PSym - length = sonsLen(typ.n) - s = nil +proc genEnumInfo(p: var TProc, typ: PType, name: PRope) = + let length = sonsLen(typ.n) + var s: PRope = nil for i in countup(0, length - 1): if (typ.n.sons[i].kind != nkSym): InternalError(typ.n.info, "genEnumInfo") - field = typ.n.sons[i].sym + let field = typ.n.sons[i].sym if i > 0: app(s, ", " & tnl) + let extName = if field.ast == nil: field.name.s else: field.ast.strVal appf(s, "{kind: 1, offset: $1, typ: $2, name: $3, len: 0, sons: null}", - [toRope(field.position), name, makeCString(field.name.s)]) - n = ropef("var NNI$1 = {kind: 2, offset: 0, typ: null, " & + [toRope(field.position), name, makeCString(extName)]) + var n = ropef("var NNI$1 = {kind: 2, offset: 0, typ: null, " & "name: null, len: $2, sons: [$3]};$n", [toRope(typ.id), toRope(length), s]) s = ropef("var $1 = {size: 0, kind: $2, base: null, node: null, " & "finalizer: null};$n", [name, toRope(ord(typ.kind))]) prepend(p.g.typeInfo, s) app(p.g.typeInfo, n) appf(p.g.typeInfo, "$1.node = NNI$2;$n", [name, toRope(typ.id)]) - if typ.sons[0] != nil: + if typ.sons[0] != nil: appf(p.g.typeInfo, "$1.base = $2;$n", [name, genTypeInfo(p, typ.sons[0])]) @@ -959,7 +956,9 @@ proc genSym(p: var TProc, n: PNode, r: var TCompRes) = of skProc, skConverter, skMethod: discard mangleName(s) r.res = s.loc.r - if lfNoDecl in s.loc.flags or s.magic != mNone or isGenericRoutine(s) or {sfImportc, sfInfixCall} * s.flags != {}: nil + if lfNoDecl in s.loc.flags or s.magic != mNone or isGenericRoutine(s) or + {sfImportc, sfInfixCall} * s.flags != {}: + nil elif s.kind == skMethod and s.getBody.kind == nkEmpty: # we cannot produce code for the dispatcher yet: nil @@ -1218,10 +1217,13 @@ proc genConStrStr(p: var TProc, n: PNode, r: var TCompRes) = proc genRepr(p: var TProc, n: PNode, r: var TCompRes) = var t = skipTypes(n.sons[1].typ, abstractVarRange) case t.kind - of tyInt..tyInt64: - unaryExpr(p, n, r, "", "reprInt($1)") + of tyInt..tyUInt64: + unaryExpr(p, n, r, "", "(\"\"+ ($1))") of tyEnum, tyOrdinal: - binaryExpr(p, n, r, "", "reprEnum($1, $2)") + gen(p, n.sons[1], r) + useMagic(p, "cstrToNimstr") + r.res = ropef("cstrToNimstr($1.node.sons[$2].name)", + [genTypeInfo(p, t), r.res]) else: # XXX: internalError(n.info, "genRepr: Not implemented") @@ -1603,7 +1605,7 @@ proc newModule(module: PSym, filename: string): BModule = proc genHeader(): PRope = result = ropef("/* Generated by the Nimrod Compiler v$1 */$n" & - "/* (c) 2012 Andreas Rumpf */$n$n" & "$nvar Globals = this;$n" & + "/* (c) 2013 Andreas Rumpf */$n$n" & "$nvar Globals = this;$n" & "var framePtr = null;$n" & "var excHandler = null;$n", [toRope(versionAsString)]) From 07585088955c1fe8fb815c40409ed9f5d66fd446 Mon Sep 17 00:00:00 2001 From: Araq Date: Sun, 27 Jan 2013 19:15:13 +0100 Subject: [PATCH 106/106] bugfix: typeinfo.extendSeq --- lib/core/typeinfo.nim | 13 ++++++++----- lib/pure/htmlparser.nim | 6 +++--- lib/pure/marshal.nim | 2 +- lib/system/debugger.nim | 39 +++++++++++++++++++++++++-------------- todo.txt | 7 +++---- 5 files changed, 40 insertions(+), 27 deletions(-) diff --git a/lib/core/typeinfo.nim b/lib/core/typeinfo.nim index 26a14f444..eb2a0c9e5 100755 --- a/lib/core/typeinfo.nim +++ b/lib/core/typeinfo.nim @@ -1,7 +1,7 @@ # # # Nimrod's Runtime Library -# (c) Copyright 2012 Dominik Picheta, Andreas Rumpf +# (c) Copyright 2013 Dominik Picheta, Andreas Rumpf # # See the file "copying.txt", included in this # distribution, for details about the copyright. @@ -139,12 +139,15 @@ proc invokeNewSeq*(x: TAny, len: int) = var z = newSeq(x.rawType, len) genericShallowAssign(x.value, addr(z), x.rawType) -proc extendSeq*(x: TAny, elems = 1) = - ## performs ``setLen(x, x.len+elems)``. `x` needs to represent a ``seq``. +proc extendSeq*(x: TAny) = + ## performs ``setLen(x, x.len+1)``. `x` needs to represent a ``seq``. assert x.rawType.kind == tySequence var y = cast[ptr PGenSeq](x.value)[] - var z = incrSeq(y, x.rawType.base.size * elems) - genericShallowAssign(x.value, addr(z), x.rawType) + var z = incrSeq(y, x.rawType.base.size) + # 'incrSeq' already freed the memory for us and copied over the RC! + # So we simply copy the raw pointer into 'x.value': + cast[ppointer](x.value)[] = z + #genericShallowAssign(x.value, addr(z), x.rawType) proc setObjectRuntimeType*(x: TAny) = ## this needs to be called to set `x`'s runtime object type field. diff --git a/lib/pure/htmlparser.nim b/lib/pure/htmlparser.nim index 4136ecf57..ad952fc41 100755 --- a/lib/pure/htmlparser.nim +++ b/lib/pure/htmlparser.nim @@ -123,7 +123,7 @@ type tagVar ## the HTML ``var`` element const - tagStrs = [ + tagToStr* = [ "a", "abbr", "acronym", "address", "applet", "area", "b", "base", "basefont", "bdo", "big", "blockquote", "body", "br", "button", "caption", "center", "cite", "code", @@ -243,13 +243,13 @@ proc binaryStrSearch(x: openarray[string], y: string): int = proc htmlTag*(n: PXmlNode): THtmlTag = ## gets `n`'s tag as a ``THtmlTag``. if n.clientData == 0: - n.clientData = binaryStrSearch(tagStrs, n.tag)+1 + n.clientData = binaryStrSearch(tagToStr, n.tag)+1 result = THtmlTag(n.clientData) proc htmlTag*(s: string): THtmlTag = ## converts `s` to a ``THtmlTag``. If `s` is no HTML tag, ``tagUnknown`` is ## returned. - result = THtmlTag(binaryStrSearch(tagStrs, s.toLower)+1) + result = THtmlTag(binaryStrSearch(tagToStr, s.toLower)+1) proc entityToUtf8*(entity: string): string = ## converts an HTML entity name like ``Ü`` to its UTF-8 equivalent. diff --git a/lib/pure/marshal.nim b/lib/pure/marshal.nim index bffc4ebb6..f52b554cc 100755 --- a/lib/pure/marshal.nim +++ b/lib/pure/marshal.nim @@ -1,7 +1,7 @@ # # # Nimrod's Runtime Library -# (c) Copyright 2012 Andreas Rumpf +# (c) Copyright 2013 Andreas Rumpf # # See the file "copying.txt", included in this # distribution, for details about the copyright. diff --git a/lib/system/debugger.nim b/lib/system/debugger.nim index 68fbccef6..f234c9daf 100755 --- a/lib/system/debugger.nim +++ b/lib/system/debugger.nim @@ -1,7 +1,7 @@ # # # Nimrod's Runtime Library -# (c) Copyright 2012 Andreas Rumpf +# (c) Copyright 2013 Andreas Rumpf # # See the file "copying.txt", included in this # distribution, for details about the copyright. @@ -579,6 +579,10 @@ proc hash(Data: Pointer, Size: int): THash = Dec(s) result = !$h +proc hashGcHeader(data: pointer): THash = + const headerSize = sizeof(int)*2 + result = hash(cast[pointer](cast[int](data) -% headerSize), headerSize) + proc genericHashAux(dest: Pointer, mt: PNimType, shallow: bool, h: THash): THash proc genericHashAux(dest: Pointer, n: ptr TNimNode, shallow: bool, @@ -606,20 +610,22 @@ proc genericHashAux(dest: Pointer, mt: PNimType, shallow: bool, result = h if x != nil: let s = cast[NimString](x) - when true: - result = result !& hash(x, s.len) + when defined(trackGcHeaders): + result = result !& hashGcHeader(x) else: - let y = cast[pointer](cast[int](x) -% 2*sizeof(int)) - result = result !& hash(y, s.len + 2*sizeof(int)) + result = result !& hash(x, s.len) of tySequence: var x = cast[ppointer](dest) var dst = cast[taddress](cast[ppointer](dest)[]) result = h if dst != 0: - for i in 0..cast[pgenericseq](dst).len-1: - result = result !& genericHashAux( - cast[pointer](dst +% i*% mt.base.size +% GenericSeqSize), - mt.Base, shallow, result) + when defined(trackGcHeaders): + result = result !& hashGcHeader(cast[ppointer](dest)[]) + else: + for i in 0..cast[pgenericseq](dst).len-1: + result = result !& genericHashAux( + cast[pointer](dst +% i*% mt.base.size +% GenericSeqSize), + mt.Base, shallow, result) of tyObject, tyTuple: # we don't need to copy m_type field for tyObject, as they are equal anyway result = genericHashAux(dest, mt.node, shallow, h) @@ -630,13 +636,18 @@ proc genericHashAux(dest: Pointer, mt: PNimType, shallow: bool, result = result !& genericHashAux(cast[pointer](d +% i*% mt.base.size), mt.base, shallow, result) of tyRef: - if shallow: - result = h !& hash(dest, mt.size) - else: - result = h + when defined(trackGcHeaders): var s = cast[ppointer](dest)[] if s != nil: - result = result !& genericHashAux(s, mt.base, shallow, result) + result = result !& hashGcHeader(s) + else: + if shallow: + result = h !& hash(dest, mt.size) + else: + result = h + var s = cast[ppointer](dest)[] + if s != nil: + result = result !& genericHashAux(s, mt.base, shallow, result) # hash the object header: #const headerSize = sizeof(int)*2 #result = result !& hash(cast[pointer](cast[int](s) -% headerSize), diff --git a/todo.txt b/todo.txt index 14f892b4e..7da08f2bb 100755 --- a/todo.txt +++ b/todo.txt @@ -3,9 +3,8 @@ version 0.9.2 - implement constructors + full 'not nil' checking - ``restrict`` pragma + backend support -- fix marshal bug - fix: 'result' is not properly cleaned for NRVO - +- fix: exhaustive checking in case statements version 0.9.4 ============= @@ -13,6 +12,8 @@ version 0.9.4 - make 'bind' default for templates and introduce 'mixin'; special rule for ``[]=`` - implicit deref for parameter matching; overloading based on 'var T' +- ``=`` should be overloadable; requires specialization for ``=`` +- optimize genericAssign in the code generator version 0.9.X @@ -25,8 +26,6 @@ version 0.9.X * nested iterators - implement the missing features wrt inheritance - improve the compiler as a service -- ``=`` should be overloadable; requires specialization for ``=`` -- optimize genericAssign in the code generator - better support for macros that rewrite procs - macros need access to types and symbols (partially implemented) - result = result shr 8 for the "system()" wrapper
TestExpected