jyapayne/Nim
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Don't include the nimhcr dev docs in the system module documentation (#10759)

Browse source
This commit is contained in:
zah 2019-03-08 16:23:36 +02:00 • committed by Andreas Rumpf
commit b9f8528db6
5 changed files with 186 additions and 188 deletions
Download patch file Download diff file Expand all files Collapse all files

5
compiler/docgen.nim
View file

@ -992,10 +992,13 @@ proc writeOutputJson*(d: PDoc, useWarning = false) =
warnUser, "unable to open file \"" & d.destFile.string & warnUser, "unable to open file \"" & d.destFile.string &
"\" for writing") "\" for writing")
proc commandDoc*(cache: IdentCache, conf: ConfigRef) = proc handleDocOutputOptions*(conf: ConfigRef) =
if optWholeProject in conf.globalOptions: if optWholeProject in conf.globalOptions:
# Backward compatibility with previous versions # Backward compatibility with previous versions
conf.outDir = AbsoluteDir(conf.outDir / conf.outFile) conf.outDir = AbsoluteDir(conf.outDir / conf.outFile)
proc commandDoc*(cache: IdentCache, conf: ConfigRef) =
handleDocOutputOptions conf
var ast = parseFile(conf.projectMainIdx, cache, conf) var ast = parseFile(conf.projectMainIdx, cache, conf)
if ast == nil: return if ast == nil: return
var d = newDocumentor(conf.projectFull, cache, conf) var d = newDocumentor(conf.projectFull, cache, conf)

4
compiler/main.nim
View file

@ -63,9 +63,7 @@ proc commandCheck(graph: ModuleGraph) =
when not defined(leanCompiler): when not defined(leanCompiler):
proc commandDoc2(graph: ModuleGraph; json: bool) = proc commandDoc2(graph: ModuleGraph; json: bool) =
if optWholeProject in graph.config.globalOptions: handleDocOutputOptions graph.config
# Backward compatibility with previous versions
graph.config.outDir = AbsoluteDir(graph.config.outDir / graph.config.outFile)
graph.config.errorMax = high(int) # do not stop after first error graph.config.errorMax = high(int) # do not stop after first error
semanticPasses(graph) semanticPasses(graph)
if json: registerPass(graph, docgen2JsonPass) if json: registerPass(graph, docgen2JsonPass)

3
compiler/options.nim
View file

@ -454,8 +454,7 @@ proc setConfigVar*(conf: ConfigRef; key, val: string) =
conf.configVars[key] = val conf.configVars[key] = val
proc getOutFile*(conf: ConfigRef; filename: RelativeFile, ext: string): AbsoluteFile = proc getOutFile*(conf: ConfigRef; filename: RelativeFile, ext: string): AbsoluteFile =
result = (if conf.outFile.isEmpty: conf.projectPath else: conf.outDir) / conf.outDir / changeFileExt(filename, ext)
changeFileExt(filename, ext)
proc absOutFile*(conf: ConfigRef): AbsoluteFile = proc absOutFile*(conf: ConfigRef): AbsoluteFile =
conf.outDir / conf.outFile conf.outDir / conf.outFile

360
lib/nimhcr.nim
View file

@ -7,186 +7,186 @@
# distribution, for details about the copyright. # distribution, for details about the copyright.
# #
## This is the Nim hot code reloading run-time for the native targets. # This is the Nim hot code reloading run-time for the native targets.
## #
## This minimal dynamic library is not subject to reloading when the # This minimal dynamic library is not subject to reloading when the
## `hotCodeReloading` build mode is enabled. It's responsible for providing # `hotCodeReloading` build mode is enabled. It's responsible for providing
## a permanent memory location for all globals and procs within a program # a permanent memory location for all globals and procs within a program
## and orchestrating the reloading. For globals, this is easily achieved # and orchestrating the reloading. For globals, this is easily achieved
## by storing them on the heap. For procs, we produce on the fly simple # by storing them on the heap. For procs, we produce on the fly simple
## trampolines that can be dynamically overwritten to jump to a different # trampolines that can be dynamically overwritten to jump to a different
## target. In the host program, all globals and procs are first registered # target. In the host program, all globals and procs are first registered
## here with ``hcrRegisterGlobal`` and ``hcrRegisterProc`` and then the # here with ``hcrRegisterGlobal`` and ``hcrRegisterProc`` and then the
## returned permanent locations are used in every reference to these symbols # returned permanent locations are used in every reference to these symbols
## onwards. # onwards.
## #
## Detailed description: # Detailed description:
## #
## When code is compiled with the hotCodeReloading option for native targets # When code is compiled with the hotCodeReloading option for native targets
## a couple of things happen for all modules in a project: # a couple of things happen for all modules in a project:
## - the useNimRtl option is forced (including when building the HCR runtime too) # - the useNimRtl option is forced (including when building the HCR runtime too)
## - all modules of a target get built into separate shared libraries # - all modules of a target get built into separate shared libraries
## - the smallest granularity of reloads is modules # - the smallest granularity of reloads is modules
## - for each .c (or .cpp) in the corresponding nimcache folder of the project # - for each .c (or .cpp) in the corresponding nimcache folder of the project
## a shared object is built with the name of the source file + DLL extension # a shared object is built with the name of the source file + DLL extension
## - only the main module produces whatever the original project type intends # - only the main module produces whatever the original project type intends
## (again in nimcache) and is then copied to its original destination # (again in nimcache) and is then copied to its original destination
## - linking is done in parallel - just like compilation # - linking is done in parallel - just like compilation
## - function calls to functions from the same project go through function pointers: # - function calls to functions from the same project go through function pointers:
## - with a few exceptions - see the nonReloadable pragma # - with a few exceptions - see the nonReloadable pragma
## - the forward declarations of the original functions become function # - the forward declarations of the original functions become function
## pointers as static globals with the same names # pointers as static globals with the same names
## - the original function definitions get suffixed with <name>_actual # - the original function definitions get suffixed with <name>_actual
## - the function pointers get initialized with the address of the corresponding # - the function pointers get initialized with the address of the corresponding
## function in the DatInit of their module through a call to either hcrRegisterProc # function in the DatInit of their module through a call to either hcrRegisterProc
## or hcrGetProc. When being registered, the <name>_actual address is passed to # or hcrGetProc. When being registered, the <name>_actual address is passed to
## hcrRegisterProc and a permanent location is returned and assigned to the pointer. # hcrRegisterProc and a permanent location is returned and assigned to the pointer.
## This way the implementation (<name>_actual) can change but the address for it # This way the implementation (<name>_actual) can change but the address for it
## will be the same - this works by just updating a jump instruction (trampoline). # will be the same - this works by just updating a jump instruction (trampoline).
## For functions from other modules hcrGetProc is used (after they are registered). # For functions from other modules hcrGetProc is used (after they are registered).
## - globals are initialized only once and their state is preserved # - globals are initialized only once and their state is preserved
## - including locals with the {.global.} pragma # - including locals with the {.global.} pragma
## - their definitions are changed into pointer definitions which are initialized # - their definitions are changed into pointer definitions which are initialized
## in the DatInit() of their module with calls to hcrRegisterGlobal (supplying the # in the DatInit() of their module with calls to hcrRegisterGlobal (supplying the
## size of the type that this HCR runtime should allocate) and a bool is returned # size of the type that this HCR runtime should allocate) and a bool is returned
## which when true triggers the initialization code for the global (only once). # which when true triggers the initialization code for the global (only once).
## Globals from other modules: a global pointer coupled with a hcrGetGlobal call. # Globals from other modules: a global pointer coupled with a hcrGetGlobal call.
## - globals which have already been initialized cannot have their values changed # - globals which have already been initialized cannot have their values changed
## by changing their initialization - use a handler or some other mechanism # by changing their initialization - use a handler or some other mechanism
## - new globals can be introduced when reloading # - new globals can be introduced when reloading
## - top-level code (global scope) is executed only once - at the first module load # - top-level code (global scope) is executed only once - at the first module load
## - the runtime knows every symbol's module owner (globals and procs) # - the runtime knows every symbol's module owner (globals and procs)
## - both the RTL and HCR shared libraries need to be near the program for execution # - both the RTL and HCR shared libraries need to be near the program for execution
## - same folder, in the PATH or LD_LIBRARY_PATH env var, etc (depending on OS) # - same folder, in the PATH or LD_LIBRARY_PATH env var, etc (depending on OS)
## - the main module is responsible for initializing the HCR runtime # - the main module is responsible for initializing the HCR runtime
## - the main module loads the RTL and HCR shared objects # - the main module loads the RTL and HCR shared objects
## - after that a call to hcrInit() is done in the main module which triggers # - after that a call to hcrInit() is done in the main module which triggers
## the loading of all modules the main one imports, and doing that for the # the loading of all modules the main one imports, and doing that for the
## dependencies of each module recursively. Basically a DFS traversal. # dependencies of each module recursively. Basically a DFS traversal.
## - then initialization takes place with several passes over all modules: # - then initialization takes place with several passes over all modules:
## - HcrInit - initializes the pointers for HCR procs such as hcrRegisterProc # - HcrInit - initializes the pointers for HCR procs such as hcrRegisterProc
## - HcrCreateTypeInfos - creates globals which will be referenced in the next pass # - HcrCreateTypeInfos - creates globals which will be referenced in the next pass
## - DatInit - usual dat init + register/get procs and get globals # - DatInit - usual dat init + register/get procs and get globals
## - Init - it does the following multiplexed operations: # - Init - it does the following multiplexed operations:
## - register globals (if already registered - then just retrieve pointer) # - register globals (if already registered - then just retrieve pointer)
## - execute top level scope (only if loaded for the first time) # - execute top level scope (only if loaded for the first time)
## - when modules are loaded the originally built shared libraries get copied in # - when modules are loaded the originally built shared libraries get copied in
## the same folder and the copies are loaded instead of the original files # the same folder and the copies are loaded instead of the original files
## - a module import tree is built in the runtime (and maintained when reloading) # - a module import tree is built in the runtime (and maintained when reloading)
## - hcrPerformCodeReload # - hcrPerformCodeReload
## - named `performCodeReload`, requires the hotcodereloading module # - named `performCodeReload`, requires the hotcodereloading module
## - explicitly called by the user - the current active callstack shouldn't contain # - explicitly called by the user - the current active callstack shouldn't contain
## any functions which are defined in modules that will be reloaded (or crash!). # any functions which are defined in modules that will be reloaded (or crash!).
## The reason is that old dynalic libraries get unloaded. # The reason is that old dynalic libraries get unloaded.
## Example: # Example:
## if A is the main module and it imports B, then only B is reloadable and only # if A is the main module and it imports B, then only B is reloadable and only
## if when calling hcrPerformCodeReload there is no function defined in B in the # if when calling hcrPerformCodeReload there is no function defined in B in the
## current active callstack at the point of the call (it has to be done from A) # current active callstack at the point of the call (it has to be done from A)
## - for reloading to take place the user has to have rebuilt parts of the application # - for reloading to take place the user has to have rebuilt parts of the application
## without changes affecting the main module in any way - it shouldn't be rebuilt. # without changes affecting the main module in any way - it shouldn't be rebuilt.
## - to determine what needs to be reloaded the runtime starts traversing the import # - to determine what needs to be reloaded the runtime starts traversing the import
## tree from the root and checks the timestamps of the loaded shared objects # tree from the root and checks the timestamps of the loaded shared objects
## - modules that are no longer referenced are unloaded and cleaned up properly # - modules that are no longer referenced are unloaded and cleaned up properly
## - symbols (procs/globals) that have been removed in the code are also cleaned up # - symbols (procs/globals) that have been removed in the code are also cleaned up
## - so changing the init of a global does nothing, but removing it, reloading, # - so changing the init of a global does nothing, but removing it, reloading,
## and then re-introducing it with a new initializer works # and then re-introducing it with a new initializer works
## - new modules can be imported, and imports can also be reodereded/removed # - new modules can be imported, and imports can also be reodereded/removed
## - hcrReloadNeeded() can be used to determine if any module needs reloading # - hcrReloadNeeded() can be used to determine if any module needs reloading
## - named `hasAnyModuleChanged`, requires the hotcodereloading module # - named `hasAnyModuleChanged`, requires the hotcodereloading module
## - code in the beforeCodeReload/afterCodeReload handlers is executed on each reload # - code in the beforeCodeReload/afterCodeReload handlers is executed on each reload
## - require the hotcodereloading module # - require the hotcodereloading module
## - such handlers can be added and removed # - such handlers can be added and removed
## - before each reload all "beforeCodeReload" handlers are executed and after # - before each reload all "beforeCodeReload" handlers are executed and after
## that all handlers (including "after") from the particular module are deleted # that all handlers (including "after") from the particular module are deleted
## - the order of execution is the same as the order of top-level code execution. # - the order of execution is the same as the order of top-level code execution.
## Example: if A imports B which imports C, then all handlers in C will be executed # Example: if A imports B which imports C, then all handlers in C will be executed
## first (from top to bottom) followed by all from B and lastly all from A # first (from top to bottom) followed by all from B and lastly all from A
## - after the reload all "after" handlers are executed the same way as "before" # - after the reload all "after" handlers are executed the same way as "before"
## - the handlers for a reloaded module are always removed when reloading and then # - the handlers for a reloaded module are always removed when reloading and then
## registered when the top-level scope is executed (thanks to `executeOnReload`) # registered when the top-level scope is executed (thanks to `executeOnReload`)
## #
## TODO - after first merge in upstream Nim: # TODO - after first merge in upstream Nim:
## #
## - profile # - profile
## - build speed with and without hot code reloading - difference should be small # - build speed with and without hot code reloading - difference should be small
## - runtime degradation of HCR-enabled code - important!!! # - runtime degradation of HCR-enabled code - important!!!
## - ARM support for the trampolines # - ARM support for the trampolines
## - investigate: # - investigate:
## - rethink the closure iterators # - rethink the closure iterators
## - ability to keep old versions of dynamic libraries alive # - ability to keep old versions of dynamic libraries alive
## - because of async server code # - because of async server code
## - perhaps with refcounting of .dlls for unfinished closures # - perhaps with refcounting of .dlls for unfinished closures
## - linking with static libs # - linking with static libs
## - all shared objects for each module will (probably) have to link to them # - all shared objects for each module will (probably) have to link to them
## - state in static libs gets duplicated # - state in static libs gets duplicated
## - linking is slow and therefore iteration time suffers # - linking is slow and therefore iteration time suffers
## - have just a single .dll for all .nim files and bulk reload? # - have just a single .dll for all .nim files and bulk reload?
## - think about the compile/link/passC/passL/emit/injectStmt pragmas # - think about the compile/link/passC/passL/emit/injectStmt pragmas
## - if a passC pragma is introduced (either written or dragged in by a new # - if a passC pragma is introduced (either written or dragged in by a new
## import) the whole command line for compilation changes - for example: # import) the whole command line for compilation changes - for example:
## winlean.nim: {.passC: "-DWIN32_LEAN_AND_MEAN".} # winlean.nim: {.passC: "-DWIN32_LEAN_AND_MEAN".}
## - play with plugins/dlls/lfIndirect/lfDynamicLib/lfExportLib - shouldn't add an extra '*' # - play with plugins/dlls/lfIndirect/lfDynamicLib/lfExportLib - shouldn't add an extra '*'
## - everything thread-local related # - everything thread-local related
## - tests # - tests
## - add a new travis build matrix entry which builds everything with HCR enabled # - add a new travis build matrix entry which builds everything with HCR enabled
## - currently building with useNimRtl is problematic - lots of problems... # - currently building with useNimRtl is problematic - lots of problems...
## - how to supply the nimrtl/nimhcr shared objects to all test binaries...? # - how to supply the nimrtl/nimhcr shared objects to all test binaries...?
## - think about building to C++ instead of only to C - added type safety # - think about building to C++ instead of only to C - added type safety
## - run tests through valgrind and the sanitizers! of HUGE importance! # - run tests through valgrind and the sanitizers! of HUGE importance!
## #
## TODO - nice to have cool stuff: # TODO - nice to have cool stuff:
## #
## - separate handling of global state for much faster reloading and manipulation # - separate handling of global state for much faster reloading and manipulation
## - imagine sliders in an IDE for tweaking variables # - imagine sliders in an IDE for tweaking variables
## - perhaps using shared memory # - perhaps using shared memory
## - multi-dll projects - how everything can be reloaded..? # - multi-dll projects - how everything can be reloaded..?
## - a single HCR instance shared across multiple .dlls # - a single HCR instance shared across multiple .dlls
## - instead of having to call hcrPerformCodeReload from a function in each dll # - instead of having to call hcrPerformCodeReload from a function in each dll
## - which currently renders the main module of each dll not reloadable # - which currently renders the main module of each dll not reloadable
## - ability to check with the current callstack if a reload is "legal" # - ability to check with the current callstack if a reload is "legal"
## - if it is in any function which is in a module about to be reloaded ==> error # - if it is in any function which is in a module about to be reloaded ==> error
## - pragma annotations for files - to be excluded from dll shenanigans # - pragma annotations for files - to be excluded from dll shenanigans
## - for such file-global pragmas look at codeReordering or injectStmt # - for such file-global pragmas look at codeReordering or injectStmt
## - how would the initialization order be kept? messy... # - how would the initialization order be kept? messy...
## - per function exclude pragmas would be TOO messy and hard... # - per function exclude pragmas would be TOO messy and hard...
## - C code calling stable exportc interface of nim code (for bindings) # - C code calling stable exportc interface of nim code (for bindings)
## - generate proxy functions with the stable names # - generate proxy functions with the stable names
## - in a non-reloadable part (the main binary) that call the function pointers # - in a non-reloadable part (the main binary) that call the function pointers
## - parameter passing/forwarding - how? use the same trampoline jumping? # - parameter passing/forwarding - how? use the same trampoline jumping?
## - extracting the dependencies for these stubs/proxies will be hard... # - extracting the dependencies for these stubs/proxies will be hard...
## - changing memory layout of types - detecting this..? # - changing memory layout of types - detecting this..?
## - implement with registerType() call to HCR runtime...? # - implement with registerType() call to HCR runtime...?
## - and checking if a previously registered type matches # - and checking if a previously registered type matches
## - issue an error # - issue an error
## - or let the user handle this by transferring the state properly # - or let the user handle this by transferring the state properly
## - perhaps in the before/afterCodeReload handlers # - perhaps in the before/afterCodeReload handlers
## - optimization: calls to procs within a module (+inlined) to use the _actual versions # - optimization: calls to procs within a module (+inlined) to use the _actual versions
## - implement executeOnReload for global vars too - not just statements (and document!) # - implement executeOnReload for global vars too - not just statements (and document!)
## - cleanup at shutdown - freeing all globals # - cleanup at shutdown - freeing all globals
## #
## TODO - unimportant: # TODO - unimportant:
## #
## - have a "bad call" trampoline that all no-longer-present functions are routed to call there # - have a "bad call" trampoline that all no-longer-present functions are routed to call there
## - so the user gets some error msg if he calls a dangling pointer instead of a crash # - so the user gets some error msg if he calls a dangling pointer instead of a crash
## - before/afterCodeReload and hasModuleChanged should be accessible only where appropriate # - before/afterCodeReload and hasModuleChanged should be accessible only where appropriate
## - nim_program_result is inaccessible in HCR mode from external C code (see nimbase.h) # - nim_program_result is inaccessible in HCR mode from external C code (see nimbase.h)
## - proper .json build file - but the format is different... multiple link commands... # - proper .json build file - but the format is different... multiple link commands...
## - avoid registering globals on each loop when using an iterator in global scope # - avoid registering globals on each loop when using an iterator in global scope
## #
## TODO - REPL: # TODO - REPL:
## - proper way (as proposed by Zahary): # - proper way (as proposed by Zahary):
## - parse the input code and put everything in global scope except for # - parse the input code and put everything in global scope except for
## statements with side effects only - those go in afterCodeReload blocks # statements with side effects only - those go in afterCodeReload blocks
## - my very hacky idea: just append to a closure iterator the new statements # - my very hacky idea: just append to a closure iterator the new statements
## followed by a yield statement. So far I can think of 2 problems: # followed by a yield statement. So far I can think of 2 problems:
## - import and some other code cannot be written inside of a proc - # - import and some other code cannot be written inside of a proc -
## has to be parsed and extracted in the outer scope # has to be parsed and extracted in the outer scope
## - when new variables are created they are actually locals to the closure # - when new variables are created they are actually locals to the closure
## so the struct for the closure state grows in memory, but it has already # so the struct for the closure state grows in memory, but it has already
## been allocated when the closure was created with the previous smaller size. # been allocated when the closure was created with the previous smaller size.
## That would lead to working with memory outside of the initially allocated # That would lead to working with memory outside of the initially allocated
## block. Perhaps something can be done about this - some way of re-allocating # block. Perhaps something can be done about this - some way of re-allocating
## the state and transferring the old... # the state and transferring the old...
when not defined(JS) and (defined(hotcodereloading) or when not defined(JS) and (defined(hotcodereloading) or
defined(createNimHcr) or defined(createNimHcr) or

2
lib/system.nim
View file

@ -3831,8 +3831,6 @@ template closureScope*(body: untyped): untyped =
template once*(body: untyped): untyped = template once*(body: untyped): untyped =
## Executes a block of code only once (the first time the block is reached). ## Executes a block of code only once (the first time the block is reached).
## When hot code reloading is enabled, protects top-level code from being
## re-executed on each module reload.
## ##
## .. code-block:: nim ## .. code-block:: nim
## ##