Coroutine rework.
* ucontext backend (default on unix)
* setjmp backend
* fibers backend (default and required on windows)
* Fixed coroutine loop timing issues
* Fixed saving of xmm registers on x64 windows
* Fixed alignment issues
* Updated coroutine sample with cooperative fibonacci calculation.
* Disable glibc security features only when platform jump functions are used
* Removed dependency on fasm.
* Using fiber api on windows.
* Other platforms and compilers will use built in assembler and .S files or API provided by platform libc.
* Replaced stack switching procs with `coroExecWithStack()` which never returns. This makes compiler always generate proper code.
This commit is contained in:
Rokas Kupstys
parent
a3b8bf300d
commit
373e667dbc
11 changed files with 421 additions and 446 deletions
|
|
@ -1,62 +0,0 @@
|
||||||
#
|
|
||||||
#
|
|
||||||
# Nim's Runtime Library
|
|
||||||
# (c) Copyright 2015 Rokas Kupstys
|
|
||||||
#
|
|
||||||
# See the file "copying.txt", included in this
|
|
||||||
# distribution, for details about the copyright.
|
|
||||||
#
|
|
||||||
# Architecture-specific optimizations and features.
|
|
||||||
# arch.nim can be imported by only a subset of the
|
|
||||||
# architectures supported by Nim.
|
|
||||||
|
|
||||||
when defined(windows):
|
|
||||||
const
|
|
||||||
ABI* = "ms"
|
|
||||||
elif defined(unix):
|
|
||||||
const
|
|
||||||
ABI* = "unix"
|
|
||||||
else:
|
|
||||||
{.error: "Unsupported ABI".}
|
|
||||||
|
|
||||||
when defined(amd64):
|
|
||||||
when defined(unix):
|
|
||||||
# unix (sysv) ABI
|
|
||||||
type
|
|
||||||
JmpBufReg* {.pure.} = enum
|
|
||||||
BX, BP, R12, R13, R14, R15, SP, IP, TOTAL
|
|
||||||
elif defined(windows):
|
|
||||||
# ms ABI
|
|
||||||
type
|
|
||||||
JmpBufReg* {.pure.} = enum
|
|
||||||
BX, BP, R12, R13, R14, R15, SP, IP, SI, DI, TOTAL
|
|
||||||
type
|
|
||||||
Reg* {.pure.} = enum
|
|
||||||
AX, BX, CX, DX, SI, DI, BP, SP, IP, R8, R9, R10, R11, R12, R13, R14, R15, TOTAL
|
|
||||||
|
|
||||||
elif defined(i386) or defined(nimdoc):
|
|
||||||
# identical fastcall calling convention on all x86 OS
|
|
||||||
type
|
|
||||||
JmpBufReg* {.pure.} = enum
|
|
||||||
BX, SI, DI, BP, SP, IP, TOTAL
|
|
||||||
|
|
||||||
Reg* {.pure.} = enum
|
|
||||||
AX, BX, CX, BP, SP, DI, SI, TOTAL
|
|
||||||
|
|
||||||
else:
|
|
||||||
{.error: "Unsupported architecture".}
|
|
||||||
|
|
||||||
{.compile: "./" & ABI & "_" & hostCPU & ".asm"}
|
|
||||||
|
|
||||||
type
|
|
||||||
JmpBuf* = array[JmpBufReg.TOTAL, pointer]
|
|
||||||
Registers* = array[Reg.TOTAL, pointer]
|
|
||||||
|
|
||||||
|
|
||||||
proc getRegisters*(ctx: var Registers) {.importc: "narch_$1", fastcall.}
|
|
||||||
|
|
||||||
proc setjmp*(ctx: var JmpBuf): int {.importc: "narch_$1", fastcall.}
|
|
||||||
proc longjmp*(ctx: JmpBuf, ret=1) {.importc: "narch_$1", fastcall.}
|
|
||||||
|
|
||||||
proc coroSwitchStack*(sp: pointer) {.importc: "narch_$1", fastcall.}
|
|
||||||
proc coroRestoreStack*() {.importc: "narch_$1", fastcall.}
|
|
||||||
|
|
@ -1,79 +0,0 @@
|
||||||
;
|
|
||||||
;
|
|
||||||
; Nim's Runtime Library
|
|
||||||
; (c) Copyright 2015 Rokas Kupstys
|
|
||||||
;
|
|
||||||
; See the file "copying.txt", included in this
|
|
||||||
; distribution, for details about the copyright.
|
|
||||||
;
|
|
||||||
|
|
||||||
section ".text" executable
|
|
||||||
public narch_getRegisters
|
|
||||||
public @narch_getRegisters@4
|
|
||||||
public narch_setjmp
|
|
||||||
public @narch_setjmp@4
|
|
||||||
public narch_longjmp
|
|
||||||
public @narch_longjmp@8
|
|
||||||
public narch_coroSwitchStack
|
|
||||||
public @narch_coroSwitchStack@4
|
|
||||||
public narch_coroRestoreStack
|
|
||||||
public @narch_coroRestoreStack@0
|
|
||||||
|
|
||||||
@narch_getRegisters@4:
|
|
||||||
narch_getRegisters:
|
|
||||||
mov [ecx], eax
|
|
||||||
mov [ecx+4], ebx
|
|
||||||
mov [ecx+8], ecx
|
|
||||||
mov [ecx+0Ch], ebp
|
|
||||||
mov [ecx+10h], esp
|
|
||||||
mov [ecx+14h], edi
|
|
||||||
mov [ecx+18h], esi
|
|
||||||
ret
|
|
||||||
|
|
||||||
|
|
||||||
@narch_setjmp@4:
|
|
||||||
narch_setjmp:
|
|
||||||
; Based on code from musl libc Copyright © 2005-2014 Rich Felker, et al.
|
|
||||||
mov [ecx], ebx
|
|
||||||
mov [ecx+4], esi
|
|
||||||
mov [ecx+8], edi
|
|
||||||
mov [ecx+0Ch], ebp
|
|
||||||
lea eax, [esp+4]
|
|
||||||
mov [ecx+10h], eax
|
|
||||||
mov eax, [esp]
|
|
||||||
mov [ecx+14h], eax
|
|
||||||
xor eax, eax
|
|
||||||
ret
|
|
||||||
|
|
||||||
|
|
||||||
@narch_longjmp@8:
|
|
||||||
narch_longjmp:
|
|
||||||
; Based on code from musl libc Copyright © 2005-2014 Rich Felker, et al.
|
|
||||||
mov eax, edx
|
|
||||||
test eax, eax
|
|
||||||
jnz @F
|
|
||||||
inc eax
|
|
||||||
@@:
|
|
||||||
mov ebx, [ecx]
|
|
||||||
mov esi, [ecx+4]
|
|
||||||
mov edi, [ecx+8]
|
|
||||||
mov ebp, [ecx+0Ch]
|
|
||||||
mov esp, [ecx+10h]
|
|
||||||
mov edx, [ecx+14h]
|
|
||||||
jmp edx
|
|
||||||
|
|
||||||
|
|
||||||
@narch_coroSwitchStack@4:
|
|
||||||
narch_coroSwitchStack:
|
|
||||||
pop eax ; return address
|
|
||||||
mov edx, esp ; old esp for saving
|
|
||||||
mov esp, ecx ; swap stack with one passed to func
|
|
||||||
push edx ; store old stack pointer on newly switched stack
|
|
||||||
jmp eax ; return
|
|
||||||
|
|
||||||
|
|
||||||
@narch_coroRestoreStack@0:
|
|
||||||
narch_coroRestoreStack:
|
|
||||||
pop eax ; return address
|
|
||||||
pop esp ; resture old stack pointer
|
|
||||||
jmp eax ; return
|
|
||||||
|
|
@ -1,90 +0,0 @@
|
||||||
;
|
|
||||||
;
|
|
||||||
; Nim's Runtime Library
|
|
||||||
; (c) Copyright 2015 Rokas Kupstys
|
|
||||||
;
|
|
||||||
; See the file "copying.txt", included in this
|
|
||||||
; distribution, for details about the copyright.
|
|
||||||
;
|
|
||||||
|
|
||||||
format MS64 COFF
|
|
||||||
|
|
||||||
section ".text" executable align 16
|
|
||||||
public narch_getRegisters
|
|
||||||
public narch_setjmp
|
|
||||||
public narch_longjmp
|
|
||||||
public narch_coroSwitchStack
|
|
||||||
public narch_coroRestoreStack
|
|
||||||
|
|
||||||
|
|
||||||
narch_getRegisters:
|
|
||||||
mov [rcx], rax
|
|
||||||
mov [rcx+8], rbx
|
|
||||||
mov [rcx+10h], rcx
|
|
||||||
mov [rcx+18h], rdx
|
|
||||||
mov [rcx+20h], rsi
|
|
||||||
mov [rcx+28h], rdi
|
|
||||||
mov [rcx+30h], rbp
|
|
||||||
mov [rcx+38h], rsp
|
|
||||||
mov rax, [rsp]
|
|
||||||
mov [rcx+40h], rax ; rip
|
|
||||||
mov [rcx+48h], r8
|
|
||||||
mov [rcx+50h], r9
|
|
||||||
mov [rcx+58h], r10
|
|
||||||
mov [rcx+60h], r11
|
|
||||||
mov [rcx+68h], r12
|
|
||||||
mov [rcx+70h], r13
|
|
||||||
mov [rcx+78h], r14
|
|
||||||
mov [rcx+80h], r15
|
|
||||||
ret
|
|
||||||
|
|
||||||
|
|
||||||
narch_setjmp:
|
|
||||||
; Based on code from musl libc Copyright © 2005-2014 Rich Felker, et al.
|
|
||||||
mov [rcx], rbx ; rcx is jmp_buf, move registers onto it
|
|
||||||
mov [rcx+8], rbp
|
|
||||||
mov [rcx+10h], r12
|
|
||||||
mov [rcx+18h], r13
|
|
||||||
mov [rcx+20h], r14
|
|
||||||
mov [rcx+28h], r15
|
|
||||||
lea rdx, [rsp+8] ; this is our rsp WITHOUT current ret addr
|
|
||||||
mov [rcx+30h], rdx
|
|
||||||
mov rdx, [rsp] ; save return addr ptr for new rip
|
|
||||||
mov [rcx+38h], rdx
|
|
||||||
mov [rcx+40h], rsi
|
|
||||||
mov [rcx+48h], rdi
|
|
||||||
xor rax, rax ; always return 0
|
|
||||||
ret
|
|
||||||
|
|
||||||
narch_longjmp:
|
|
||||||
; Based on code from musl libc Copyright © 2005-2014 Rich Felker, et al.
|
|
||||||
mov rax, rdx ; val will be longjmp return
|
|
||||||
test rax, rax
|
|
||||||
jnz @F
|
|
||||||
inc rax ; if val==0, val=1 per longjmp semantics
|
|
||||||
@@:
|
|
||||||
mov rbx, [rcx] ; rax is the jmp_buf, restore regs from it
|
|
||||||
mov rbp, [rcx+8]
|
|
||||||
mov r12, [rcx+10h]
|
|
||||||
mov r13, [rcx+18h]
|
|
||||||
mov r14, [rcx+20h]
|
|
||||||
mov r15, [rcx+28h]
|
|
||||||
mov rsp, [rcx+30h] ; this ends up being the stack pointer
|
|
||||||
mov rdx, [rcx+38h] ; this is the instruction pointer
|
|
||||||
jmp rdx ; goto saved address without altering rsp
|
|
||||||
|
|
||||||
|
|
||||||
narch_coroSwitchStack:
|
|
||||||
pop rax ; return address
|
|
||||||
mov rdx, rsp ; old rsp for saving
|
|
||||||
mov rsp, rcx ; swap stack with one passed to func
|
|
||||||
push rdx ; store old stack pointer on newly switched stack
|
|
||||||
sub rsp, 28h ; stack alignment + shadow space
|
|
||||||
jmp rax ; return
|
|
||||||
|
|
||||||
|
|
||||||
narch_coroRestoreStack:
|
|
||||||
pop rax ; return address
|
|
||||||
add rsp, 28h ; stack alignment + shadow space
|
|
||||||
pop rsp ; resture old stack pointer
|
|
||||||
jmp rax ; return
|
|
||||||
|
|
@ -1,12 +0,0 @@
|
||||||
;
|
|
||||||
;
|
|
||||||
; Nim's Runtime Library
|
|
||||||
; (c) Copyright 2015 Rokas Kupstys
|
|
||||||
;
|
|
||||||
; See the file "copying.txt", included in this
|
|
||||||
; distribution, for details about the copyright.
|
|
||||||
;
|
|
||||||
|
|
||||||
format MS COFF
|
|
||||||
|
|
||||||
include 'i386.asm'
|
|
||||||
|
|
@ -1,89 +0,0 @@
|
||||||
;
|
|
||||||
;
|
|
||||||
; Nim's Runtime Library
|
|
||||||
; (c) Copyright 2015 Rokas Kupstys
|
|
||||||
;
|
|
||||||
; See the file "copying.txt", included in this
|
|
||||||
; distribution, for details about the copyright.
|
|
||||||
;
|
|
||||||
|
|
||||||
format ELF64
|
|
||||||
|
|
||||||
section ".text" executable align 16
|
|
||||||
public narch_getRegisters
|
|
||||||
public narch_setjmp
|
|
||||||
public narch_longjmp
|
|
||||||
public narch_coroSwitchStack
|
|
||||||
public narch_coroRestoreStack
|
|
||||||
|
|
||||||
|
|
||||||
narch_getRegisters:
|
|
||||||
mov [rdi], rax
|
|
||||||
mov [rdi+8], rbx
|
|
||||||
mov [rdi+10h], rcx
|
|
||||||
mov [rdi+18h], rdx
|
|
||||||
mov [rdi+20h], rsi
|
|
||||||
mov [rdi+28h], rdi
|
|
||||||
mov [rdi+30h], rbp
|
|
||||||
mov [rdi+38h], rsp
|
|
||||||
mov rax, [rsp]
|
|
||||||
mov [rdi+40h], rax ; rip
|
|
||||||
mov [rdi+48h], r8
|
|
||||||
mov [rdi+50h], r9
|
|
||||||
mov [rdi+58h], r10
|
|
||||||
mov [rdi+60h], r11
|
|
||||||
mov [rdi+68h], r12
|
|
||||||
mov [rdi+70h], r13
|
|
||||||
mov [rdi+78h], r14
|
|
||||||
mov [rdi+80h], r15
|
|
||||||
ret
|
|
||||||
|
|
||||||
|
|
||||||
narch_setjmp:
|
|
||||||
; Based on code from musl libc Copyright © 2005-2014 Rich Felker, et al.
|
|
||||||
mov [rdi], rbx ; rdi is jmp_buf, move registers onto it
|
|
||||||
mov [rdi+8], rbp
|
|
||||||
mov [rdi+10h], r12
|
|
||||||
mov [rdi+18h], r13
|
|
||||||
mov [rdi+20h], r14
|
|
||||||
mov [rdi+28h], r15
|
|
||||||
lea rdx, [rsp+8] ; this is our rsp WITHOUT current ret addr
|
|
||||||
mov [rdi+30h], rdx
|
|
||||||
mov rdx, [rsp] ; save return addr ptr for new rip
|
|
||||||
mov [rdi+38h], rdx
|
|
||||||
xor rax, rax ; always return 0
|
|
||||||
ret
|
|
||||||
|
|
||||||
|
|
||||||
narch_longjmp:
|
|
||||||
; Based on code from musl libc Copyright © 2005-2014 Rich Felker, et al.
|
|
||||||
mov rax, rsi ; val will be longjmp return
|
|
||||||
test rax, rax
|
|
||||||
jnz @F
|
|
||||||
inc rax ; if val==0, val=1 per longjmp semantics
|
|
||||||
@@:
|
|
||||||
mov rbx, [rdi] ; rdi is the jmp_buf, restore regs from it
|
|
||||||
mov rbp, [rdi+8]
|
|
||||||
mov r12, [rdi+10h]
|
|
||||||
mov r13, [rdi+18h]
|
|
||||||
mov r14, [rdi+20h]
|
|
||||||
mov r15, [rdi+28h]
|
|
||||||
mov rsp, [rdi+30h] ; this ends up being the stack pointer
|
|
||||||
mov rdx, [rdi+38h] ; this is the instruction pointer
|
|
||||||
jmp rdx ; goto saved address without altering rsp
|
|
||||||
|
|
||||||
|
|
||||||
narch_coroSwitchStack:
|
|
||||||
pop rsi ; return address
|
|
||||||
mov rdx, rsp ; old rsp for saving
|
|
||||||
mov rsp, rdi ; swap stack with one passed to func
|
|
||||||
push rdx ; store old stack pointer on newly switched stack
|
|
||||||
sub rsp, 8h ; stack alignment
|
|
||||||
jmp rsi ; return
|
|
||||||
|
|
||||||
|
|
||||||
narch_coroRestoreStack:
|
|
||||||
pop rsi ; return address
|
|
||||||
add rsp, 8h ; stack alignment
|
|
||||||
pop rsp ; resture old stack pointer
|
|
||||||
jmp rsi ; return
|
|
||||||
|
|
@ -1,12 +0,0 @@
|
||||||
;
|
|
||||||
;
|
|
||||||
; Nim's Runtime Library
|
|
||||||
; (c) Copyright 2015 Rokas Kupstys
|
|
||||||
;
|
|
||||||
; See the file "copying.txt", included in this
|
|
||||||
; distribution, for details about the copyright.
|
|
||||||
;
|
|
||||||
|
|
||||||
format ELF
|
|
||||||
|
|
||||||
include 'i386.asm'
|
|
||||||
96
lib/arch/x86/amd64.S
Normal file
96
lib/arch/x86/amd64.S
Normal file
|
|
@ -0,0 +1,96 @@
|
||||||
|
#
|
||||||
|
#
|
||||||
|
# Nim's Runtime Library
|
||||||
|
# (c) Copyright 2015 Rokas Kupstys
|
||||||
|
#
|
||||||
|
# See the file "copying.txt", included in this
|
||||||
|
# distribution, for details about the copyright.
|
||||||
|
#
|
||||||
|
# Partially based on code from musl libc Copyright © 2005-2014 Rich Felker, et al.
|
||||||
|
|
||||||
|
.globl narch_coroExecWithStack
|
||||||
|
.globl narch_setjmp
|
||||||
|
.globl narch_longjmp
|
||||||
|
.text
|
||||||
|
|
||||||
|
|
||||||
|
# SysV ABI - first argument is rdi.
|
||||||
|
# MS ABI - first argument is rcx.
|
||||||
|
#if defined(__MINGW32__) || defined(__MINGW64__)
|
||||||
|
#define REG_ARG1 rcx
|
||||||
|
#define REG_ARG2 rdx
|
||||||
|
#else
|
||||||
|
#define REG_ARG1 rdi
|
||||||
|
#define REG_ARG2 rsi
|
||||||
|
#endif
|
||||||
|
|
||||||
|
|
||||||
|
narch_coroExecWithStack:
|
||||||
|
mov %REG_ARG2, %rsp # swap stack with one passed to func
|
||||||
|
sub $0x30, %rsp # shadow space (for ms ABI) 0x20 + 0x10 for possible misalignment
|
||||||
|
and $-0x10, %rsp # 16-byte stack alignment
|
||||||
|
call *%REG_ARG1
|
||||||
|
|
||||||
|
|
||||||
|
narch_setjmp:
|
||||||
|
add $0x10, %REG_ARG1 # 16-byte alignment
|
||||||
|
and $-0x10, %REG_ARG1
|
||||||
|
mov %rbx, 0x00(%REG_ARG1) # jmp_buf, move registers onto it
|
||||||
|
mov %rbp, 0x08(%REG_ARG1)
|
||||||
|
mov %r12, 0x10(%REG_ARG1)
|
||||||
|
mov %r13, 0x18(%REG_ARG1)
|
||||||
|
mov %r14, 0x20(%REG_ARG1)
|
||||||
|
mov %r15, 0x28(%REG_ARG1)
|
||||||
|
lea 0x08(%rsp), %rdx # this is our rsp WITHOUT current ret addr
|
||||||
|
mov %rdx, 0x30(%REG_ARG1)
|
||||||
|
mov (%rsp), %rdx # save return addr ptr for new rip
|
||||||
|
mov %rdx, 0x38(%REG_ARG1)
|
||||||
|
mov %rsi, 0x40(%REG_ARG1)
|
||||||
|
mov %rdi, 0x48(%REG_ARG1)
|
||||||
|
#if defined(__MINGW32__) || defined(__MINGW64__)
|
||||||
|
movaps %xmm6, 0x50(%REG_ARG1)
|
||||||
|
movaps %xmm7, 0x60(%REG_ARG1)
|
||||||
|
movaps %xmm8, 0x70(%REG_ARG1)
|
||||||
|
movaps %xmm9, 0x80(%REG_ARG1)
|
||||||
|
movaps %xmm10, 0x90(%REG_ARG1)
|
||||||
|
movaps %xmm11, 0xA0(%REG_ARG1)
|
||||||
|
movaps %xmm12, 0xB0(%REG_ARG1)
|
||||||
|
movaps %xmm13, 0xC0(%REG_ARG1)
|
||||||
|
movaps %xmm14, 0xD0(%REG_ARG1)
|
||||||
|
movaps %xmm15, 0xE0(%REG_ARG1)
|
||||||
|
#endif
|
||||||
|
xor %rax, %rax # always return 0
|
||||||
|
ret
|
||||||
|
|
||||||
|
|
||||||
|
narch_longjmp:
|
||||||
|
add $0x10, %REG_ARG1 # 16-byte alignment
|
||||||
|
and $-0x10, %REG_ARG1 #
|
||||||
|
mov %REG_ARG2, %rax # val will be longjmp return
|
||||||
|
test %rax, %rax
|
||||||
|
jnz narch_longjmp_1
|
||||||
|
inc %rax # if val==0, val=1 per longjmp semantics
|
||||||
|
narch_longjmp_1:
|
||||||
|
mov 0x00(%REG_ARG1), %rbx # jmp_buf, restore regs from it
|
||||||
|
mov 0x08(%REG_ARG1), %rbp
|
||||||
|
mov 0x10(%REG_ARG1), %r12
|
||||||
|
mov 0x18(%REG_ARG1), %r13
|
||||||
|
mov 0x20(%REG_ARG1), %r14
|
||||||
|
mov 0x28(%REG_ARG1), %r15
|
||||||
|
mov 0x30(%REG_ARG1), %rsp # this ends up being the stack pointer
|
||||||
|
mov 0x38(%REG_ARG1), %rdx # this is the instruction pointer
|
||||||
|
mov 0x40(%REG_ARG1), %rsi
|
||||||
|
mov 0x48(%REG_ARG1), %rdi
|
||||||
|
#if defined(__MINGW32__) || defined(__MINGW64__)
|
||||||
|
movaps 0x50(%REG_ARG1), %xmm6
|
||||||
|
movaps 0x60(%REG_ARG1), %xmm7
|
||||||
|
movaps 0x70(%REG_ARG1), %xmm8
|
||||||
|
movaps 0x80(%REG_ARG1), %xmm9
|
||||||
|
movaps 0x90(%REG_ARG1), %xmm10
|
||||||
|
movaps 0xA0(%REG_ARG1), %xmm11
|
||||||
|
movaps 0xB0(%REG_ARG1), %xmm12
|
||||||
|
movaps 0xC0(%REG_ARG1), %xmm13
|
||||||
|
movaps 0xD0(%REG_ARG1), %xmm14
|
||||||
|
movaps 0xE0(%REG_ARG1), %xmm15
|
||||||
|
#endif
|
||||||
|
jmp *%rdx # goto saved address without altering rsp
|
||||||
64
lib/arch/x86/i386.S
Normal file
64
lib/arch/x86/i386.S
Normal file
|
|
@ -0,0 +1,64 @@
|
||||||
|
#
|
||||||
|
#
|
||||||
|
# Nim's Runtime Library
|
||||||
|
# (c) Copyright 2015 Rokas Kupstys
|
||||||
|
#
|
||||||
|
# See the file "copying.txt", included in this
|
||||||
|
# distribution, for details about the copyright.
|
||||||
|
#
|
||||||
|
# Partially based on code from musl libc Copyright © 2005-2014 Rich Felker, et al.
|
||||||
|
|
||||||
|
.globl narch_coroExecWithStack
|
||||||
|
.globl narch_setjmp
|
||||||
|
.globl narch_longjmp
|
||||||
|
#if defined(__MINGW32__) || defined(__MINGW64__)
|
||||||
|
.globl @narch_coroExecWithStack@8
|
||||||
|
.globl @narch_setjmp@4
|
||||||
|
.globl @narch_longjmp@8
|
||||||
|
#endif
|
||||||
|
.text
|
||||||
|
|
||||||
|
|
||||||
|
#if defined(__MINGW32__) || defined(__MINGW64__)
|
||||||
|
@narch_coroExecWithStack@8:
|
||||||
|
#endif
|
||||||
|
narch_coroExecWithStack:
|
||||||
|
mov %edx, %esp # swap stack with one passed to func
|
||||||
|
sub $0x10, %esp # 16-byte alignment
|
||||||
|
and $-0x10, %esp #
|
||||||
|
sub $4, %esp # Simulate misalignment caused by return addr
|
||||||
|
jmp *%ecx
|
||||||
|
|
||||||
|
|
||||||
|
#if defined(__MINGW32__) || defined(__MINGW64__)
|
||||||
|
@narch_setjmp@4:
|
||||||
|
#endif
|
||||||
|
narch_setjmp:
|
||||||
|
mov %ebx, (%ecx)
|
||||||
|
mov %esi, 0x04(%ecx)
|
||||||
|
mov %edi, 0x08(%ecx)
|
||||||
|
mov %ebp, 0x0C(%ecx)
|
||||||
|
lea 0x04(%esp), %eax
|
||||||
|
mov %eax, 0x10(%ecx)
|
||||||
|
mov (%esp), %eax
|
||||||
|
mov %eax, 0x14(%ecx)
|
||||||
|
xor %eax, %eax
|
||||||
|
ret
|
||||||
|
|
||||||
|
|
||||||
|
#if defined(__MINGW32__) || defined(__MINGW64__)
|
||||||
|
@narch_longjmp@8:
|
||||||
|
#endif
|
||||||
|
narch_longjmp:
|
||||||
|
mov %edx, %eax
|
||||||
|
test %eax, %eax
|
||||||
|
jnz narch_longjmp_1
|
||||||
|
inc %eax
|
||||||
|
narch_longjmp_1:
|
||||||
|
mov (%ecx), %ebx
|
||||||
|
mov 0x04(%ecx), %esi
|
||||||
|
mov 0x08(%ecx), %edi
|
||||||
|
mov 0x0C(%ecx), %ebp
|
||||||
|
mov 0x10(%ecx), %esp
|
||||||
|
mov 0x14(%ecx), %edx
|
||||||
|
jmp *%edx
|
||||||
|
|
@ -6,138 +6,303 @@
|
||||||
# See the file "copying.txt", included in this
|
# See the file "copying.txt", included in this
|
||||||
# distribution, for details about the copyright.
|
# distribution, for details about the copyright.
|
||||||
#
|
#
|
||||||
|
## Nim coroutines implementation supports several context switching methods:
|
||||||
|
## ucontext: available on unix and alike (default)
|
||||||
|
## setjmp: available on unix and alike (x86/64 only)
|
||||||
|
## Fibers: available and required on windows.
|
||||||
|
##
|
||||||
|
## -d:nimCoroutines Required to build this module.
|
||||||
|
## -d:nimCoroutinesUcontext Use ucontext backend.
|
||||||
|
## -d:nimCoroutinesSetjmp Use setjmp backend.
|
||||||
|
## -d:nimCoroutinesSetjmpBundled Use bundled setjmp implementation.
|
||||||
|
|
||||||
when not defined(nimCoroutines) and not defined(nimdoc):
|
when not defined(nimCoroutines) and not defined(nimdoc):
|
||||||
{.error: "Coroutines require -d:nimCoroutines".}
|
{.error: "Coroutines require -d:nimCoroutines".}
|
||||||
|
|
||||||
import os, times
|
import os
|
||||||
import macros
|
import macros
|
||||||
import arch
|
|
||||||
import lists
|
import lists
|
||||||
|
include system/timers
|
||||||
|
|
||||||
const defaultStackSize = 512 * 1024
|
const defaultStackSize = 512 * 1024
|
||||||
|
|
||||||
type Coroutine = ref object
|
|
||||||
# prev: ptr Coroutine
|
|
||||||
# next: ptr Coroutine
|
|
||||||
ctx: JmpBuf
|
|
||||||
fn: proc()
|
|
||||||
started: bool
|
|
||||||
lastRun: float
|
|
||||||
sleepTime: float
|
|
||||||
stack: pointer
|
|
||||||
stacksize: int
|
|
||||||
|
|
||||||
var coroutines = initDoublyLinkedList[Coroutine]()
|
|
||||||
var current: Coroutine
|
|
||||||
var mainCtx: JmpBuf
|
|
||||||
|
|
||||||
|
|
||||||
proc GC_addStack(starts: pointer) {.cdecl, importc.}
|
proc GC_addStack(starts: pointer) {.cdecl, importc.}
|
||||||
proc GC_removeStack(starts: pointer) {.cdecl, importc.}
|
proc GC_removeStack(starts: pointer) {.cdecl, importc.}
|
||||||
proc GC_setCurrentStack(starts, pos: pointer) {.cdecl, importc.}
|
proc GC_setCurrentStack(starts, pos: pointer) {.cdecl, importc.}
|
||||||
|
|
||||||
proc start*(c: proc(), stacksize: int=defaultStackSize) =
|
const
|
||||||
## Adds coroutine to event loop. It does not run immediately.
|
CORO_BACKEND_UCONTEXT = 0
|
||||||
var coro = Coroutine()
|
CORO_BACKEND_SETJMP = 1
|
||||||
coro.fn = c
|
CORO_BACKEND_FIBERS = 2
|
||||||
while coro.stack == nil:
|
|
||||||
coro.stack = alloc0(stacksize)
|
when defined(windows):
|
||||||
coro.stacksize = stacksize
|
const coroBackend = CORO_BACKEND_FIBERS
|
||||||
coroutines.append(coro)
|
when defined(nimCoroutinesUcontext):
|
||||||
|
{.warning: "ucontext coroutine backend is not available on windows, defaulting to fibers.".}
|
||||||
|
when defined(nimCoroutinesSetjmp):
|
||||||
|
{.warning: "setjmp coroutine backend is not available on windows, defaulting to fibers.".}
|
||||||
|
elif defined(nimCoroutinesSetjmp) or defined(nimCoroutinesSetjmpBundled):
|
||||||
|
const coroBackend = CORO_BACKEND_SETJMP
|
||||||
|
else:
|
||||||
|
const coroBackend = CORO_BACKEND_UCONTEXT
|
||||||
|
|
||||||
|
when coroBackend == CORO_BACKEND_FIBERS:
|
||||||
|
import windows.winlean
|
||||||
|
type
|
||||||
|
Context = pointer
|
||||||
|
Fiber {.final, pure.} = object
|
||||||
|
parameter: pointer
|
||||||
|
pad1: pointer
|
||||||
|
stackStart: pointer
|
||||||
|
stackEnd: pointer
|
||||||
|
|
||||||
|
elif coroBackend == CORO_BACKEND_UCONTEXT:
|
||||||
|
type
|
||||||
|
stack_t {.importc, header: "<sys/ucontext.h>".} = object
|
||||||
|
ss_sp: pointer
|
||||||
|
ss_flags: int
|
||||||
|
ss_size: int
|
||||||
|
|
||||||
|
ucontext_t {.importc, header: "<sys/ucontext.h>".} = object
|
||||||
|
uc_link: ptr ucontext_t
|
||||||
|
uc_stack: stack_t
|
||||||
|
|
||||||
|
Context = ucontext_t
|
||||||
|
|
||||||
|
proc getcontext(context: var ucontext_t): int32 {.importc, header: "<sys/ucontext.h>".}
|
||||||
|
proc setcontext(context: var ucontext_t): int32 {.importc, header: "<sys/ucontext.h>".}
|
||||||
|
proc swapcontext(fromCtx, toCtx: var ucontext_t): int32 {.importc, header: "<sys/ucontext.h>".}
|
||||||
|
proc makecontext(context: var ucontext_t, fn: pointer, argc: int32) {.importc, header: "<sys/ucontext.h>", varargs.}
|
||||||
|
|
||||||
|
elif coroBackend == CORO_BACKEND_SETJMP:
|
||||||
|
proc coroExecWithStack*(fn: pointer, stack: pointer) {.noreturn, importc: "narch_$1", fastcall.}
|
||||||
|
when defined(amd64):
|
||||||
|
{.compile: "../arch/x86/amd64.S".}
|
||||||
|
elif defined(i386):
|
||||||
|
{.compile: "../arch/x86/i386.S".}
|
||||||
|
else:
|
||||||
|
# coroExecWithStack is defined in assembly. To support other platforms
|
||||||
|
# please provide implementation of this procedure.
|
||||||
|
{.error: "Unsupported architecture.".}
|
||||||
|
|
||||||
|
when defined(nimCoroutinesSetjmpBundled):
|
||||||
|
# Use setjmp/longjmp implementation shipped with compiler.
|
||||||
|
when defined(amd64):
|
||||||
|
type
|
||||||
|
JmpBuf = array[0x50 + 0x10, uint8]
|
||||||
|
elif defined(i386):
|
||||||
|
type
|
||||||
|
JmpBuf = array[0x1C, uint8]
|
||||||
|
else:
|
||||||
|
# Bundled setjmp/longjmp are defined in assembly. To support other
|
||||||
|
# platforms please provide implementations of these procedures.
|
||||||
|
{.error: "Unsupported architecture.".}
|
||||||
|
|
||||||
|
proc setjmp(ctx: var JmpBuf): int {.importc: "narch_$1".}
|
||||||
|
proc longjmp(ctx: JmpBuf, ret=1) {.importc: "narch_$1".}
|
||||||
|
else:
|
||||||
|
# Use setjmp/longjmp implementation provided by the system.
|
||||||
|
type
|
||||||
|
JmpBuf {.importc: "jmp_buf", header: "<setjmp.h>".} = object
|
||||||
|
|
||||||
|
proc setjmp(ctx: var JmpBuf): int {.importc, header: "<setjmp.h>".}
|
||||||
|
proc longjmp(ctx: JmpBuf, ret=1) {.importc, header: "<setjmp.h>".}
|
||||||
|
|
||||||
|
type
|
||||||
|
Context = JmpBuf
|
||||||
|
|
||||||
|
when defined(unix):
|
||||||
|
# GLibc fails with "*** longjmp causes uninitialized stack frame ***" because
|
||||||
|
# our custom stacks are not initialized to a magic value.
|
||||||
|
{.passC: "-U_FORTIFY_SOURCE -D_FORTIFY_SOURCE=0"}
|
||||||
|
|
||||||
|
const
|
||||||
|
CORO_EXECUTING = 1
|
||||||
|
CORO_FINISHED = 2
|
||||||
|
|
||||||
|
type
|
||||||
|
Stack = object
|
||||||
|
start: pointer
|
||||||
|
ends: pointer
|
||||||
|
size: int
|
||||||
|
|
||||||
|
Coroutine = ref object
|
||||||
|
execContext: Context
|
||||||
|
fn: proc()
|
||||||
|
state: int
|
||||||
|
lastRun: Ticks
|
||||||
|
sleepTime: float
|
||||||
|
stack: Stack
|
||||||
|
|
||||||
|
CoroutineLoopContext = ref object
|
||||||
|
coroutines: DoublyLinkedList[Coroutine]
|
||||||
|
current: DoublyLinkedNode[Coroutine]
|
||||||
|
loop: Coroutine
|
||||||
|
|
||||||
|
# Per-thread coroutine loop state.
|
||||||
|
var ctx: CoroutineLoopContext
|
||||||
|
|
||||||
|
proc getCurrent(): Coroutine =
|
||||||
|
## Returns current executing coroutine object.
|
||||||
|
var node = ctx.current
|
||||||
|
if node != nil:
|
||||||
|
return node.value
|
||||||
|
return nil
|
||||||
|
|
||||||
|
proc initialize() =
|
||||||
|
## Initializes coroutine state of current thread.
|
||||||
|
if ctx == nil:
|
||||||
|
ctx = CoroutineLoopContext()
|
||||||
|
ctx.coroutines = initDoublyLinkedList[Coroutine]()
|
||||||
|
ctx.loop = Coroutine()
|
||||||
|
ctx.loop.state = CORO_EXECUTING
|
||||||
|
when coroBackend == CORO_BACKEND_FIBERS:
|
||||||
|
ctx.loop.execContext = ConvertThreadToFiberEx(nil, FIBER_FLAG_FLOAT_SWITCH)
|
||||||
|
|
||||||
|
proc runCurrentTask()
|
||||||
|
|
||||||
|
proc switchTo(current, to: Coroutine) =
|
||||||
|
## Switches execution from `current` into `to` context.
|
||||||
|
to.lastRun = getTicks()
|
||||||
|
when coroBackend == CORO_BACKEND_FIBERS:
|
||||||
|
SwitchToFiber(to.execContext)
|
||||||
|
elif coroBackend == CORO_BACKEND_UCONTEXT:
|
||||||
|
discard swapcontext(current.execContext, to.execContext)
|
||||||
|
elif coroBackend == CORO_BACKEND_SETJMP:
|
||||||
|
var res = setjmp(current.execContext)
|
||||||
|
if res == 0:
|
||||||
|
if to.state == CORO_EXECUTING:
|
||||||
|
# Coroutine is resumed.
|
||||||
|
longjmp(to.execContext, 1)
|
||||||
|
elif to.state == CORO_CREATED:
|
||||||
|
# Coroutine is started.
|
||||||
|
coroExecWithStack(runCurrentTask, to.stack.ends)
|
||||||
|
doAssert false
|
||||||
|
else:
|
||||||
|
{.error: "Invalid coroutine backend set.".}
|
||||||
|
|
||||||
{.push stackTrace: off.}
|
|
||||||
proc suspend*(sleepTime: float=0) =
|
proc suspend*(sleepTime: float=0) =
|
||||||
## Stops coroutine execution and resumes no sooner than after ``sleeptime`` seconds.
|
## Stops coroutine execution and resumes no sooner than after ``sleeptime`` seconds.
|
||||||
## Until then other coroutines are executed.
|
## Until then other coroutines are executed.
|
||||||
##
|
var frame = getFrame()
|
||||||
## This is similar to a `yield`:idx:, or a `yieldFrom`:idx in Python.
|
|
||||||
var oldFrame = getFrame()
|
|
||||||
var sp {.volatile.}: pointer
|
var sp {.volatile.}: pointer
|
||||||
GC_setCurrentStack(current.stack, cast[pointer](addr sp))
|
var current = getCurrent()
|
||||||
|
GC_setCurrentStack(current.stack.start, cast[pointer](addr sp))
|
||||||
current.sleepTime = sleepTime
|
current.sleepTime = sleepTime
|
||||||
current.lastRun = epochTime()
|
switchTo(current, ctx.loop)
|
||||||
if setjmp(current.ctx) == 0:
|
setFrame(frame)
|
||||||
longjmp(mainCtx, 1)
|
|
||||||
setFrame(oldFrame)
|
proc runCurrentTask() =
|
||||||
{.pop.}
|
## Starts execution of current coroutine and updates it's state through coroutine's life.
|
||||||
|
var current = getCurrent()
|
||||||
|
current.state = CORO_EXECUTING
|
||||||
|
current.fn() # Start coroutine execution
|
||||||
|
current.state = CORO_FINISHED
|
||||||
|
suspend(0) # Exit coroutine without returning from coroExecWithStack()
|
||||||
|
doAssert false
|
||||||
|
|
||||||
|
proc start*(c: proc(), stacksize: int=defaultStackSize) =
|
||||||
|
## Schedule coroutine for execution. It does not run immediately.
|
||||||
|
if ctx == nil:
|
||||||
|
initialize()
|
||||||
|
|
||||||
|
var coro = Coroutine()
|
||||||
|
coro.fn = c
|
||||||
|
when coroBackend == CORO_BACKEND_FIBERS:
|
||||||
|
coro.execContext = CreateFiberEx(stacksize, stacksize,
|
||||||
|
FIBER_FLAG_FLOAT_SWITCH, (proc(p: pointer): void {.stdcall.} = runCurrentTask()), nil)
|
||||||
|
var fiber = cast[ptr Fiber](coro.execContext)
|
||||||
|
coro.stack.start = fiber.stackStart
|
||||||
|
coro.stack.ends = fiber.stackEnd
|
||||||
|
coro.stack.size = stacksize
|
||||||
|
else:
|
||||||
|
var stack: pointer
|
||||||
|
while stack == nil:
|
||||||
|
stack = alloc0(stacksize)
|
||||||
|
coro.stack.start = stack
|
||||||
|
coro.stack.ends = cast[pointer](cast[ByteAddress](stack) + stacksize)
|
||||||
|
when coroBackend == CORO_BACKEND_UCONTEXT:
|
||||||
|
discard getcontext(coro.execContext)
|
||||||
|
coro.execContext.uc_stack.ss_sp = coro.stack.ends
|
||||||
|
coro.execContext.uc_stack.ss_size = coro.stack.size
|
||||||
|
coro.execContext.uc_link = addr ctx.loop.execContext
|
||||||
|
makecontext(coro.execContext, runCurrentTask, 0)
|
||||||
|
coro.stack.size = stacksize
|
||||||
|
GC_addStack(coro.stack.ends)
|
||||||
|
ctx.coroutines.append(coro)
|
||||||
|
|
||||||
proc run*() =
|
proc run*() =
|
||||||
## Starts main event loop which exits when all coroutines exit. Calling this proc
|
initialize()
|
||||||
## starts execution of first coroutine.
|
## Starts main coroutine scheduler loop which exits when all coroutines exit.
|
||||||
var node = coroutines.head
|
## Calling this proc starts execution of first coroutine.
|
||||||
var minDelay: int = 0 # in milliseconds
|
ctx.current = ctx.coroutines.head
|
||||||
var frame: PFrame
|
var minDelay: float = 0
|
||||||
while node != nil:
|
while ctx.current != nil:
|
||||||
var coro = node.value
|
var current = getCurrent()
|
||||||
current = coro
|
|
||||||
os.sleep(minDelay)
|
|
||||||
|
|
||||||
var remaining = int((coro.sleepTime - (epochTime() - coro.lastRun)) * 1000)
|
var remaining = current.sleepTime - (float(getTicks() - current.lastRun) / 1_000_000_000)
|
||||||
if remaining <= 0:
|
if remaining <= 0:
|
||||||
remaining = 0
|
# Save main loop context. Suspending coroutine will resume after this statement with
|
||||||
let res = setjmp(mainCtx)
|
var frame = getFrame()
|
||||||
if res == 0:
|
switchTo(ctx.loop, current)
|
||||||
frame = getFrame()
|
setFrame(frame)
|
||||||
if coro.started: # coroutine resumes
|
else:
|
||||||
longjmp(coro.ctx, 1)
|
|
||||||
else:
|
|
||||||
coro.started = true # coroutine starts
|
|
||||||
var stackEnd = cast[pointer](cast[ByteAddress](coro.stack) + coro.stacksize)
|
|
||||||
GC_addStack(coro.stack)
|
|
||||||
coroSwitchStack(stackEnd)
|
|
||||||
coro.fn()
|
|
||||||
coroRestoreStack()
|
|
||||||
GC_removeStack(coro.stack)
|
|
||||||
var next = node.prev
|
|
||||||
coroutines.remove(node)
|
|
||||||
dealloc(coro.stack)
|
|
||||||
node = next
|
|
||||||
setFrame(frame)
|
|
||||||
else:
|
|
||||||
setFrame(frame)
|
|
||||||
|
|
||||||
elif remaining > 0:
|
|
||||||
if minDelay > 0 and remaining > 0:
|
if minDelay > 0 and remaining > 0:
|
||||||
minDelay = min(remaining, minDelay)
|
minDelay = min(remaining, minDelay)
|
||||||
else:
|
else:
|
||||||
minDelay = remaining
|
minDelay = remaining
|
||||||
|
|
||||||
if node == nil or node.next == nil:
|
if current.state == CORO_FINISHED:
|
||||||
node = coroutines.head
|
GC_removeStack(current.stack.start)
|
||||||
|
var next = ctx.current.prev
|
||||||
|
ctx.coroutines.remove(ctx.current)
|
||||||
|
when coroBackend != CORO_BACKEND_FIBERS:
|
||||||
|
dealloc(current.stack.start)
|
||||||
|
current.stack.start = nil
|
||||||
|
current.stack.ends = nil
|
||||||
|
ctx.current = next
|
||||||
|
elif ctx.current == nil or ctx.current.next == nil:
|
||||||
|
ctx.current = ctx.coroutines.head
|
||||||
|
os.sleep(int(minDelay * 1000))
|
||||||
else:
|
else:
|
||||||
node = node.next
|
ctx.current = ctx.current.next
|
||||||
|
|
||||||
proc alive*(c: proc()): bool =
|
proc alive*(c: proc()): bool =
|
||||||
## Returns ``true`` if coroutine has not returned, ``false`` otherwise.
|
## Returns ``true`` if coroutine has not returned, ``false`` otherwise.
|
||||||
for coro in items(coroutines):
|
for coro in items(ctx.coroutines):
|
||||||
if coro.fn == c:
|
if coro.fn == c:
|
||||||
return true
|
return coro.state != CORO_FINISHED
|
||||||
|
|
||||||
proc wait*(c: proc(), interval=0.01) =
|
proc wait*(c: proc(), interval=0.01) =
|
||||||
## Returns only after coroutine ``c`` has returned. ``interval`` is time in seconds how often.
|
## Returns only after coroutine ``c`` has returned. ``interval`` is time in seconds how often.
|
||||||
while alive(c):
|
while alive(c):
|
||||||
suspend interval
|
suspend(interval)
|
||||||
|
|
||||||
when defined(nimCoroutines) and isMainModule:
|
when isMainModule:
|
||||||
var stackCheckValue = 1100220033
|
var
|
||||||
proc c2()
|
stackCheckValue = 1100220033
|
||||||
|
first: float64 = 0
|
||||||
|
second: float64 = 1
|
||||||
|
steps = 10
|
||||||
|
i: int
|
||||||
|
order = newSeq[int](10)
|
||||||
|
|
||||||
proc c1() =
|
proc Fibonacci(id: int, sleep: float32) =
|
||||||
for i in 0 .. 3:
|
var sleepTime: float
|
||||||
echo "c1"
|
while steps > 0:
|
||||||
suspend 0.05
|
echo id, " executing, slept for ", sleepTime
|
||||||
echo "c1 exits"
|
order[i] = id
|
||||||
|
i += 1
|
||||||
|
steps -= 1
|
||||||
|
swap first, second
|
||||||
|
second += first
|
||||||
|
var sleepStart = getTicks()
|
||||||
|
suspend(sleep)
|
||||||
|
sleepTime = float(getTicks() - sleepStart) / 1_000_000_000
|
||||||
|
|
||||||
|
start(proc() = Fibonacci(1, 0.01))
|
||||||
proc c2() =
|
start(proc() = Fibonacci(2, 0.021))
|
||||||
for i in 0 .. 3:
|
|
||||||
echo "c2"
|
|
||||||
suspend 0.025
|
|
||||||
wait(c1)
|
|
||||||
echo "c2 exits"
|
|
||||||
|
|
||||||
start(c1)
|
|
||||||
start(c2)
|
|
||||||
run()
|
run()
|
||||||
echo "done ", stackCheckValue
|
doAssert stackCheckValue == 1100220033
|
||||||
|
doAssert first == 55.0
|
||||||
|
doAssert order == @[1, 2, 1, 1, 2, 1, 1, 2, 1, 1]
|
||||||
|
|
|
||||||
|
|
@ -12,9 +12,6 @@
|
||||||
# Refcounting + Mark&Sweep. Complex algorithms avoided.
|
# Refcounting + Mark&Sweep. Complex algorithms avoided.
|
||||||
# Been there, done that, didn't work.
|
# Been there, done that, didn't work.
|
||||||
|
|
||||||
when defined(nimCoroutines):
|
|
||||||
import arch
|
|
||||||
|
|
||||||
{.push profiler:off.}
|
{.push profiler:off.}
|
||||||
|
|
||||||
const
|
const
|
||||||
|
|
|
||||||
|
|
@ -277,11 +277,8 @@ else:
|
||||||
# Used to traverse the stack and registers assuming
|
# Used to traverse the stack and registers assuming
|
||||||
# that 'setjmp' will save registers in the C stack.
|
# that 'setjmp' will save registers in the C stack.
|
||||||
type PStackSlice = ptr array[0..7, pointer]
|
type PStackSlice = ptr array[0..7, pointer]
|
||||||
var registers {.noinit.}: Registers
|
var registers {.noinit.}: C_JmpBuf
|
||||||
getRegisters(registers)
|
discard c_setjmp(registers)
|
||||||
for i in registers.low .. registers.high:
|
|
||||||
gcMark(gch, cast[PPointer](registers[i]))
|
|
||||||
|
|
||||||
for stack in items(gch.stack):
|
for stack in items(gch.stack):
|
||||||
stack.maxStackSize = max(stack.maxStackSize, stackSize(stack.starts))
|
stack.maxStackSize = max(stack.maxStackSize, stackSize(stack.starts))
|
||||||
var max = cast[ByteAddress](stack.starts)
|
var max = cast[ByteAddress](stack.starts)
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue