jyapayne/Nim
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge pull request #830 from zielmicha/osproc

Browse source 
osproc: introduce poEvalCommand, poUsePath, fix remaining quoting issues.
This commit is contained in:
Andreas Rumpf 2014-02-01 16:11:09 -08:00
commit 087448d167
2 changed files with 536 additions and 534 deletions
Download patch file Download diff file Expand all files Collapse all files

891
lib/posix/posix.nim
View file

File diff suppressed because it is too large Load diff

179
lib/pure/osproc.nim
View file

@ -13,7 +13,7 @@
include "system/inclrtl" include "system/inclrtl"
import import
strutils, os, strtabs, streams strutils, os, strtabs, streams, sequtils
when defined(windows): when defined(windows):
import winlean import winlean
@ -36,11 +36,17 @@ type
TProcessOption* = enum ## options that can be passed `startProcess` TProcessOption* = enum ## options that can be passed `startProcess`
poEchoCmd, ## echo the command before execution poEchoCmd, ## echo the command before execution
poUseShell, ## use the shell to execute the command; NOTE: This poUsePath, ## Asks system to search for executable using PATH environment
## often creates a security hole! ## variable.
## On Windows, this is the default.
poEvalCommand, ## Pass `command` directly to the shell, without quoting.
## Use it only if `command` comes from trused source.
poStdErrToStdOut, ## merge stdout and stderr to the stdout stream poStdErrToStdOut, ## merge stdout and stderr to the stdout stream
poParentStreams ## use the parent's streams poParentStreams ## use the parent's streams
template poUseShell*: TProcessOption {.deprecated.} = poUsePath
## Deprecated alias for poUsePath.
proc quoteShellWindows*(s: string): string {.noSideEffect, rtl, extern: "nosp$1".} = proc quoteShellWindows*(s: string): string {.noSideEffect, rtl, extern: "nosp$1".} =
## Quote s, so it can be safely passed to Windows API. ## Quote s, so it can be safely passed to Windows API.
## Based on Python's subprocess.list2cmdline ## Based on Python's subprocess.list2cmdline
@ -94,12 +100,17 @@ proc quoteShell*(s: string): string {.noSideEffect, rtl, extern: "nosp$1".} =
{.error:"quoteShell is not supported on your system".} {.error:"quoteShell is not supported on your system".}
proc execProcess*(command: string, proc execProcess*(command: string,
args: openarray[string] = [],
env: PStringTable = nil,
options: set[TProcessOption] = {poStdErrToStdOut, options: set[TProcessOption] = {poStdErrToStdOut,
poUseShell}): TaintedString {. poUsePath,
poEvalCommand}): TaintedString {.
rtl, extern: "nosp$1", rtl, extern: "nosp$1",
tags: [FExecIO, FReadIO].} tags: [FExecIO, FReadIO].}
## A convenience procedure that executes ``command`` with ``startProcess`` ## A convenience procedure that executes ``command`` with ``startProcess``
## and returns its output as a string. ## and returns its output as a string.
## WARNING: this function uses poEvalCommand by default for backward compatibility.
## Make sure to pass options explicitly.
proc execCmd*(command: string): int {.rtl, extern: "nosp$1", tags: [FExecIO].} proc execCmd*(command: string): int {.rtl, extern: "nosp$1", tags: [FExecIO].}
## Executes ``command`` and returns its error code. Standard input, output, ## Executes ``command`` and returns its error code. Standard input, output,
@ -109,8 +120,8 @@ proc execCmd*(command: string): int {.rtl, extern: "nosp$1", tags: [FExecIO].}
proc startProcess*(command: string, proc startProcess*(command: string,
workingDir: string = "", workingDir: string = "",
args: openArray[string] = [], args: openArray[string] = [],
env: PStringTable = nil, env: PStringTable = nil,
options: set[TProcessOption] = {poStdErrToStdOut}): options: set[TProcessOption] = {poStdErrToStdOut}):
PProcess {.rtl, extern: "nosp$1", tags: [FExecIO, FReadEnv].} PProcess {.rtl, extern: "nosp$1", tags: [FExecIO, FReadEnv].}
## Starts a process. `Command` is the executable file, `workingDir` is the ## Starts a process. `Command` is the executable file, `workingDir` is the
## process's working directory. If ``workingDir == ""`` the current directory ## process's working directory. If ``workingDir == ""`` the current directory
@ -127,16 +138,10 @@ proc startProcess*(command: string,
## but ``EOS`` is raised in case of an error. ## but ``EOS`` is raised in case of an error.
proc startCmd*(command: string, options: set[TProcessOption] = { proc startCmd*(command: string, options: set[TProcessOption] = {
poStdErrToStdOut, poUseShell}): PProcess {. poStdErrToStdOut, poUsePath}): PProcess {.
tags: [FExecIO, FReadEnv].} = tags: [FExecIO, FReadEnv], deprecated.} =
## a simpler version of `startProcess` that parses the command line into ## Deprecated - use `startProcess` directly.
## program and arguments and then calls `startProcess` with the empty string result = startProcess(command=command, options=options + {poEvalCommand})
## for `workingDir` and the nil string table for `env`.
var c = parseCmdLine(command)
var a: seq[string]
newSeq(a, c.len-1) # avoid slicing for now (still unstable)
for i in 1 .. c.len-1: a[i-1] = c[i]
result = startProcess(command=c[0], args=a, options=options)
proc close*(p: PProcess) {.rtl, extern: "nosp$1", tags: [].} proc close*(p: PProcess) {.rtl, extern: "nosp$1", tags: [].}
## When the process has finished executing, cleanup related handles ## When the process has finished executing, cleanup related handles
@ -157,7 +162,7 @@ proc processID*(p: PProcess): int {.rtl, extern: "nosp$1".} =
## returns `p`'s process ID. ## returns `p`'s process ID.
return p.id return p.id
proc waitForExit*(p: PProcess, timeout: int = -1): int {.rtl, proc waitForExit*(p: PProcess, timeout: int = -1): int {.rtl,
extern: "nosp$1", tags: [].} extern: "nosp$1", tags: [].}
## waits for the process to finish and returns `p`'s error code. ## waits for the process to finish and returns `p`'s error code.
@ -167,19 +172,19 @@ proc peekExitCode*(p: PProcess): int {.tags: [].}
proc inputStream*(p: PProcess): PStream {.rtl, extern: "nosp$1", tags: [].} proc inputStream*(p: PProcess): PStream {.rtl, extern: "nosp$1", tags: [].}
## returns ``p``'s input stream for writing to. ## returns ``p``'s input stream for writing to.
## ##
## **Warning**: The returned `PStream` should not be closed manually as it ## **Warning**: The returned `PStream` should not be closed manually as it
## is closed when closing the PProcess ``p``. ## is closed when closing the PProcess ``p``.
proc outputStream*(p: PProcess): PStream {.rtl, extern: "nosp$1", tags: [].} proc outputStream*(p: PProcess): PStream {.rtl, extern: "nosp$1", tags: [].}
## returns ``p``'s output stream for reading from. ## returns ``p``'s output stream for reading from.
## ##
## **Warning**: The returned `PStream` should not be closed manually as it ## **Warning**: The returned `PStream` should not be closed manually as it
## is closed when closing the PProcess ``p``. ## is closed when closing the PProcess ``p``.
proc errorStream*(p: PProcess): PStream {.rtl, extern: "nosp$1", tags: [].} proc errorStream*(p: PProcess): PStream {.rtl, extern: "nosp$1", tags: [].}
## returns ``p``'s error stream for reading from. ## returns ``p``'s error stream for reading from.
## ##
## **Warning**: The returned `PStream` should not be closed manually as it ## **Warning**: The returned `PStream` should not be closed manually as it
## is closed when closing the PProcess ``p``. ## is closed when closing the PProcess ``p``.
proc inputHandle*(p: PProcess): TFileHandle {.rtl, extern: "nosp$1", proc inputHandle*(p: PProcess): TFileHandle {.rtl, extern: "nosp$1",
@ -245,15 +250,15 @@ proc countProcessors*(): int {.rtl, extern: "nosp$1".} =
proc execProcesses*(cmds: openArray[string], proc execProcesses*(cmds: openArray[string],
options = {poStdErrToStdOut, poParentStreams}, options = {poStdErrToStdOut, poParentStreams},
n = countProcessors()): int {.rtl, extern: "nosp$1", n = countProcessors()): int {.rtl, extern: "nosp$1",
tags: [FExecIO, FTime, FReadEnv].} = tags: [FExecIO, FTime, FReadEnv]} =
## executes the commands `cmds` in parallel. Creates `n` processes ## executes the commands `cmds` in parallel. Creates `n` processes
## that execute in parallel. The highest return value of all processes ## that execute in parallel. The highest return value of all processes
## is returned. ## is returned.
when defined(posix): when defined(posix):
# poParentStreams causes problems on Posix, so we simply disable it: # poParentStreams causes problems on Posix, so we simply disable it:
var options = options - {poParentStreams} var options = options - {poParentStreams}
assert n > 0 assert n > 0
if n > 1: if n > 1:
var q: seq[PProcess] var q: seq[PProcess]
@ -307,13 +312,17 @@ proc select*(readfds: var seq[PProcess], timeout = 500): int
when not defined(useNimRtl): when not defined(useNimRtl):
proc execProcess(command: string, proc execProcess(command: string,
args: openarray[string] = [],
env: PStringTable = nil,
options: set[TProcessOption] = {poStdErrToStdOut, options: set[TProcessOption] = {poStdErrToStdOut,
poUseShell}): TaintedString = poUsePath,
var p = startCmd(command, options=options) poEvalCommand}): TaintedString =
var p = startProcess(command, args=args, env=env, options=options)
var outp = outputStream(p) var outp = outputStream(p)
result = TaintedString"" result = TaintedString""
var line = newStringOfCap(120).TaintedString var line = newStringOfCap(120).TaintedString
while true: while true:
# FIXME: converts CR-LF to LF.
if outp.readLine(line): if outp.readLine(line):
result.string.add(line.string) result.string.add(line.string)
result.string.add("\n") result.string.add("\n")
@ -427,8 +436,9 @@ when defined(Windows) and not defined(useNimRtl):
result.errHandle = TFileHandle(si.hStdError) result.errHandle = TFileHandle(si.hStdError)
var cmdl: cstring var cmdl: cstring
when false: # poUseShell in options: if poEvalCommand in options:
cmdl = buildCommandLine(getEnv("COMSPEC"), @["/c", command] & args) cmdl = command
assert args.len == 0
else: else:
cmdl = buildCommandLine(command, args) cmdl = buildCommandLine(command, args)
var wd: cstring = nil var wd: cstring = nil
@ -441,7 +451,7 @@ when defined(Windows) and not defined(useNimRtl):
var ee = newWideCString(e) var ee = newWideCString(e)
var wwd = newWideCString(wd) var wwd = newWideCString(wd)
success = winlean.createProcessW(nil, success = winlean.createProcessW(nil,
tmp, nil, nil, 1, NORMAL_PRIORITY_CLASS or CREATE_UNICODE_ENVIRONMENT, tmp, nil, nil, 1, NORMAL_PRIORITY_CLASS or CREATE_UNICODE_ENVIRONMENT,
ee, wwd, si, procInfo) ee, wwd, si, procInfo)
else: else:
success = winlean.createProcessA(nil, success = winlean.createProcessA(nil,
@ -455,7 +465,6 @@ when defined(Windows) and not defined(useNimRtl):
fileClose(si.hStdError) fileClose(si.hStdError)
if e != nil: dealloc(e) if e != nil: dealloc(e)
dealloc(cmdl)
if success == 0: osError(lastError) if success == 0: osError(lastError)
# Close the handle now so anyone waiting is woken: # Close the handle now so anyone waiting is woken:
discard closeHandle(procInfo.hThread) discard closeHandle(procInfo.hThread)
@ -495,7 +504,7 @@ when defined(Windows) and not defined(useNimRtl):
proc peekExitCode(p: PProcess): int = proc peekExitCode(p: PProcess): int =
var b = waitForSingleObject(p.fProcessHandle, 50) == WAIT_TIMEOUT var b = waitForSingleObject(p.fProcessHandle, 50) == WAIT_TIMEOUT
if b: result = -1 if b: result = -1
else: else:
var res: int32 var res: int32
discard getExitCodeProcess(p.fProcessHandle, res) discard getExitCodeProcess(p.fProcessHandle, res)
return res return res
@ -538,13 +547,13 @@ when defined(Windows) and not defined(useNimRtl):
result = -1 result = -1
discard closeHandle(process) discard closeHandle(process)
proc select(readfds: var seq[PProcess], timeout = 500): int = proc select(readfds: var seq[PProcess], timeout = 500): int =
assert readfds.len <= MAXIMUM_WAIT_OBJECTS assert readfds.len <= MAXIMUM_WAIT_OBJECTS
var rfds: TWOHandleArray var rfds: TWOHandleArray
for i in 0..readfds.len()-1: for i in 0..readfds.len()-1:
rfds[i] = readfds[i].fProcessHandle rfds[i] = readfds[i].fProcessHandle
var ret = waitForMultipleObjects(readfds.len.int32, var ret = waitForMultipleObjects(readfds.len.int32,
addr(rfds), 0'i32, timeout.int32) addr(rfds), 0'i32, timeout.int32)
case ret case ret
of WAIT_TIMEOUT: of WAIT_TIMEOUT:
@ -561,22 +570,7 @@ elif not defined(useNimRtl):
readIdx = 0 readIdx = 0
writeIdx = 1 writeIdx = 1
proc addCmdArgs(command: string, args: openArray[string]): string = proc envToCStringArray(t: PStringTable): cstringArray =
result = quoteShell(command)
for i in 0 .. high(args):
add(result, " ")
add(result, quoteShell(args[i]))
proc toCStringArray(b, a: openArray[string]): cstringArray =
result = cast[cstringArray](alloc0((a.len + b.len + 1) * sizeof(cstring)))
for i in 0..high(b):
result[i] = cast[cstring](alloc(b[i].len+1))
copyMem(result[i], cstring(b[i]), b[i].len+1)
for i in 0..high(a):
result[i+b.len] = cast[cstring](alloc(a[i].len+1))
copyMem(result[i+b.len], cstring(a[i]), a[i].len+1)
proc toCStringArray(t: PStringTable): cstringArray =
result = cast[cstringArray](alloc0((t.len + 1) * sizeof(cstring))) result = cast[cstringArray](alloc0((t.len + 1) * sizeof(cstring)))
var i = 0 var i = 0
for key, val in pairs(t): for key, val in pairs(t):
@ -595,7 +589,7 @@ elif not defined(useNimRtl):
result[i] = cast[cstring](alloc(x.len+1)) result[i] = cast[cstring](alloc(x.len+1))
copyMem(result[i], addr(x[0]), x.len+1) copyMem(result[i], addr(x[0]), x.len+1)
inc(i) inc(i)
proc startProcess(command: string, proc startProcess(command: string,
workingDir: string = "", workingDir: string = "",
args: openArray[string] = [], args: openArray[string] = [],
@ -609,23 +603,38 @@ elif not defined(useNimRtl):
if pipe(pStdin) != 0'i32 or pipe(pStdout) != 0'i32 or if pipe(pStdin) != 0'i32 or pipe(pStdout) != 0'i32 or
pipe(pStderr) != 0'i32: pipe(pStderr) != 0'i32:
osError(osLastError()) osError(osLastError())
var sys_command: string
var sys_args_raw: seq[string]
if poEvalCommand in options:
sys_command = "/bin/sh"
sys_args_raw = @[sys_command, "-c", command]
assert args.len == 0
else:
sys_command = command
sys_args_raw = @[command]
for arg in args.items:
sys_args_raw.add arg
var sys_args = allocCStringArray(sys_args_raw)
finally: deallocCStringArray(sys_args)
var pid: TPid var pid: TPid
when defined(posix_spawn) and not defined(useFork): when defined(posix_spawn) and not defined(useFork):
var attr: Tposix_spawnattr var attr: Tposix_spawnattr
var fops: Tposix_spawn_file_actions var fops: Tposix_spawn_file_actions
template chck(e: expr) = template chck(e: expr) =
if e != 0'i32: osError(osLastError()) if e != 0'i32: osError(osLastError())
chck posix_spawn_file_actions_init(fops) chck posix_spawn_file_actions_init(fops)
chck posix_spawnattr_init(attr) chck posix_spawnattr_init(attr)
var mask: Tsigset var mask: Tsigset
chck sigemptyset(mask) chck sigemptyset(mask)
chck posix_spawnattr_setsigmask(attr, mask) chck posix_spawnattr_setsigmask(attr, mask)
chck posix_spawnattr_setpgroup(attr, 0'i32) chck posix_spawnattr_setpgroup(attr, 0'i32)
chck posix_spawnattr_setflags(attr, POSIX_SPAWN_USEVFORK or chck posix_spawnattr_setflags(attr, POSIX_SPAWN_USEVFORK or
POSIX_SPAWN_SETSIGMASK or POSIX_SPAWN_SETSIGMASK or
POSIX_SPAWN_SETPGROUP) POSIX_SPAWN_SETPGROUP)
@ -639,27 +648,22 @@ elif not defined(useNimRtl):
if poStdErrToStdOut in options: if poStdErrToStdOut in options:
chck posix_spawn_file_actions_adddup2(fops, pStdout[writeIdx], 2) chck posix_spawn_file_actions_adddup2(fops, pStdout[writeIdx], 2)
else: else:
chck posix_spawn_file_actions_adddup2(fops, pStderr[writeIdx], 2) chck posix_spawn_file_actions_adddup2(fops, p_stderr[writeIdx], 2)
var e = if env == nil: envToCStringArray() else: toCStringArray(env) var sys_env = if env == nil: envToCStringArray() else: envToCStringArray(env)
var a: cstringArray
var res: cint var res: cint
# This is incorrect!
if workingDir.len > 0: os.setCurrentDir(workingDir) if workingDir.len > 0: os.setCurrentDir(workingDir)
if poUseShell notin options: if poUsePath in options:
a = toCStringArray([extractFilename(command)], args) res = posix_spawnp(pid, sys_command, fops, attr, sys_args, sys_env)
res = posix_spawn(pid, command, fops, attr, a, e)
else: else:
var x = addCmdArgs(command, args) res = posix_spawn(pid, sys_command, fops, attr, sys_args, sys_env)
a = toCStringArray(["sh", "-c"], [x]) deallocCStringArray(sys_env)
res = posix_spawn(pid, "/bin/sh", fops, attr, a, e)
deallocCStringArray(a)
deallocCStringArray(e)
discard posix_spawn_file_actions_destroy(fops) discard posix_spawn_file_actions_destroy(fops)
discard posix_spawnattr_destroy(attr) discard posix_spawnattr_destroy(attr)
chck res chck res
else: else:
pid = fork() pid = fork()
if pid < 0: osError(osLastError()) if pid < 0: osError(osLastError())
if pid == 0: if pid == 0:
@ -680,19 +684,18 @@ elif not defined(useNimRtl):
if setpgid(0, 0) == -1: quit("setpgid call failed: " & $strerror(errno)) if setpgid(0, 0) == -1: quit("setpgid call failed: " & $strerror(errno))
if workingDir.len > 0: os.setCurrentDir(workingDir) if workingDir.len > 0: os.setCurrentDir(workingDir)
if poUseShell notin options:
var a = toCStringArray([extractFilename(command)], args) if env == nil:
if env == nil: if poUsePath in options:
discard execv(command, a) discard execvp(sys_command, sys_args)
else: else:
discard execve(command, a, toCStringArray(env)) discard execv(sys_command, sys_args)
else: else:
var x = addCmdArgs(command, args) var c_env = envToCStringArray(env)
var a = toCStringArray(["sh", "-c"], [x]) if poUsePath in options:
if env == nil: discard execvpe(sys_command, sys_args, c_env)
discard execv("/bin/sh", a)
else: else:
discard execve("/bin/sh", a, toCStringArray(env)) discard execve(sys_command, sys_args, c_env)
# too risky to raise an exception here: # too risky to raise an exception here:
quit("execve call failed: " & $strerror(errno)) quit("execve call failed: " & $strerror(errno))
# Parent process. Copy process information. # Parent process. Copy process information.
@ -790,13 +793,13 @@ elif not defined(useNimRtl):
proc execCmd(command: string): int = proc execCmd(command: string): int =
result = csystem(command) result = csystem(command)
proc createFdSet(fd: var TFdSet, s: seq[PProcess], m: var int) = proc createFdSet(fd: var TFdSet, s: seq[PProcess], m: var int) =
FD_ZERO(fd) FD_ZERO(fd)
for i in items(s): for i in items(s):
m = max(m, int(i.outHandle)) m = max(m, int(i.outHandle))
FD_SET(cint(i.outHandle), fd) FD_SET(cint(i.outHandle), fd)
proc pruneProcessSet(s: var seq[PProcess], fd: var TFdSet) = proc pruneProcessSet(s: var seq[PProcess], fd: var TFdSet) =
var i = 0 var i = 0
var L = s.len var L = s.len
while i < L: while i < L:
@ -807,26 +810,26 @@ elif not defined(useNimRtl):
inc(i) inc(i)
setLen(s, L) setLen(s, L)
proc select(readfds: var seq[PProcess], timeout = 500): int = proc select(readfds: var seq[PProcess], timeout = 500): int =
var tv: Ttimeval var tv: TTimeVal
tv.tv_sec = 0 tv.tv_sec = 0
tv.tv_usec = timeout * 1000 tv.tv_usec = timeout * 1000
var rd: TFdSet var rd: TFdSet
var m = 0 var m = 0
createFdSet((rd), readfds, m) createFdSet((rd), readfds, m)
if timeout != -1: if timeout != -1:
result = int(select(cint(m+1), addr(rd), nil, nil, addr(tv))) result = int(select(cint(m+1), addr(rd), nil, nil, addr(tv)))
else: else:
result = int(select(cint(m+1), addr(rd), nil, nil, nil)) result = int(select(cint(m+1), addr(rd), nil, nil, nil))
pruneProcessSet(readfds, (rd)) pruneProcessSet(readfds, (rd))
proc execCmdEx*(command: string, options: set[TProcessOption] = { proc execCmdEx*(command: string, options: set[TProcessOption] = {
poStdErrToStdOut, poUseShell}): tuple[ poStdErrToStdOut, poUsePath}): tuple[
output: TaintedString, output: TaintedString,
exitCode: int] {.tags: [FExecIO, FReadIO].} = exitCode: int] {.tags: [FExecIO, FReadIO].} =
## a convenience proc that runs the `command`, grabs all its output and ## a convenience proc that runs the `command`, grabs all its output and
## exit code and returns both. ## exit code and returns both.