Revision c74cda62 lib/utils.py
b/lib/utils.py | ||
---|---|---|
96 | 96 |
#: MAC checker regexp |
97 | 97 |
_MAC_CHECK = re.compile("^([0-9a-f]{2}:){5}[0-9a-f]{2}$", re.I) |
98 | 98 |
|
99 |
(_TIMEOUT_NONE, |
|
100 |
_TIMEOUT_TERM, |
|
101 |
_TIMEOUT_KILL) = range(3) |
|
102 |
|
|
99 | 103 |
|
100 | 104 |
class RunResult(object): |
101 | 105 |
"""Holds the result of running external programs. |
... | ... | |
120 | 124 |
"failed", "fail_reason", "cmd"] |
121 | 125 |
|
122 | 126 |
|
123 |
def __init__(self, exit_code, signal_, stdout, stderr, cmd): |
|
127 |
def __init__(self, exit_code, signal_, stdout, stderr, cmd, timeout_action, |
|
128 |
timeout): |
|
124 | 129 |
self.cmd = cmd |
125 | 130 |
self.exit_code = exit_code |
126 | 131 |
self.signal = signal_ |
... | ... | |
128 | 133 |
self.stderr = stderr |
129 | 134 |
self.failed = (signal_ is not None or exit_code != 0) |
130 | 135 |
|
136 |
fail_msgs = [] |
|
131 | 137 |
if self.signal is not None: |
132 |
self.fail_reason = "terminated by signal %s" % self.signal
|
|
138 |
fail_msgs.append("terminated by signal %s" % self.signal)
|
|
133 | 139 |
elif self.exit_code is not None: |
134 |
self.fail_reason = "exited with exit code %s" % self.exit_code
|
|
140 |
fail_msgs.append("exited with exit code %s" % self.exit_code)
|
|
135 | 141 |
else: |
136 |
self.fail_reason = "unable to determine termination reason" |
|
142 |
fail_msgs.append("unable to determine termination reason") |
|
143 |
|
|
144 |
if timeout_action == _TIMEOUT_TERM: |
|
145 |
fail_msgs.append("terminated after timeout of %.2f seconds" % timeout) |
|
146 |
elif timeout_action == _TIMEOUT_KILL: |
|
147 |
fail_msgs.append(("force termination after timeout of %.2f seconds" |
|
148 |
" and linger for another %.2f seconds") % |
|
149 |
(timeout, constants.CHILD_LINGER_TIMEOUT)) |
|
150 |
|
|
151 |
if fail_msgs and self.failed: |
|
152 |
self.fail_reason = CommaJoin(fail_msgs) |
|
137 | 153 |
|
138 | 154 |
if self.failed: |
139 | 155 |
logging.debug("Command '%s' failed (%s); output: %s", |
... | ... | |
165 | 181 |
|
166 | 182 |
|
167 | 183 |
def RunCmd(cmd, env=None, output=None, cwd="/", reset_env=False, |
168 |
interactive=False): |
|
184 |
interactive=False, timeout=None):
|
|
169 | 185 |
"""Execute a (shell) command. |
170 | 186 |
|
171 | 187 |
The command should not read from its standard input, as it will be |
... | ... | |
187 | 203 |
@type interactive: boolean |
188 | 204 |
@param interactive: weather we pipe stdin, stdout and stderr |
189 | 205 |
(default behaviour) or run the command interactive |
206 |
@type timeout: int |
|
207 |
@param timeout: If not None, timeout in seconds until child process gets |
|
208 |
killed |
|
190 | 209 |
@rtype: L{RunResult} |
191 | 210 |
@return: RunResult instance |
192 | 211 |
@raise errors.ProgrammerError: if we call this when forks are disabled |
... | ... | |
216 | 235 |
|
217 | 236 |
try: |
218 | 237 |
if output is None: |
219 |
out, err, status = _RunCmdPipe(cmd, cmd_env, shell, cwd, interactive) |
|
238 |
out, err, status, timeout_action = _RunCmdPipe(cmd, cmd_env, shell, cwd, |
|
239 |
interactive, timeout) |
|
220 | 240 |
else: |
241 |
timeout_action = _TIMEOUT_NONE |
|
221 | 242 |
status = _RunCmdFile(cmd, cmd_env, shell, output, cwd) |
222 | 243 |
out = err = "" |
223 | 244 |
except OSError, err: |
... | ... | |
234 | 255 |
exitcode = None |
235 | 256 |
signal_ = -status |
236 | 257 |
|
237 |
return RunResult(exitcode, signal_, out, err, strcmd) |
|
258 |
return RunResult(exitcode, signal_, out, err, strcmd, timeout_action, timeout)
|
|
238 | 259 |
|
239 | 260 |
|
240 | 261 |
def SetupDaemonEnv(cwd="/", umask=077): |
... | ... | |
460 | 481 |
RetryOnSignal(os.write, fd, err) |
461 | 482 |
|
462 | 483 |
|
463 |
def _RunCmdPipe(cmd, env, via_shell, cwd, interactive): |
|
484 |
def _CheckIfAlive(child): |
|
485 |
"""Raises L{RetryAgain} if child is still alive. |
|
486 |
|
|
487 |
@raises RetryAgain: If child is still alive |
|
488 |
|
|
489 |
""" |
|
490 |
if child.poll() is None: |
|
491 |
raise RetryAgain() |
|
492 |
|
|
493 |
|
|
494 |
def _WaitForProcess(child, timeout): |
|
495 |
"""Waits for the child to terminate or until we reach timeout. |
|
496 |
|
|
497 |
""" |
|
498 |
try: |
|
499 |
Retry(_CheckIfAlive, (1.0, 1.2, 5.0), max(0, timeout), args=[child]) |
|
500 |
except RetryTimeout: |
|
501 |
pass |
|
502 |
|
|
503 |
|
|
504 |
def _RunCmdPipe(cmd, env, via_shell, cwd, interactive, timeout, |
|
505 |
_linger_timeout=constants.CHILD_LINGER_TIMEOUT): |
|
464 | 506 |
"""Run a command and return its output. |
465 | 507 |
|
466 | 508 |
@type cmd: string or list |
... | ... | |
473 | 515 |
@param cwd: the working directory for the program |
474 | 516 |
@type interactive: boolean |
475 | 517 |
@param interactive: Run command interactive (without piping) |
518 |
@type timeout: int |
|
519 |
@param timeout: Timeout after the programm gets terminated |
|
476 | 520 |
@rtype: tuple |
477 | 521 |
@return: (out, err, status) |
478 | 522 |
|
... | ... | |
495 | 539 |
|
496 | 540 |
out = StringIO() |
497 | 541 |
err = StringIO() |
542 |
|
|
543 |
linger_timeout = None |
|
544 |
|
|
545 |
if timeout is None: |
|
546 |
poll_timeout = None |
|
547 |
else: |
|
548 |
poll_timeout = RunningTimeout(timeout, True).Remaining |
|
549 |
|
|
550 |
msg_timeout = ("Command %s (%d) run into execution timeout, terminating" % |
|
551 |
(cmd, child.pid)) |
|
552 |
msg_linger = ("Command %s (%d) run into linger timeout, killing" % |
|
553 |
(cmd, child.pid)) |
|
554 |
|
|
555 |
timeout_action = _TIMEOUT_NONE |
|
556 |
|
|
498 | 557 |
if not interactive: |
499 | 558 |
child.stdin.close() |
500 | 559 |
poller.register(child.stdout, select.POLLIN) |
... | ... | |
507 | 566 |
SetNonblockFlag(fd, True) |
508 | 567 |
|
509 | 568 |
while fdmap: |
510 |
pollresult = RetryOnSignal(poller.poll) |
|
569 |
if poll_timeout: |
|
570 |
current_timeout = poll_timeout() |
|
571 |
if current_timeout < 0: |
|
572 |
if linger_timeout is None: |
|
573 |
logging.warning(msg_timeout) |
|
574 |
if child.poll() is None: |
|
575 |
timeout_action = _TIMEOUT_TERM |
|
576 |
IgnoreProcessNotFound(os.kill, child.pid, signal.SIGTERM) |
|
577 |
linger_timeout = RunningTimeout(_linger_timeout, True).Remaining |
|
578 |
lt = linger_timeout() |
|
579 |
if lt < 0: |
|
580 |
break |
|
581 |
|
|
582 |
pt = max(0, lt) |
|
583 |
else: |
|
584 |
pt = current_timeout |
|
585 |
else: |
|
586 |
pt = None |
|
587 |
|
|
588 |
pollresult = RetryOnSignal(poller.poll, pt) |
|
511 | 589 |
|
512 | 590 |
for fd, event in pollresult: |
513 | 591 |
if event & select.POLLIN or event & select.POLLPRI: |
... | ... | |
523 | 601 |
poller.unregister(fd) |
524 | 602 |
del fdmap[fd] |
525 | 603 |
|
604 |
if timeout is not None: |
|
605 |
assert callable(poll_timeout) |
|
606 |
|
|
607 |
# We have no I/O left but it might still run |
|
608 |
if child.poll() is None: |
|
609 |
_WaitForProcess(child, poll_timeout()) |
|
610 |
|
|
611 |
# Terminate if still alive after timeout |
|
612 |
if child.poll() is None: |
|
613 |
if linger_timeout is None: |
|
614 |
logging.warning(msg_timeout) |
|
615 |
timeout_action = _TIMEOUT_TERM |
|
616 |
IgnoreProcessNotFound(os.kill, child.pid, signal.SIGTERM) |
|
617 |
lt = _linger_timeout |
|
618 |
else: |
|
619 |
lt = linger_timeout() |
|
620 |
_WaitForProcess(child, lt) |
|
621 |
|
|
622 |
# Okay, still alive after timeout and linger timeout? Kill it! |
|
623 |
if child.poll() is None: |
|
624 |
timeout_action = _TIMEOUT_KILL |
|
625 |
logging.warning(msg_linger) |
|
626 |
IgnoreProcessNotFound(os.kill, child.pid, signal.SIGKILL) |
|
627 |
|
|
526 | 628 |
out = out.getvalue() |
527 | 629 |
err = err.getvalue() |
528 | 630 |
|
529 | 631 |
status = child.wait() |
530 |
return out, err, status |
|
632 |
return out, err, status, timeout_action
|
|
531 | 633 |
|
532 | 634 |
|
533 | 635 |
def _RunCmdFile(cmd, env, via_shell, output, cwd): |
Also available in: Unified diff