Revision c74cda62 lib/utils.py

b/lib/utils.py
96 96
#: MAC checker regexp
97 97
_MAC_CHECK = re.compile("^([0-9a-f]{2}:){5}[0-9a-f]{2}$", re.I)
98 98

  
99
(_TIMEOUT_NONE,
100
 _TIMEOUT_TERM,
101
 _TIMEOUT_KILL) = range(3)
102

  
99 103

  
100 104
class RunResult(object):
101 105
  """Holds the result of running external programs.
......
120 124
               "failed", "fail_reason", "cmd"]
121 125

  
122 126

  
123
  def __init__(self, exit_code, signal_, stdout, stderr, cmd):
127
  def __init__(self, exit_code, signal_, stdout, stderr, cmd, timeout_action,
128
               timeout):
124 129
    self.cmd = cmd
125 130
    self.exit_code = exit_code
126 131
    self.signal = signal_
......
128 133
    self.stderr = stderr
129 134
    self.failed = (signal_ is not None or exit_code != 0)
130 135

  
136
    fail_msgs = []
131 137
    if self.signal is not None:
132
      self.fail_reason = "terminated by signal %s" % self.signal
138
      fail_msgs.append("terminated by signal %s" % self.signal)
133 139
    elif self.exit_code is not None:
134
      self.fail_reason = "exited with exit code %s" % self.exit_code
140
      fail_msgs.append("exited with exit code %s" % self.exit_code)
135 141
    else:
136
      self.fail_reason = "unable to determine termination reason"
142
      fail_msgs.append("unable to determine termination reason")
143

  
144
    if timeout_action == _TIMEOUT_TERM:
145
      fail_msgs.append("terminated after timeout of %.2f seconds" % timeout)
146
    elif timeout_action == _TIMEOUT_KILL:
147
      fail_msgs.append(("force termination after timeout of %.2f seconds"
148
                        " and linger for another %.2f seconds") %
149
                       (timeout, constants.CHILD_LINGER_TIMEOUT))
150

  
151
    if fail_msgs and self.failed:
152
      self.fail_reason = CommaJoin(fail_msgs)
137 153

  
138 154
    if self.failed:
139 155
      logging.debug("Command '%s' failed (%s); output: %s",
......
165 181

  
166 182

  
167 183
def RunCmd(cmd, env=None, output=None, cwd="/", reset_env=False,
168
           interactive=False):
184
           interactive=False, timeout=None):
169 185
  """Execute a (shell) command.
170 186

  
171 187
  The command should not read from its standard input, as it will be
......
187 203
  @type interactive: boolean
188 204
  @param interactive: weather we pipe stdin, stdout and stderr
189 205
                      (default behaviour) or run the command interactive
206
  @type timeout: int
207
  @param timeout: If not None, timeout in seconds until child process gets
208
                  killed
190 209
  @rtype: L{RunResult}
191 210
  @return: RunResult instance
192 211
  @raise errors.ProgrammerError: if we call this when forks are disabled
......
216 235

  
217 236
  try:
218 237
    if output is None:
219
      out, err, status = _RunCmdPipe(cmd, cmd_env, shell, cwd, interactive)
238
      out, err, status, timeout_action = _RunCmdPipe(cmd, cmd_env, shell, cwd,
239
                                                     interactive, timeout)
220 240
    else:
241
      timeout_action = _TIMEOUT_NONE
221 242
      status = _RunCmdFile(cmd, cmd_env, shell, output, cwd)
222 243
      out = err = ""
223 244
  except OSError, err:
......
234 255
    exitcode = None
235 256
    signal_ = -status
236 257

  
237
  return RunResult(exitcode, signal_, out, err, strcmd)
258
  return RunResult(exitcode, signal_, out, err, strcmd, timeout_action, timeout)
238 259

  
239 260

  
240 261
def SetupDaemonEnv(cwd="/", umask=077):
......
460 481
  RetryOnSignal(os.write, fd, err)
461 482

  
462 483

  
463
def _RunCmdPipe(cmd, env, via_shell, cwd, interactive):
484
def _CheckIfAlive(child):
485
  """Raises L{RetryAgain} if child is still alive.
486

  
487
  @raises RetryAgain: If child is still alive
488

  
489
  """
490
  if child.poll() is None:
491
    raise RetryAgain()
492

  
493

  
494
def _WaitForProcess(child, timeout):
495
  """Waits for the child to terminate or until we reach timeout.
496

  
497
  """
498
  try:
499
    Retry(_CheckIfAlive, (1.0, 1.2, 5.0), max(0, timeout), args=[child])
500
  except RetryTimeout:
501
    pass
502

  
503

  
504
def _RunCmdPipe(cmd, env, via_shell, cwd, interactive, timeout,
505
                _linger_timeout=constants.CHILD_LINGER_TIMEOUT):
464 506
  """Run a command and return its output.
465 507

  
466 508
  @type  cmd: string or list
......
473 515
  @param cwd: the working directory for the program
474 516
  @type interactive: boolean
475 517
  @param interactive: Run command interactive (without piping)
518
  @type timeout: int
519
  @param timeout: Timeout after the programm gets terminated
476 520
  @rtype: tuple
477 521
  @return: (out, err, status)
478 522

  
......
495 539

  
496 540
  out = StringIO()
497 541
  err = StringIO()
542

  
543
  linger_timeout = None
544

  
545
  if timeout is None:
546
    poll_timeout = None
547
  else:
548
    poll_timeout = RunningTimeout(timeout, True).Remaining
549

  
550
  msg_timeout = ("Command %s (%d) run into execution timeout, terminating" %
551
                 (cmd, child.pid))
552
  msg_linger = ("Command %s (%d) run into linger timeout, killing" %
553
                (cmd, child.pid))
554

  
555
  timeout_action = _TIMEOUT_NONE
556

  
498 557
  if not interactive:
499 558
    child.stdin.close()
500 559
    poller.register(child.stdout, select.POLLIN)
......
507 566
      SetNonblockFlag(fd, True)
508 567

  
509 568
    while fdmap:
510
      pollresult = RetryOnSignal(poller.poll)
569
      if poll_timeout:
570
        current_timeout = poll_timeout()
571
        if current_timeout < 0:
572
          if linger_timeout is None:
573
            logging.warning(msg_timeout)
574
            if child.poll() is None:
575
              timeout_action = _TIMEOUT_TERM
576
              IgnoreProcessNotFound(os.kill, child.pid, signal.SIGTERM)
577
            linger_timeout = RunningTimeout(_linger_timeout, True).Remaining
578
          lt = linger_timeout()
579
          if lt < 0:
580
            break
581

  
582
          pt = max(0, lt)
583
        else:
584
          pt = current_timeout
585
      else:
586
        pt = None
587

  
588
      pollresult = RetryOnSignal(poller.poll, pt)
511 589

  
512 590
      for fd, event in pollresult:
513 591
        if event & select.POLLIN or event & select.POLLPRI:
......
523 601
          poller.unregister(fd)
524 602
          del fdmap[fd]
525 603

  
604
  if timeout is not None:
605
    assert callable(poll_timeout)
606

  
607
    # We have no I/O left but it might still run
608
    if child.poll() is None:
609
      _WaitForProcess(child, poll_timeout())
610

  
611
    # Terminate if still alive after timeout
612
    if child.poll() is None:
613
      if linger_timeout is None:
614
        logging.warning(msg_timeout)
615
        timeout_action = _TIMEOUT_TERM
616
        IgnoreProcessNotFound(os.kill, child.pid, signal.SIGTERM)
617
        lt = _linger_timeout
618
      else:
619
        lt = linger_timeout()
620
      _WaitForProcess(child, lt)
621

  
622
    # Okay, still alive after timeout and linger timeout? Kill it!
623
    if child.poll() is None:
624
      timeout_action = _TIMEOUT_KILL
625
      logging.warning(msg_linger)
626
      IgnoreProcessNotFound(os.kill, child.pid, signal.SIGKILL)
627

  
526 628
  out = out.getvalue()
527 629
  err = err.getvalue()
528 630

  
529 631
  status = child.wait()
530
  return out, err, status
632
  return out, err, status, timeout_action
531 633

  
532 634

  
533 635
def _RunCmdFile(cmd, env, via_shell, output, cwd):

Also available in: Unified diff