| Branch: | Tag: | Revision:

root / lib / @ f3044516

History | View | Annotate | Download (102.6 kB)

1 2f31098c Iustin Pop
2 a8083063 Iustin Pop
3 a8083063 Iustin Pop
4 600535f0 Manuel Franceschini
# Copyright (C) 2006, 2007, 2010 Google Inc.
5 a8083063 Iustin Pop
6 a8083063 Iustin Pop
# This program is free software; you can redistribute it and/or modify
7 a8083063 Iustin Pop
# it under the terms of the GNU General Public License as published by
8 a8083063 Iustin Pop
# the Free Software Foundation; either version 2 of the License, or
9 a8083063 Iustin Pop
# (at your option) any later version.
10 a8083063 Iustin Pop
11 a8083063 Iustin Pop
# This program is distributed in the hope that it will be useful, but
12 a8083063 Iustin Pop
# WITHOUT ANY WARRANTY; without even the implied warranty of
13 a8083063 Iustin Pop
14 a8083063 Iustin Pop
# General Public License for more details.
15 a8083063 Iustin Pop
16 a8083063 Iustin Pop
# You should have received a copy of the GNU General Public License
17 a8083063 Iustin Pop
# along with this program; if not, write to the Free Software
18 a8083063 Iustin Pop
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
19 a8083063 Iustin Pop
# 02110-1301, USA.
20 a8083063 Iustin Pop
21 a8083063 Iustin Pop
22 58885d79 Iustin Pop
"""Ganeti utility module.
23 58885d79 Iustin Pop

24 58885d79 Iustin Pop
This module holds functions that can be used in both daemons (all) and
25 58885d79 Iustin Pop
the command line scripts.
26 899d2a81 Michael Hanselmann

27 a8083063 Iustin Pop
28 a8083063 Iustin Pop
29 a8083063 Iustin Pop
30 a8083063 Iustin Pop
import os
31 c1dd99d4 Michael Hanselmann
import sys
32 a8083063 Iustin Pop
import time
33 113b55aa Iustin Pop
import subprocess
34 a8083063 Iustin Pop
import re
35 a8083063 Iustin Pop
import socket
36 a8083063 Iustin Pop
import tempfile
37 a8083063 Iustin Pop
import shutil
38 4ca1b175 Alexander Schreiber
import errno
39 2f8b60b3 Iustin Pop
import pwd
40 78feb6fb Guido Trotter
import itertools
41 9c233417 Iustin Pop
import select
42 9c233417 Iustin Pop
import fcntl
43 8f765069 Iustin Pop
import resource
44 bb698c1f Iustin Pop
import logging
45 551b6283 Iustin Pop
import logging.handlers
46 de499029 Michael Hanselmann
import signal
47 bdd5e420 Michael Hanselmann
import OpenSSL
48 27e46076 Michael Hanselmann
import datetime
49 27e46076 Michael Hanselmann
import calendar
50 68857643 Michael Hanselmann
import hmac
51 339be5a8 Michael Hanselmann
import collections
52 9c233417 Iustin Pop
53 9c233417 Iustin Pop
from cStringIO import StringIO
54 a8083063 Iustin Pop
55 7ffe8fba Carlos Valiente
56 23e0ef8c Guido Trotter
  # pylint: disable-msg=F0401
57 4b6fa0bf Luca Bigliardi
  import ctypes
58 4b6fa0bf Luca Bigliardi
except ImportError:
59 4b6fa0bf Luca Bigliardi
  ctypes = None
60 4b6fa0bf Luca Bigliardi
61 a8083063 Iustin Pop
from ganeti import errors
62 3aecd2c7 Iustin Pop
from ganeti import constants
63 716a32cb Guido Trotter
from ganeti import compat
64 a8083063 Iustin Pop
65 16abfbc2 Alexander Schreiber
66 a8083063 Iustin Pop
_locksheld = []
67 a8083063 Iustin Pop
_re_shell_unquoted = re.compile('^[-.,=:/_+@A-Za-z0-9]+$')
68 a8083063 Iustin Pop
69 e67bd559 Michael Hanselmann
debug_locks = False
70 58885d79 Iustin Pop
71 58885d79 Iustin Pop
#: when set to True, L{RunCmd} is disabled
72 b74159ee Iustin Pop
no_fork = False
73 f362096f Iustin Pop
74 13998ef2 Michael Hanselmann
_RANDOM_UUID_FILE = "/proc/sys/kernel/random/uuid"
75 13998ef2 Michael Hanselmann
76 68857643 Michael Hanselmann
HEX_CHAR_RE = r"[a-zA-Z0-9]"
77 68857643 Michael Hanselmann
VALID_X509_SIGNATURE_SALT = re.compile("^%s+$" % HEX_CHAR_RE, re.S)
78 68857643 Michael Hanselmann
X509_SIGNATURE = re.compile(r"^%s:\s*(?P<salt>%s+)/(?P<sign>%s+)$" %
79 68857643 Michael Hanselmann
80 68857643 Michael Hanselmann
                             HEX_CHAR_RE, HEX_CHAR_RE),
81 68857643 Michael Hanselmann
                            re.S | re.I)
82 68857643 Michael Hanselmann
83 28f34048 Michael Hanselmann
_VALID_SERVICE_NAME_RE = re.compile("^[-_.a-zA-Z0-9]{1,128}$")
84 28f34048 Michael Hanselmann
85 24d70417 Michael Hanselmann
# Certificate verification results
86 24d70417 Michael Hanselmann
87 24d70417 Michael Hanselmann
 CERT_ERROR) = range(1, 3)
88 24d70417 Michael Hanselmann
89 4b6fa0bf Luca Bigliardi
# Flags for mlockall() (from bits/mman.h)
90 4b6fa0bf Luca Bigliardi
91 4b6fa0bf Luca Bigliardi
92 4b6fa0bf Luca Bigliardi
93 7c0d6283 Michael Hanselmann
94 a8083063 Iustin Pop
class RunResult(object):
95 58885d79 Iustin Pop
  """Holds the result of running external programs.
96 58885d79 Iustin Pop

97 58885d79 Iustin Pop
  @type exit_code: int
98 58885d79 Iustin Pop
  @ivar exit_code: the exit code of the program, or None (if the program
99 58885d79 Iustin Pop
      didn't exit())
100 58885d79 Iustin Pop
  @type signal: int or None
101 58885d79 Iustin Pop
  @ivar signal: the signal that caused the program to finish, or None
102 58885d79 Iustin Pop
      (if the program wasn't terminated by a signal)
103 58885d79 Iustin Pop
  @type stdout: str
104 58885d79 Iustin Pop
  @ivar stdout: the standard output of the program
105 58885d79 Iustin Pop
  @type stderr: str
106 58885d79 Iustin Pop
  @ivar stderr: the standard error of the program
107 58885d79 Iustin Pop
  @type failed: boolean
108 58885d79 Iustin Pop
  @ivar failed: True in case the program was
109 58885d79 Iustin Pop
      terminated by a signal or exited with a non-zero exit code
110 58885d79 Iustin Pop
  @ivar fail_reason: a string detailing the termination reason
111 a8083063 Iustin Pop

112 a8083063 Iustin Pop
113 a8083063 Iustin Pop
  __slots__ = ["exit_code", "signal", "stdout", "stderr",
114 a8083063 Iustin Pop
               "failed", "fail_reason", "cmd"]
115 a8083063 Iustin Pop
116 a8083063 Iustin Pop
117 38206f3c Iustin Pop
  def __init__(self, exit_code, signal_, stdout, stderr, cmd):
118 a8083063 Iustin Pop
    self.cmd = cmd
119 a8083063 Iustin Pop
    self.exit_code = exit_code
120 38206f3c Iustin Pop
    self.signal = signal_
121 a8083063 Iustin Pop
    self.stdout = stdout
122 a8083063 Iustin Pop
    self.stderr = stderr
123 38206f3c Iustin Pop
    self.failed = (signal_ is not None or exit_code != 0)
124 a8083063 Iustin Pop
125 a8083063 Iustin Pop
    if self.signal is not None:
126 a8083063 Iustin Pop
      self.fail_reason = "terminated by signal %s" % self.signal
127 a8083063 Iustin Pop
    elif self.exit_code is not None:
128 a8083063 Iustin Pop
      self.fail_reason = "exited with exit code %s" % self.exit_code
129 a8083063 Iustin Pop
130 a8083063 Iustin Pop
      self.fail_reason = "unable to determine termination reason"
131 a8083063 Iustin Pop
132 bb698c1f Iustin Pop
    if self.failed:
133 bb698c1f Iustin Pop
      logging.debug("Command '%s' failed (%s); output: %s",
134 bb698c1f Iustin Pop
                    self.cmd, self.fail_reason, self.output)
135 f362096f Iustin Pop
136 a8083063 Iustin Pop
  def _GetOutput(self):
137 a8083063 Iustin Pop
    """Returns the combined stdout and stderr for easier usage.
138 a8083063 Iustin Pop

139 a8083063 Iustin Pop
140 a8083063 Iustin Pop
    return self.stdout + self.stderr
141 a8083063 Iustin Pop
142 a8083063 Iustin Pop
  output = property(_GetOutput, None, None, "Return full output")
143 a8083063 Iustin Pop
144 a8083063 Iustin Pop
145 bb3776b4 Michael Hanselmann
def _BuildCmdEnvironment(env, reset):
146 c1dd99d4 Michael Hanselmann
  """Builds the environment for an external program.
147 c1dd99d4 Michael Hanselmann

148 c1dd99d4 Michael Hanselmann
149 bb3776b4 Michael Hanselmann
  if reset:
150 bb3776b4 Michael Hanselmann
    cmd_env = {}
151 bb3776b4 Michael Hanselmann
152 bb3776b4 Michael Hanselmann
    cmd_env = os.environ.copy()
153 bb3776b4 Michael Hanselmann
    cmd_env["LC_ALL"] = "C"
154 bb3776b4 Michael Hanselmann
155 c1dd99d4 Michael Hanselmann
  if env is not None:
156 c1dd99d4 Michael Hanselmann
157 bb3776b4 Michael Hanselmann
158 c1dd99d4 Michael Hanselmann
  return cmd_env
159 c1dd99d4 Michael Hanselmann
160 c1dd99d4 Michael Hanselmann
161 0963d545 René Nussbaumer
def RunCmd(cmd, env=None, output=None, cwd="/", reset_env=False,
162 0963d545 René Nussbaumer
163 a8083063 Iustin Pop
  """Execute a (shell) command.
164 a8083063 Iustin Pop

165 a8083063 Iustin Pop
  The command should not read from its standard input, as it will be
166 a8083063 Iustin Pop
167 a8083063 Iustin Pop

168 c1dd99d4 Michael Hanselmann
  @type cmd: string or list
169 36117c2b Iustin Pop
  @param cmd: Command to run
170 2557ff82 Guido Trotter
  @type env: dict
171 c1dd99d4 Michael Hanselmann
  @param env: Additional environment variables
172 36117c2b Iustin Pop
  @type output: str
173 58885d79 Iustin Pop
  @param output: if desired, the output of the command can be
174 36117c2b Iustin Pop
      saved in a file instead of the RunResult instance; this
175 36117c2b Iustin Pop
      parameter denotes the file name (if not None)
176 8797df43 Iustin Pop
  @type cwd: string
177 8797df43 Iustin Pop
  @param cwd: if specified, will be used as the working
178 8797df43 Iustin Pop
      directory for the command; the default will be /
179 bf4daac9 Guido Trotter
  @type reset_env: boolean
180 bf4daac9 Guido Trotter
  @param reset_env: whether to reset or keep the default os environment
181 0963d545 René Nussbaumer
  @type interactive: boolean
182 0963d545 René Nussbaumer
  @param interactive: weather we pipe stdin, stdout and stderr
183 0963d545 René Nussbaumer
                      (default behaviour) or run the command interactive
184 36117c2b Iustin Pop
  @rtype: L{RunResult}
185 58885d79 Iustin Pop
  @return: RunResult instance
186 5bbd3f7f Michael Hanselmann
  @raise errors.ProgrammerError: if we call this when forks are disabled
187 a8083063 Iustin Pop

188 a8083063 Iustin Pop
189 b74159ee Iustin Pop
  if no_fork:
190 b74159ee Iustin Pop
    raise errors.ProgrammerError("utils.RunCmd() called with fork() disabled")
191 b74159ee Iustin Pop
192 0963d545 René Nussbaumer
  if output and interactive:
193 0963d545 René Nussbaumer
    raise errors.ProgrammerError("Parameters 'output' and 'interactive' can"
194 0963d545 René Nussbaumer
                                 " not be provided at the same time")
195 0963d545 René Nussbaumer
196 c1dd99d4 Michael Hanselmann
  if isinstance(cmd, basestring):
197 c1dd99d4 Michael Hanselmann
    strcmd = cmd
198 c1dd99d4 Michael Hanselmann
    shell = True
199 c1dd99d4 Michael Hanselmann
200 a8083063 Iustin Pop
    cmd = [str(val) for val in cmd]
201 c1dd99d4 Michael Hanselmann
    strcmd = ShellQuoteArgs(cmd)
202 113b55aa Iustin Pop
    shell = False
203 c1dd99d4 Michael Hanselmann
204 c1dd99d4 Michael Hanselmann
  if output:
205 c1dd99d4 Michael Hanselmann
    logging.debug("RunCmd %s, output file '%s'", strcmd, output)
206 113b55aa Iustin Pop
207 c1dd99d4 Michael Hanselmann
    logging.debug("RunCmd %s", strcmd)
208 2557ff82 Guido Trotter
209 bb3776b4 Michael Hanselmann
  cmd_env = _BuildCmdEnvironment(env, reset_env)
210 2557ff82 Guido Trotter
211 c803b052 Iustin Pop
212 c803b052 Iustin Pop
    if output is None:
213 0963d545 René Nussbaumer
      out, err, status = _RunCmdPipe(cmd, cmd_env, shell, cwd, interactive)
214 c803b052 Iustin Pop
215 c803b052 Iustin Pop
      status = _RunCmdFile(cmd, cmd_env, shell, output, cwd)
216 c803b052 Iustin Pop
      out = err = ""
217 c803b052 Iustin Pop
  except OSError, err:
218 c803b052 Iustin Pop
    if err.errno == errno.ENOENT:
219 c803b052 Iustin Pop
      raise errors.OpExecError("Can't execute '%s': not found (%s)" %
220 c803b052 Iustin Pop
                               (strcmd, err))
221 c803b052 Iustin Pop
222 c803b052 Iustin Pop
223 36117c2b Iustin Pop
224 36117c2b Iustin Pop
  if status >= 0:
225 36117c2b Iustin Pop
    exitcode = status
226 36117c2b Iustin Pop
    signal_ = None
227 36117c2b Iustin Pop
228 36117c2b Iustin Pop
    exitcode = None
229 36117c2b Iustin Pop
    signal_ = -status
230 36117c2b Iustin Pop
231 36117c2b Iustin Pop
  return RunResult(exitcode, signal_, out, err, strcmd)
232 36117c2b Iustin Pop
233 ae59efea Michael Hanselmann
234 c1dd99d4 Michael Hanselmann
def StartDaemon(cmd, env=None, cwd="/", output=None, output_fd=None,
235 c1dd99d4 Michael Hanselmann
236 c1dd99d4 Michael Hanselmann
  """Start a daemon process after forking twice.
237 c1dd99d4 Michael Hanselmann

238 c1dd99d4 Michael Hanselmann
  @type cmd: string or list
239 c1dd99d4 Michael Hanselmann
  @param cmd: Command to run
240 c1dd99d4 Michael Hanselmann
  @type env: dict
241 c1dd99d4 Michael Hanselmann
  @param env: Additional environment variables
242 c1dd99d4 Michael Hanselmann
  @type cwd: string
243 c1dd99d4 Michael Hanselmann
  @param cwd: Working directory for the program
244 c1dd99d4 Michael Hanselmann
  @type output: string
245 c1dd99d4 Michael Hanselmann
  @param output: Path to file in which to save the output
246 c1dd99d4 Michael Hanselmann
  @type output_fd: int
247 c1dd99d4 Michael Hanselmann
  @param output_fd: File descriptor for output
248 c1dd99d4 Michael Hanselmann
  @type pidfile: string
249 c1dd99d4 Michael Hanselmann
  @param pidfile: Process ID file
250 c1dd99d4 Michael Hanselmann
  @rtype: int
251 c1dd99d4 Michael Hanselmann
  @return: Daemon process ID
252 c1dd99d4 Michael Hanselmann
  @raise errors.ProgrammerError: if we call this when forks are disabled
253 c1dd99d4 Michael Hanselmann

254 c1dd99d4 Michael Hanselmann
255 c1dd99d4 Michael Hanselmann
  if no_fork:
256 c1dd99d4 Michael Hanselmann
    raise errors.ProgrammerError("utils.StartDaemon() called with fork()"
257 c1dd99d4 Michael Hanselmann
                                 " disabled")
258 c1dd99d4 Michael Hanselmann
259 c1dd99d4 Michael Hanselmann
  if output and not (bool(output) ^ (output_fd is not None)):
260 c1dd99d4 Michael Hanselmann
    raise errors.ProgrammerError("Only one of 'output' and 'output_fd' can be"
261 c1dd99d4 Michael Hanselmann
                                 " specified")
262 c1dd99d4 Michael Hanselmann
263 c1dd99d4 Michael Hanselmann
  if isinstance(cmd, basestring):
264 c1dd99d4 Michael Hanselmann
    cmd = ["/bin/sh", "-c", cmd]
265 c1dd99d4 Michael Hanselmann
266 c1dd99d4 Michael Hanselmann
  strcmd = ShellQuoteArgs(cmd)
267 c1dd99d4 Michael Hanselmann
268 c1dd99d4 Michael Hanselmann
  if output:
269 c1dd99d4 Michael Hanselmann
    logging.debug("StartDaemon %s, output file '%s'", strcmd, output)
270 c1dd99d4 Michael Hanselmann
271 c1dd99d4 Michael Hanselmann
    logging.debug("StartDaemon %s", strcmd)
272 c1dd99d4 Michael Hanselmann
273 bb3776b4 Michael Hanselmann
  cmd_env = _BuildCmdEnvironment(env, False)
274 c1dd99d4 Michael Hanselmann
275 c1dd99d4 Michael Hanselmann
  # Create pipe for sending PID back
276 c1dd99d4 Michael Hanselmann
  (pidpipe_read, pidpipe_write) = os.pipe()
277 c1dd99d4 Michael Hanselmann
278 c1dd99d4 Michael Hanselmann
279 c1dd99d4 Michael Hanselmann
      # Create pipe for sending error messages
280 c1dd99d4 Michael Hanselmann
      (errpipe_read, errpipe_write) = os.pipe()
281 c1dd99d4 Michael Hanselmann
282 c1dd99d4 Michael Hanselmann
283 c1dd99d4 Michael Hanselmann
          # First fork
284 c1dd99d4 Michael Hanselmann
          pid = os.fork()
285 c1dd99d4 Michael Hanselmann
          if pid == 0:
286 c1dd99d4 Michael Hanselmann
287 c1dd99d4 Michael Hanselmann
              # Child process, won't return
288 e0bb431e Michael Hanselmann
              _StartDaemonChild(errpipe_read, errpipe_write,
289 e0bb431e Michael Hanselmann
                                pidpipe_read, pidpipe_write,
290 e0bb431e Michael Hanselmann
                                cmd, cmd_env, cwd,
291 e0bb431e Michael Hanselmann
                                output, output_fd, pidfile)
292 c1dd99d4 Michael Hanselmann
293 c1dd99d4 Michael Hanselmann
              # Well, maybe child process failed
294 e0bb431e Michael Hanselmann
              os._exit(1) # pylint: disable-msg=W0212
295 c1dd99d4 Michael Hanselmann
296 c1dd99d4 Michael Hanselmann
297 c1dd99d4 Michael Hanselmann
298 c1dd99d4 Michael Hanselmann
        # Wait for daemon to be started (or an error message to arrive) and read
299 c1dd99d4 Michael Hanselmann
        # up to 100 KB as an error message
300 c1dd99d4 Michael Hanselmann
        errormsg = RetryOnSignal(, errpipe_read, 100 * 1024)
301 c1dd99d4 Michael Hanselmann
302 c1dd99d4 Michael Hanselmann
303 c1dd99d4 Michael Hanselmann
304 c1dd99d4 Michael Hanselmann
305 c1dd99d4 Michael Hanselmann
306 c1dd99d4 Michael Hanselmann
    # Read up to 128 bytes for PID
307 c1dd99d4 Michael Hanselmann
    pidtext = RetryOnSignal(, pidpipe_read, 128)
308 c1dd99d4 Michael Hanselmann
309 c1dd99d4 Michael Hanselmann
310 c1dd99d4 Michael Hanselmann
311 c1dd99d4 Michael Hanselmann
  # Try to avoid zombies by waiting for child process
312 c1dd99d4 Michael Hanselmann
313 c1dd99d4 Michael Hanselmann
    os.waitpid(pid, 0)
314 c1dd99d4 Michael Hanselmann
  except OSError:
315 c1dd99d4 Michael Hanselmann
316 c1dd99d4 Michael Hanselmann
317 c1dd99d4 Michael Hanselmann
  if errormsg:
318 c1dd99d4 Michael Hanselmann
    raise errors.OpExecError("Error when starting daemon process: %r" %
319 c1dd99d4 Michael Hanselmann
320 c1dd99d4 Michael Hanselmann
321 c1dd99d4 Michael Hanselmann
322 c1dd99d4 Michael Hanselmann
    return int(pidtext)
323 c1dd99d4 Michael Hanselmann
  except (ValueError, TypeError), err:
324 c1dd99d4 Michael Hanselmann
    raise errors.OpExecError("Error while trying to parse PID %r: %s" %
325 c1dd99d4 Michael Hanselmann
                             (pidtext, err))
326 c1dd99d4 Michael Hanselmann
327 c1dd99d4 Michael Hanselmann
328 e0bb431e Michael Hanselmann
def _StartDaemonChild(errpipe_read, errpipe_write,
329 e0bb431e Michael Hanselmann
                      pidpipe_read, pidpipe_write,
330 e0bb431e Michael Hanselmann
                      args, env, cwd,
331 e0bb431e Michael Hanselmann
                      output, fd_output, pidfile):
332 c1dd99d4 Michael Hanselmann
  """Child process for starting daemon.
333 c1dd99d4 Michael Hanselmann

334 c1dd99d4 Michael Hanselmann
335 c1dd99d4 Michael Hanselmann
336 c1dd99d4 Michael Hanselmann
    # Close parent's side
337 c1dd99d4 Michael Hanselmann
338 c1dd99d4 Michael Hanselmann
339 c1dd99d4 Michael Hanselmann
340 c1dd99d4 Michael Hanselmann
    # First child process
341 c1dd99d4 Michael Hanselmann
342 c1dd99d4 Michael Hanselmann
343 c1dd99d4 Michael Hanselmann
344 c1dd99d4 Michael Hanselmann
345 c1dd99d4 Michael Hanselmann
    # And fork for the second time
346 c1dd99d4 Michael Hanselmann
    pid = os.fork()
347 c1dd99d4 Michael Hanselmann
    if pid != 0:
348 c1dd99d4 Michael Hanselmann
      # Exit first child process
349 c1dd99d4 Michael Hanselmann
      os._exit(0) # pylint: disable-msg=W0212
350 c1dd99d4 Michael Hanselmann
351 c1dd99d4 Michael Hanselmann
    # Make sure pipe is closed on execv* (and thereby notifies original process)
352 c1dd99d4 Michael Hanselmann
    SetCloseOnExecFlag(errpipe_write, True)
353 c1dd99d4 Michael Hanselmann
354 c1dd99d4 Michael Hanselmann
    # List of file descriptors to be left open
355 c1dd99d4 Michael Hanselmann
    noclose_fds = [errpipe_write]
356 c1dd99d4 Michael Hanselmann
357 c1dd99d4 Michael Hanselmann
    # Open PID file
358 c1dd99d4 Michael Hanselmann
    if pidfile:
359 c1dd99d4 Michael Hanselmann
360 c1dd99d4 Michael Hanselmann
        # TODO: Atomic replace with another locked file instead of writing into
361 c1dd99d4 Michael Hanselmann
        # it after creating
362 c1dd99d4 Michael Hanselmann
        fd_pidfile =, os.O_WRONLY | os.O_CREAT, 0600)
363 c1dd99d4 Michael Hanselmann
364 c1dd99d4 Michael Hanselmann
        # Lock the PID file (and fail if not possible to do so). Any code
365 c1dd99d4 Michael Hanselmann
        # wanting to send a signal to the daemon should try to lock the PID
366 c1dd99d4 Michael Hanselmann
        # file before reading it. If acquiring the lock succeeds, the daemon is
367 c1dd99d4 Michael Hanselmann
        # no longer running and the signal should not be sent.
368 c1dd99d4 Michael Hanselmann
369 c1dd99d4 Michael Hanselmann
370 c1dd99d4 Michael Hanselmann
        os.write(fd_pidfile, "%d\n" % os.getpid())
371 c1dd99d4 Michael Hanselmann
      except Exception, err:
372 c1dd99d4 Michael Hanselmann
        raise Exception("Creating and locking PID file failed: %s" % err)
373 c1dd99d4 Michael Hanselmann
374 c1dd99d4 Michael Hanselmann
      # Keeping the file open to hold the lock
375 c1dd99d4 Michael Hanselmann
376 c1dd99d4 Michael Hanselmann
377 c1dd99d4 Michael Hanselmann
      SetCloseOnExecFlag(fd_pidfile, False)
378 c1dd99d4 Michael Hanselmann
379 c1dd99d4 Michael Hanselmann
      fd_pidfile = None
380 c1dd99d4 Michael Hanselmann
381 c1dd99d4 Michael Hanselmann
    # Open /dev/null
382 c1dd99d4 Michael Hanselmann
    fd_devnull =, os.O_RDWR)
383 c1dd99d4 Michael Hanselmann
384 c1dd99d4 Michael Hanselmann
    assert not output or (bool(output) ^ (fd_output is not None))
385 c1dd99d4 Michael Hanselmann
386 c1dd99d4 Michael Hanselmann
    if fd_output is not None:
387 c1dd99d4 Michael Hanselmann
388 c1dd99d4 Michael Hanselmann
    elif output:
389 c1dd99d4 Michael Hanselmann
      # Open output file
390 c1dd99d4 Michael Hanselmann
391 c1dd99d4 Michael Hanselmann
        # TODO: Implement flag to set append=yes/no
392 c1dd99d4 Michael Hanselmann
        fd_output =, os.O_WRONLY | os.O_CREAT, 0600)
393 c1dd99d4 Michael Hanselmann
      except EnvironmentError, err:
394 c1dd99d4 Michael Hanselmann
        raise Exception("Opening output file failed: %s" % err)
395 c1dd99d4 Michael Hanselmann
396 c1dd99d4 Michael Hanselmann
      fd_output = fd_devnull
397 c1dd99d4 Michael Hanselmann
398 c1dd99d4 Michael Hanselmann
    # Redirect standard I/O
399 c1dd99d4 Michael Hanselmann
    os.dup2(fd_devnull, 0)
400 c1dd99d4 Michael Hanselmann
    os.dup2(fd_output, 1)
401 c1dd99d4 Michael Hanselmann
    os.dup2(fd_output, 2)
402 c1dd99d4 Michael Hanselmann
403 c1dd99d4 Michael Hanselmann
    # Send daemon PID to parent
404 c1dd99d4 Michael Hanselmann
    RetryOnSignal(os.write, pidpipe_write, str(os.getpid()))
405 c1dd99d4 Michael Hanselmann
406 c1dd99d4 Michael Hanselmann
    # Close all file descriptors except stdio and error message pipe
407 c1dd99d4 Michael Hanselmann
408 c1dd99d4 Michael Hanselmann
409 c1dd99d4 Michael Hanselmann
    # Change working directory
410 c1dd99d4 Michael Hanselmann
411 c1dd99d4 Michael Hanselmann
412 c1dd99d4 Michael Hanselmann
    if env is None:
413 c1dd99d4 Michael Hanselmann
      os.execvp(args[0], args)
414 c1dd99d4 Michael Hanselmann
415 c1dd99d4 Michael Hanselmann
      os.execvpe(args[0], args, env)
416 c1dd99d4 Michael Hanselmann
  except: # pylint: disable-msg=W0702
417 c1dd99d4 Michael Hanselmann
418 c1dd99d4 Michael Hanselmann
      # Report errors to original process
419 c1dd99d4 Michael Hanselmann
      buf = str(sys.exc_info()[1])
420 c1dd99d4 Michael Hanselmann
421 c1dd99d4 Michael Hanselmann
      RetryOnSignal(os.write, errpipe_write, buf)
422 c1dd99d4 Michael Hanselmann
    except: # pylint: disable-msg=W0702
423 c1dd99d4 Michael Hanselmann
      # Ignore errors in error handling
424 c1dd99d4 Michael Hanselmann
425 c1dd99d4 Michael Hanselmann
426 c1dd99d4 Michael Hanselmann
  os._exit(1) # pylint: disable-msg=W0212
427 c1dd99d4 Michael Hanselmann
428 c1dd99d4 Michael Hanselmann
429 0963d545 René Nussbaumer
def _RunCmdPipe(cmd, env, via_shell, cwd, interactive):
430 36117c2b Iustin Pop
  """Run a command and return its output.
431 36117c2b Iustin Pop

432 36117c2b Iustin Pop
  @type  cmd: string or list
433 36117c2b Iustin Pop
  @param cmd: Command to run
434 36117c2b Iustin Pop
  @type env: dict
435 36117c2b Iustin Pop
  @param env: The environment to use
436 36117c2b Iustin Pop
  @type via_shell: bool
437 36117c2b Iustin Pop
  @param via_shell: if we should run via the shell
438 8797df43 Iustin Pop
  @type cwd: string
439 8797df43 Iustin Pop
  @param cwd: the working directory for the program
440 0963d545 René Nussbaumer
  @type interactive: boolean
441 0963d545 René Nussbaumer
  @param interactive: Run command interactive (without piping)
442 36117c2b Iustin Pop
  @rtype: tuple
443 36117c2b Iustin Pop
  @return: (out, err, status)
444 36117c2b Iustin Pop

445 36117c2b Iustin Pop
446 9c233417 Iustin Pop
  poller = select.poll()
447 0963d545 René Nussbaumer
448 0963d545 René Nussbaumer
  stderr = subprocess.PIPE
449 0963d545 René Nussbaumer
  stdout = subprocess.PIPE
450 0963d545 René Nussbaumer
  stdin = subprocess.PIPE
451 0963d545 René Nussbaumer
452 0963d545 René Nussbaumer
  if interactive:
453 0963d545 René Nussbaumer
    stderr = stdout = stdin = None
454 0963d545 René Nussbaumer
455 36117c2b Iustin Pop
  child = subprocess.Popen(cmd, shell=via_shell,
456 0963d545 René Nussbaumer
457 0963d545 René Nussbaumer
458 0963d545 René Nussbaumer
459 8797df43 Iustin Pop
                           close_fds=True, env=env,
460 8797df43 Iustin Pop
461 113b55aa Iustin Pop
462 9c233417 Iustin Pop
  out = StringIO()
463 9c233417 Iustin Pop
  err = StringIO()
464 0963d545 René Nussbaumer
  if not interactive:
465 0963d545 René Nussbaumer
466 0963d545 René Nussbaumer
    poller.register(child.stdout, select.POLLIN)
467 0963d545 René Nussbaumer
    poller.register(child.stderr, select.POLLIN)
468 0963d545 René Nussbaumer
    fdmap = {
469 0963d545 René Nussbaumer
      child.stdout.fileno(): (out, child.stdout),
470 0963d545 René Nussbaumer
      child.stderr.fileno(): (err, child.stderr),
471 0963d545 René Nussbaumer
472 0963d545 René Nussbaumer
    for fd in fdmap:
473 0963d545 René Nussbaumer
      SetNonblockFlag(fd, True)
474 0963d545 René Nussbaumer
475 0963d545 René Nussbaumer
    while fdmap:
476 0963d545 René Nussbaumer
      pollresult = RetryOnSignal(poller.poll)
477 0963d545 René Nussbaumer
478 0963d545 René Nussbaumer
      for fd, event in pollresult:
479 0963d545 René Nussbaumer
        if event & select.POLLIN or event & select.POLLPRI:
480 0963d545 René Nussbaumer
          data = fdmap[fd][1].read()
481 0963d545 René Nussbaumer
          # no data from read signifies EOF (the same as POLLHUP)
482 0963d545 René Nussbaumer
          if not data:
483 0963d545 René Nussbaumer
484 0963d545 René Nussbaumer
            del fdmap[fd]
485 0963d545 René Nussbaumer
486 0963d545 René Nussbaumer
487 0963d545 René Nussbaumer
        if (event & select.POLLNVAL or event & select.POLLHUP or
488 0963d545 René Nussbaumer
            event & select.POLLERR):
489 9c233417 Iustin Pop
490 9c233417 Iustin Pop
          del fdmap[fd]
491 9c233417 Iustin Pop
492 9c233417 Iustin Pop
  out = out.getvalue()
493 9c233417 Iustin Pop
  err = err.getvalue()
494 a8083063 Iustin Pop
495 a8083063 Iustin Pop
  status = child.wait()
496 36117c2b Iustin Pop
  return out, err, status
497 a8083063 Iustin Pop
498 36117c2b Iustin Pop
499 8797df43 Iustin Pop
def _RunCmdFile(cmd, env, via_shell, output, cwd):
500 36117c2b Iustin Pop
  """Run a command and save its output to a file.
501 36117c2b Iustin Pop

502 36117c2b Iustin Pop
  @type  cmd: string or list
503 36117c2b Iustin Pop
  @param cmd: Command to run
504 36117c2b Iustin Pop
  @type env: dict
505 36117c2b Iustin Pop
  @param env: The environment to use
506 36117c2b Iustin Pop
  @type via_shell: bool
507 36117c2b Iustin Pop
  @param via_shell: if we should run via the shell
508 36117c2b Iustin Pop
  @type output: str
509 36117c2b Iustin Pop
  @param output: the filename in which to save the output
510 8797df43 Iustin Pop
  @type cwd: string
511 8797df43 Iustin Pop
  @param cwd: the working directory for the program
512 36117c2b Iustin Pop
  @rtype: int
513 36117c2b Iustin Pop
  @return: the exit status
514 36117c2b Iustin Pop

515 36117c2b Iustin Pop
516 36117c2b Iustin Pop
  fh = open(output, "a")
517 36117c2b Iustin Pop
518 36117c2b Iustin Pop
    child = subprocess.Popen(cmd, shell=via_shell,
519 36117c2b Iustin Pop
520 36117c2b Iustin Pop
521 36117c2b Iustin Pop
522 8797df43 Iustin Pop
                             close_fds=True, env=env,
523 8797df43 Iustin Pop
524 36117c2b Iustin Pop
525 36117c2b Iustin Pop
526 36117c2b Iustin Pop
    status = child.wait()
527 36117c2b Iustin Pop
528 36117c2b Iustin Pop
529 36117c2b Iustin Pop
  return status
530 a8083063 Iustin Pop
531 a8083063 Iustin Pop
532 73027ed2 Michael Hanselmann
def SetCloseOnExecFlag(fd, enable):
533 73027ed2 Michael Hanselmann
  """Sets or unsets the close-on-exec flag on a file descriptor.
534 73027ed2 Michael Hanselmann

535 73027ed2 Michael Hanselmann
  @type fd: int
536 73027ed2 Michael Hanselmann
  @param fd: File descriptor
537 73027ed2 Michael Hanselmann
  @type enable: bool
538 73027ed2 Michael Hanselmann
  @param enable: Whether to set or unset it.
539 73027ed2 Michael Hanselmann

540 73027ed2 Michael Hanselmann
541 73027ed2 Michael Hanselmann
  flags = fcntl.fcntl(fd, fcntl.F_GETFD)
542 73027ed2 Michael Hanselmann
543 73027ed2 Michael Hanselmann
  if enable:
544 73027ed2 Michael Hanselmann
    flags |= fcntl.FD_CLOEXEC
545 73027ed2 Michael Hanselmann
546 73027ed2 Michael Hanselmann
    flags &= ~fcntl.FD_CLOEXEC
547 73027ed2 Michael Hanselmann
548 73027ed2 Michael Hanselmann
  fcntl.fcntl(fd, fcntl.F_SETFD, flags)
549 73027ed2 Michael Hanselmann
550 73027ed2 Michael Hanselmann
551 287a1740 Michael Hanselmann
def SetNonblockFlag(fd, enable):
552 287a1740 Michael Hanselmann
  """Sets or unsets the O_NONBLOCK flag on on a file descriptor.
553 287a1740 Michael Hanselmann

554 287a1740 Michael Hanselmann
  @type fd: int
555 287a1740 Michael Hanselmann
  @param fd: File descriptor
556 287a1740 Michael Hanselmann
  @type enable: bool
557 287a1740 Michael Hanselmann
  @param enable: Whether to set or unset it
558 287a1740 Michael Hanselmann

559 287a1740 Michael Hanselmann
560 287a1740 Michael Hanselmann
  flags = fcntl.fcntl(fd, fcntl.F_GETFL)
561 287a1740 Michael Hanselmann
562 287a1740 Michael Hanselmann
  if enable:
563 287a1740 Michael Hanselmann
    flags |= os.O_NONBLOCK
564 287a1740 Michael Hanselmann
565 287a1740 Michael Hanselmann
    flags &= ~os.O_NONBLOCK
566 287a1740 Michael Hanselmann
567 287a1740 Michael Hanselmann
  fcntl.fcntl(fd, fcntl.F_SETFL, flags)
568 287a1740 Michael Hanselmann
569 287a1740 Michael Hanselmann
570 edcb5d9e Michael Hanselmann
def RetryOnSignal(fn, *args, **kwargs):
571 edcb5d9e Michael Hanselmann
  """Calls a function again if it failed due to EINTR.
572 edcb5d9e Michael Hanselmann

573 edcb5d9e Michael Hanselmann
574 edcb5d9e Michael Hanselmann
  while True:
575 edcb5d9e Michael Hanselmann
576 edcb5d9e Michael Hanselmann
      return fn(*args, **kwargs)
577 965d0e5b Guido Trotter
    except EnvironmentError, err:
578 edcb5d9e Michael Hanselmann
      if err.errno != errno.EINTR:
579 edcb5d9e Michael Hanselmann
580 965d0e5b Guido Trotter
    except (socket.error, select.error), err:
581 965d0e5b Guido Trotter
      # In python 2.6 and above select.error is an IOError, so it's handled
582 965d0e5b Guido Trotter
      # above, in 2.5 and below it's not, and it's handled here.
583 edcb5d9e Michael Hanselmann
      if not (err.args and err.args[0] == errno.EINTR):
584 edcb5d9e Michael Hanselmann
585 edcb5d9e Michael Hanselmann
586 edcb5d9e Michael Hanselmann
587 6bb65e3a Guido Trotter
def RunParts(dir_name, env=None, reset_env=False):
588 6bb65e3a Guido Trotter
  """Run Scripts or programs in a directory
589 6bb65e3a Guido Trotter

590 6bb65e3a Guido Trotter
  @type dir_name: string
591 6bb65e3a Guido Trotter
  @param dir_name: absolute path to a directory
592 6bb65e3a Guido Trotter
  @type env: dict
593 6bb65e3a Guido Trotter
  @param env: The environment to use
594 6bb65e3a Guido Trotter
  @type reset_env: boolean
595 6bb65e3a Guido Trotter
  @param reset_env: whether to reset or keep the default os environment
596 6bb65e3a Guido Trotter
  @rtype: list of tuples
597 6bb65e3a Guido Trotter
  @return: list of (name, (one of RUNDIR_STATUS), RunResult)
598 6bb65e3a Guido Trotter

599 6bb65e3a Guido Trotter
600 6bb65e3a Guido Trotter
  rr = []
601 6bb65e3a Guido Trotter
602 6bb65e3a Guido Trotter
603 6bb65e3a Guido Trotter
    dir_contents = ListVisibleFiles(dir_name)
604 6bb65e3a Guido Trotter
  except OSError, err:
605 6bb65e3a Guido Trotter
    logging.warning("RunParts: skipping %s (cannot list: %s)", dir_name, err)
606 6bb65e3a Guido Trotter
    return rr
607 6bb65e3a Guido Trotter
608 6bb65e3a Guido Trotter
  for relname in sorted(dir_contents):
609 c4feafe8 Iustin Pop
    fname = PathJoin(dir_name, relname)
610 6bb65e3a Guido Trotter
    if not (os.path.isfile(fname) and os.access(fname, os.X_OK) and
611 6bb65e3a Guido Trotter
            constants.EXT_PLUGIN_MASK.match(relname) is not None):
612 6bb65e3a Guido Trotter
      rr.append((relname, constants.RUNPARTS_SKIP, None))
613 6bb65e3a Guido Trotter
614 6bb65e3a Guido Trotter
615 6bb65e3a Guido Trotter
        result = RunCmd([fname], env=env, reset_env=reset_env)
616 6bb65e3a Guido Trotter
      except Exception, err: # pylint: disable-msg=W0703
617 6bb65e3a Guido Trotter
        rr.append((relname, constants.RUNPARTS_ERR, str(err)))
618 6bb65e3a Guido Trotter
619 6bb65e3a Guido Trotter
        rr.append((relname, constants.RUNPARTS_RUN, result))
620 6bb65e3a Guido Trotter
621 6bb65e3a Guido Trotter
  return rr
622 6bb65e3a Guido Trotter
623 6bb65e3a Guido Trotter
624 a8083063 Iustin Pop
def RemoveFile(filename):
625 a8083063 Iustin Pop
  """Remove a file ignoring some errors.
626 a8083063 Iustin Pop

627 a8083063 Iustin Pop
  Remove a file, ignoring non-existing ones or directories. Other
628 a8083063 Iustin Pop
  errors are passed.
629 a8083063 Iustin Pop

630 58885d79 Iustin Pop
  @type filename: str
631 58885d79 Iustin Pop
  @param filename: the file to be removed
632 58885d79 Iustin Pop

633 a8083063 Iustin Pop
634 a8083063 Iustin Pop
635 a8083063 Iustin Pop
636 a8083063 Iustin Pop
  except OSError, err:
637 4ca1b175 Alexander Schreiber
    if err.errno not in (errno.ENOENT, errno.EISDIR):
638 a8083063 Iustin Pop
639 a8083063 Iustin Pop
640 72087dcd Balazs Lecz
641 72087dcd Balazs Lecz
def RemoveDir(dirname):
642 72087dcd Balazs Lecz
  """Remove an empty directory.
643 72087dcd Balazs Lecz

644 72087dcd Balazs Lecz
  Remove a directory, ignoring non-existing ones.
645 72087dcd Balazs Lecz
  Other errors are passed. This includes the case,
646 72087dcd Balazs Lecz
  where the directory is not empty, so it can't be removed.
647 72087dcd Balazs Lecz

648 72087dcd Balazs Lecz
  @type dirname: str
649 72087dcd Balazs Lecz
  @param dirname: the empty directory to be removed
650 72087dcd Balazs Lecz

651 72087dcd Balazs Lecz
652 72087dcd Balazs Lecz
653 72087dcd Balazs Lecz
654 72087dcd Balazs Lecz
  except OSError, err:
655 72087dcd Balazs Lecz
    if err.errno != errno.ENOENT:
656 72087dcd Balazs Lecz
657 72087dcd Balazs Lecz
658 a8083063 Iustin Pop
659 6e797216 Michael Hanselmann
def RenameFile(old, new, mkdir=False, mkdir_mode=0750):
660 6e797216 Michael Hanselmann
  """Renames a file.
661 6e797216 Michael Hanselmann

662 6e797216 Michael Hanselmann
  @type old: string
663 6e797216 Michael Hanselmann
  @param old: Original path
664 6e797216 Michael Hanselmann
  @type new: string
665 6e797216 Michael Hanselmann
  @param new: New path
666 6e797216 Michael Hanselmann
  @type mkdir: bool
667 6e797216 Michael Hanselmann
  @param mkdir: Whether to create target directory if it doesn't exist
668 6e797216 Michael Hanselmann
  @type mkdir_mode: int
669 6e797216 Michael Hanselmann
  @param mkdir_mode: Mode for newly created directories
670 6e797216 Michael Hanselmann

671 6e797216 Michael Hanselmann
672 6e797216 Michael Hanselmann
673 6e797216 Michael Hanselmann
    return os.rename(old, new)
674 6e797216 Michael Hanselmann
  except OSError, err:
675 6e797216 Michael Hanselmann
    # In at least one use case of this function, the job queue, directory
676 6e797216 Michael Hanselmann
    # creation is very rare. Checking for the directory before renaming is not
677 6e797216 Michael Hanselmann
    # as efficient.
678 6e797216 Michael Hanselmann
    if mkdir and err.errno == errno.ENOENT:
679 6e797216 Michael Hanselmann
      # Create directory and try again
680 cc2f004d Michael Hanselmann
      Makedirs(os.path.dirname(new), mode=mkdir_mode)
681 a426508d Michael Hanselmann
682 6e797216 Michael Hanselmann
      return os.rename(old, new)
683 a426508d Michael Hanselmann
684 6e797216 Michael Hanselmann
685 6e797216 Michael Hanselmann
686 6e797216 Michael Hanselmann
687 76e5f8b5 Michael Hanselmann
def Makedirs(path, mode=0750):
688 76e5f8b5 Michael Hanselmann
  """Super-mkdir; create a leaf directory and all intermediate ones.
689 76e5f8b5 Michael Hanselmann

690 76e5f8b5 Michael Hanselmann
  This is a wrapper around C{os.makedirs} adding error handling not implemented
691 76e5f8b5 Michael Hanselmann
  before Python 2.5.
692 76e5f8b5 Michael Hanselmann

693 76e5f8b5 Michael Hanselmann
694 76e5f8b5 Michael Hanselmann
695 76e5f8b5 Michael Hanselmann
    os.makedirs(path, mode)
696 76e5f8b5 Michael Hanselmann
  except OSError, err:
697 76e5f8b5 Michael Hanselmann
    # Ignore EEXIST. This is only handled in os.makedirs as included in
698 76e5f8b5 Michael Hanselmann
    # Python 2.5 and above.
699 76e5f8b5 Michael Hanselmann
    if err.errno != errno.EEXIST or not os.path.exists(path):
700 76e5f8b5 Michael Hanselmann
701 76e5f8b5 Michael Hanselmann
702 76e5f8b5 Michael Hanselmann
703 055f822b Michael Hanselmann
def ResetTempfileModule():
704 055f822b Michael Hanselmann
  """Resets the random name generator of the tempfile module.
705 055f822b Michael Hanselmann

706 055f822b Michael Hanselmann
  This function should be called after C{os.fork} in the child process to
707 055f822b Michael Hanselmann
  ensure it creates a newly seeded random generator. Otherwise it would
708 055f822b Michael Hanselmann
  generate the same random parts as the parent process. If several processes
709 055f822b Michael Hanselmann
  race for the creation of a temporary file, this could lead to one not getting
710 055f822b Michael Hanselmann
  a temporary name.
711 055f822b Michael Hanselmann

712 055f822b Michael Hanselmann
713 055f822b Michael Hanselmann
  # pylint: disable-msg=W0212
714 055f822b Michael Hanselmann
  if hasattr(tempfile, "_once_lock") and hasattr(tempfile, "_name_sequence"):
715 055f822b Michael Hanselmann
716 055f822b Michael Hanselmann
717 055f822b Michael Hanselmann
      # Reset random name generator
718 055f822b Michael Hanselmann
      tempfile._name_sequence = None
719 055f822b Michael Hanselmann
720 055f822b Michael Hanselmann
721 055f822b Michael Hanselmann
722 055f822b Michael Hanselmann
    logging.critical("The tempfile module misses at least one of the"
723 055f822b Michael Hanselmann
                     " '_once_lock' and '_name_sequence' attributes")
724 055f822b Michael Hanselmann
725 055f822b Michael Hanselmann
726 a8083063 Iustin Pop
def _FingerprintFile(filename):
727 a8083063 Iustin Pop
  """Compute the fingerprint of a file.
728 a8083063 Iustin Pop

729 a8083063 Iustin Pop
  If the file does not exist, a None will be returned
730 a8083063 Iustin Pop
731 a8083063 Iustin Pop

732 58885d79 Iustin Pop
  @type filename: str
733 58885d79 Iustin Pop
  @param filename: the filename to checksum
734 58885d79 Iustin Pop
  @rtype: str
735 58885d79 Iustin Pop
  @return: the hex digest of the sha checksum of the contents
736 58885d79 Iustin Pop
      of the file
737 a8083063 Iustin Pop

738 a8083063 Iustin Pop
739 a8083063 Iustin Pop
  if not (os.path.exists(filename) and os.path.isfile(filename)):
740 a8083063 Iustin Pop
    return None
741 a8083063 Iustin Pop
742 a8083063 Iustin Pop
  f = open(filename)
743 a8083063 Iustin Pop
744 716a32cb Guido Trotter
  fp = compat.sha1_hash()
745 a8083063 Iustin Pop
  while True:
746 a8083063 Iustin Pop
    data =
747 a8083063 Iustin Pop
    if not data:
748 a8083063 Iustin Pop
749 a8083063 Iustin Pop
750 a8083063 Iustin Pop
751 a8083063 Iustin Pop
752 a8083063 Iustin Pop
  return fp.hexdigest()
753 a8083063 Iustin Pop
754 a8083063 Iustin Pop
755 a8083063 Iustin Pop
def FingerprintFiles(files):
756 a8083063 Iustin Pop
  """Compute fingerprints for a list of files.
757 a8083063 Iustin Pop

758 58885d79 Iustin Pop
  @type files: list
759 58885d79 Iustin Pop
  @param files: the list of filename to fingerprint
760 58885d79 Iustin Pop
  @rtype: dict
761 58885d79 Iustin Pop
  @return: a dictionary filename: fingerprint, holding only
762 58885d79 Iustin Pop
      existing files
763 a8083063 Iustin Pop

764 a8083063 Iustin Pop
765 a8083063 Iustin Pop
  ret = {}
766 a8083063 Iustin Pop
767 a8083063 Iustin Pop
  for filename in files:
768 a8083063 Iustin Pop
    cksum = _FingerprintFile(filename)
769 a8083063 Iustin Pop
    if cksum:
770 a8083063 Iustin Pop
      ret[filename] = cksum
771 a8083063 Iustin Pop
772 a8083063 Iustin Pop
  return ret
773 a8083063 Iustin Pop
774 a8083063 Iustin Pop
775 a5728081 Guido Trotter
def ForceDictType(target, key_types, allowed_values=None):
776 a5728081 Guido Trotter
  """Force the values of a dict to have certain types.
777 a5728081 Guido Trotter

778 a5728081 Guido Trotter
  @type target: dict
779 a5728081 Guido Trotter
  @param target: the dict to update
780 a5728081 Guido Trotter
  @type key_types: dict
781 a5728081 Guido Trotter
  @param key_types: dict mapping target dict keys to types
782 a5728081 Guido Trotter
                    in constants.ENFORCEABLE_TYPES
783 a5728081 Guido Trotter
  @type allowed_values: list
784 a5728081 Guido Trotter
  @keyword allowed_values: list of specially allowed values
785 a5728081 Guido Trotter

786 a5728081 Guido Trotter
787 a5728081 Guido Trotter
  if allowed_values is None:
788 a5728081 Guido Trotter
    allowed_values = []
789 a5728081 Guido Trotter
790 8b46606c Guido Trotter
  if not isinstance(target, dict):
791 8b46606c Guido Trotter
    msg = "Expected dictionary, got '%s'" % target
792 8b46606c Guido Trotter
    raise errors.TypeEnforcementError(msg)
793 8b46606c Guido Trotter
794 a5728081 Guido Trotter
  for key in target:
795 a5728081 Guido Trotter
    if key not in key_types:
796 a5728081 Guido Trotter
      msg = "Unknown key '%s'" % key
797 a5728081 Guido Trotter
      raise errors.TypeEnforcementError(msg)
798 a5728081 Guido Trotter
799 a5728081 Guido Trotter
    if target[key] in allowed_values:
800 a5728081 Guido Trotter
801 a5728081 Guido Trotter
802 29921401 Iustin Pop
    ktype = key_types[key]
803 29921401 Iustin Pop
    if ktype not in constants.ENFORCEABLE_TYPES:
804 29921401 Iustin Pop
      msg = "'%s' has non-enforceable type %s" % (key, ktype)
805 a5728081 Guido Trotter
      raise errors.ProgrammerError(msg)
806 a5728081 Guido Trotter
807 59525e1f Michael Hanselmann
    if ktype in (constants.VTYPE_STRING, constants.VTYPE_MAYBE_STRING):
808 59525e1f Michael Hanselmann
      if target[key] is None and ktype == constants.VTYPE_MAYBE_STRING:
809 59525e1f Michael Hanselmann
810 59525e1f Michael Hanselmann
      elif not isinstance(target[key], basestring):
811 a5728081 Guido Trotter
        if isinstance(target[key], bool) and not target[key]:
812 a5728081 Guido Trotter
          target[key] = ''
813 a5728081 Guido Trotter
814 a5728081 Guido Trotter
          msg = "'%s' (value %s) is not a valid string" % (key, target[key])
815 a5728081 Guido Trotter
          raise errors.TypeEnforcementError(msg)
816 29921401 Iustin Pop
    elif ktype == constants.VTYPE_BOOL:
817 a5728081 Guido Trotter
      if isinstance(target[key], basestring) and target[key]:
818 a5728081 Guido Trotter
        if target[key].lower() == constants.VALUE_FALSE:
819 a5728081 Guido Trotter
          target[key] = False
820 a5728081 Guido Trotter
        elif target[key].lower() == constants.VALUE_TRUE:
821 a5728081 Guido Trotter
          target[key] = True
822 a5728081 Guido Trotter
823 a5728081 Guido Trotter
          msg = "'%s' (value %s) is not a valid boolean" % (key, target[key])
824 a5728081 Guido Trotter
          raise errors.TypeEnforcementError(msg)
825 a5728081 Guido Trotter
      elif target[key]:
826 a5728081 Guido Trotter
        target[key] = True
827 a5728081 Guido Trotter
828 a5728081 Guido Trotter
        target[key] = False
829 29921401 Iustin Pop
    elif ktype == constants.VTYPE_SIZE:
830 a5728081 Guido Trotter
831 a5728081 Guido Trotter
        target[key] = ParseUnit(target[key])
832 a5728081 Guido Trotter
      except errors.UnitParseError, err:
833 a5728081 Guido Trotter
        msg = "'%s' (value %s) is not a valid size. error: %s" % \
834 a5728081 Guido Trotter
              (key, target[key], err)
835 a5728081 Guido Trotter
        raise errors.TypeEnforcementError(msg)
836 29921401 Iustin Pop
    elif ktype == constants.VTYPE_INT:
837 a5728081 Guido Trotter
838 a5728081 Guido Trotter
        target[key] = int(target[key])
839 a5728081 Guido Trotter
      except (ValueError, TypeError):
840 a5728081 Guido Trotter
        msg = "'%s' (value %s) is not a valid integer" % (key, target[key])
841 a5728081 Guido Trotter
        raise errors.TypeEnforcementError(msg)
842 a5728081 Guido Trotter
843 a5728081 Guido Trotter
844 a01b500b Michael Hanselmann
def _GetProcStatusPath(pid):
845 a01b500b Michael Hanselmann
  """Returns the path for a PID's proc status file.
846 a01b500b Michael Hanselmann

847 a01b500b Michael Hanselmann
  @type pid: int
848 a01b500b Michael Hanselmann
  @param pid: Process ID
849 a01b500b Michael Hanselmann
  @rtype: string
850 a01b500b Michael Hanselmann

851 a01b500b Michael Hanselmann
852 a01b500b Michael Hanselmann
  return "/proc/%d/status" % pid
853 a01b500b Michael Hanselmann
854 a01b500b Michael Hanselmann
855 a8083063 Iustin Pop
def IsProcessAlive(pid):
856 a8083063 Iustin Pop
  """Check if a given pid exists on the system.
857 a8083063 Iustin Pop

858 44bf25ff Iustin Pop
  @note: zombie status is not handled, so zombie processes
859 44bf25ff Iustin Pop
      will be returned as alive
860 58885d79 Iustin Pop
  @type pid: int
861 58885d79 Iustin Pop
  @param pid: the process ID to check
862 58885d79 Iustin Pop
  @rtype: boolean
863 58885d79 Iustin Pop
  @return: True if the process exists
864 a8083063 Iustin Pop

865 a8083063 Iustin Pop
866 5ef5ea45 Guido Trotter
  def _TryStat(name):
867 5ef5ea45 Guido Trotter
868 5ef5ea45 Guido Trotter
869 5ef5ea45 Guido Trotter
      return True
870 5ef5ea45 Guido Trotter
    except EnvironmentError, err:
871 5ef5ea45 Guido Trotter
      if err.errno in (errno.ENOENT, errno.ENOTDIR):
872 5ef5ea45 Guido Trotter
        return False
873 5ef5ea45 Guido Trotter
      elif err.errno == errno.EINVAL:
874 5ef5ea45 Guido Trotter
        raise RetryAgain(err)
875 5ef5ea45 Guido Trotter
876 5ef5ea45 Guido Trotter
877 5ef5ea45 Guido Trotter
  assert isinstance(pid, int), "pid must be an integer"
878 d9f311d7 Iustin Pop
  if pid <= 0:
879 d9f311d7 Iustin Pop
    return False
880 d9f311d7 Iustin Pop
881 5ef5ea45 Guido Trotter
  # /proc in a multiprocessor environment can have strange behaviors.
882 5ef5ea45 Guido Trotter
  # Retry the os.stat a few times until we get a good result.
883 a8083063 Iustin Pop
884 a01b500b Michael Hanselmann
    return Retry(_TryStat, (0.01, 1.5, 0.1), 0.5,
885 a01b500b Michael Hanselmann
886 5ef5ea45 Guido Trotter
  except RetryTimeout, err:
887 5ef5ea45 Guido Trotter
888 a8083063 Iustin Pop
889 a8083063 Iustin Pop
890 a01b500b Michael Hanselmann
def _ParseSigsetT(sigset):
891 a01b500b Michael Hanselmann
  """Parse a rendered sigset_t value.
892 a01b500b Michael Hanselmann

893 a01b500b Michael Hanselmann
  This is the opposite of the Linux kernel's fs/proc/array.c:render_sigset_t
894 a01b500b Michael Hanselmann
895 a01b500b Michael Hanselmann

896 a01b500b Michael Hanselmann
  @type sigset: string
897 a01b500b Michael Hanselmann
  @param sigset: Rendered signal set from /proc/$pid/status
898 a01b500b Michael Hanselmann
  @rtype: set
899 a01b500b Michael Hanselmann
  @return: Set of all enabled signal numbers
900 a01b500b Michael Hanselmann

901 a01b500b Michael Hanselmann
902 a01b500b Michael Hanselmann
  result = set()
903 a01b500b Michael Hanselmann
904 a01b500b Michael Hanselmann
  signum = 0
905 a01b500b Michael Hanselmann
  for ch in reversed(sigset):
906 a01b500b Michael Hanselmann
    chv = int(ch, 16)
907 a01b500b Michael Hanselmann
908 a01b500b Michael Hanselmann
    # The following could be done in a loop, but it's easier to read and
909 a01b500b Michael Hanselmann
    # understand in the unrolled form
910 a01b500b Michael Hanselmann
    if chv & 1:
911 a01b500b Michael Hanselmann
      result.add(signum + 1)
912 a01b500b Michael Hanselmann
    if chv & 2:
913 a01b500b Michael Hanselmann
      result.add(signum + 2)
914 a01b500b Michael Hanselmann
    if chv & 4:
915 a01b500b Michael Hanselmann
      result.add(signum + 3)
916 a01b500b Michael Hanselmann
    if chv & 8:
917 a01b500b Michael Hanselmann
      result.add(signum + 4)
918 a01b500b Michael Hanselmann
919 a01b500b Michael Hanselmann
    signum += 4
920 a01b500b Michael Hanselmann
921 a01b500b Michael Hanselmann
  return result
922 a01b500b Michael Hanselmann
923 a01b500b Michael Hanselmann
924 a01b500b Michael Hanselmann
def _GetProcStatusField(pstatus, field):
925 a01b500b Michael Hanselmann
  """Retrieves a field from the contents of a proc status file.
926 a01b500b Michael Hanselmann

927 a01b500b Michael Hanselmann
  @type pstatus: string
928 a01b500b Michael Hanselmann
  @param pstatus: Contents of /proc/$pid/status
929 a01b500b Michael Hanselmann
  @type field: string
930 a01b500b Michael Hanselmann
  @param field: Name of field whose value should be returned
931 a01b500b Michael Hanselmann
  @rtype: string
932 a01b500b Michael Hanselmann

933 a01b500b Michael Hanselmann
934 a01b500b Michael Hanselmann
  for line in pstatus.splitlines():
935 a01b500b Michael Hanselmann
    parts = line.split(":", 1)
936 a01b500b Michael Hanselmann
937 a01b500b Michael Hanselmann
    if len(parts) < 2 or parts[0] != field:
938 a01b500b Michael Hanselmann
939 a01b500b Michael Hanselmann
940 a01b500b Michael Hanselmann
    return parts[1].strip()
941 a01b500b Michael Hanselmann
942 a01b500b Michael Hanselmann
  return None
943 a01b500b Michael Hanselmann
944 a01b500b Michael Hanselmann
945 a01b500b Michael Hanselmann
def IsProcessHandlingSignal(pid, signum, status_path=None):
946 a01b500b Michael Hanselmann
  """Checks whether a process is handling a signal.
947 a01b500b Michael Hanselmann

948 a01b500b Michael Hanselmann
  @type pid: int
949 a01b500b Michael Hanselmann
  @param pid: Process ID
950 a01b500b Michael Hanselmann
  @type signum: int
951 a01b500b Michael Hanselmann
  @param signum: Signal number
952 a01b500b Michael Hanselmann
  @rtype: bool
953 a01b500b Michael Hanselmann

954 a01b500b Michael Hanselmann
955 a01b500b Michael Hanselmann
  if status_path is None:
956 a01b500b Michael Hanselmann
    status_path = _GetProcStatusPath(pid)
957 a01b500b Michael Hanselmann
958 a01b500b Michael Hanselmann
959 a01b500b Michael Hanselmann
    proc_status = ReadFile(status_path)
960 a01b500b Michael Hanselmann
  except EnvironmentError, err:
961 a01b500b Michael Hanselmann
    # In at least one case, reading /proc/$pid/status failed with ESRCH.
962 a01b500b Michael Hanselmann
    if err.errno in (errno.ENOENT, errno.ENOTDIR, errno.EINVAL, errno.ESRCH):
963 a01b500b Michael Hanselmann
      return False
964 a01b500b Michael Hanselmann
965 a01b500b Michael Hanselmann
966 a01b500b Michael Hanselmann
  sigcgt = _GetProcStatusField(proc_status, "SigCgt")
967 a01b500b Michael Hanselmann
  if sigcgt is None:
968 a01b500b Michael Hanselmann
    raise RuntimeError("%s is missing 'SigCgt' field" % status_path)
969 a01b500b Michael Hanselmann
970 a01b500b Michael Hanselmann
  # Now check whether signal is handled
971 a01b500b Michael Hanselmann
  return signum in _ParseSigsetT(sigcgt)
972 a01b500b Michael Hanselmann
973 a01b500b Michael Hanselmann
974 d9f311d7 Iustin Pop
def ReadPidFile(pidfile):
975 58885d79 Iustin Pop
  """Read a pid from a file.
976 fee80e90 Guido Trotter

977 58885d79 Iustin Pop
  @type  pidfile: string
978 58885d79 Iustin Pop
  @param pidfile: path to the file containing the pid
979 58885d79 Iustin Pop
  @rtype: int
980 1de62f37 Guido Trotter
  @return: The process id, if the file exists and contains a valid PID,
981 d9f311d7 Iustin Pop
           otherwise 0
982 fee80e90 Guido Trotter

983 fee80e90 Guido Trotter
984 fee80e90 Guido Trotter
985 682f7601 Guido Trotter
    raw_data = ReadOneLineFile(pidfile)
986 d9f311d7 Iustin Pop
  except EnvironmentError, err:
987 d9f311d7 Iustin Pop
    if err.errno != errno.ENOENT:
988 13998ef2 Michael Hanselmann
      logging.exception("Can't read pid file")
989 d9f311d7 Iustin Pop
    return 0
990 fee80e90 Guido Trotter
991 fee80e90 Guido Trotter
992 13998ef2 Michael Hanselmann
    pid = int(raw_data)
993 691744c4 Iustin Pop
  except (TypeError, ValueError), err:
994 8161a646 Iustin Pop"Can't parse pid file contents", exc_info=True)
995 d9f311d7 Iustin Pop
    return 0
996 fee80e90 Guido Trotter
997 d9f311d7 Iustin Pop
  return pid
998 fee80e90 Guido Trotter
999 fee80e90 Guido Trotter
1000 debed9ae Michael Hanselmann
def ReadLockedPidFile(path):
1001 debed9ae Michael Hanselmann
  """Reads a locked PID file.
1002 debed9ae Michael Hanselmann

1003 debed9ae Michael Hanselmann
  This can be used together with L{StartDaemon}.
1004 debed9ae Michael Hanselmann

1005 debed9ae Michael Hanselmann
  @type path: string
1006 debed9ae Michael Hanselmann
  @param path: Path to PID file
1007 debed9ae Michael Hanselmann
  @return: PID as integer or, if file was unlocked or couldn't be opened, None
1008 debed9ae Michael Hanselmann

1009 debed9ae Michael Hanselmann
1010 debed9ae Michael Hanselmann
1011 debed9ae Michael Hanselmann
    fd =, os.O_RDONLY)
1012 debed9ae Michael Hanselmann
  except EnvironmentError, err:
1013 debed9ae Michael Hanselmann
    if err.errno == errno.ENOENT:
1014 debed9ae Michael Hanselmann
      # PID file doesn't exist
1015 debed9ae Michael Hanselmann
      return None
1016 debed9ae Michael Hanselmann
1017 debed9ae Michael Hanselmann
1018 debed9ae Michael Hanselmann
1019 debed9ae Michael Hanselmann
1020 debed9ae Michael Hanselmann
      # Try to acquire lock
1021 debed9ae Michael Hanselmann
1022 debed9ae Michael Hanselmann
    except errors.LockError:
1023 debed9ae Michael Hanselmann
      # Couldn't lock, daemon is running
1024 debed9ae Michael Hanselmann
      return int(, 100))
1025 debed9ae Michael Hanselmann
1026 debed9ae Michael Hanselmann
1027 debed9ae Michael Hanselmann
1028 debed9ae Michael Hanselmann
  return None
1029 debed9ae Michael Hanselmann
1030 debed9ae Michael Hanselmann
1031 256eb94b Guido Trotter
def MatchNameComponent(key, name_list, case_sensitive=True):
1032 a8083063 Iustin Pop
  """Try to match a name against a list.
1033 a8083063 Iustin Pop

1034 a8083063 Iustin Pop
  This function will try to match a name like test1 against a list
1035 58885d79 Iustin Pop
  like C{['', '', ...]}. Against
1036 58885d79 Iustin Pop
  this list, I{'test1'} as well as I{'test1.example'} will match, but
1037 58885d79 Iustin Pop
  not I{'test1.ex'}. A multiple match will be considered as no match
1038 58885d79 Iustin Pop
  at all (e.g. I{'test1'} against C{['',
1039 3a541d90 Iustin Pop
  '']}), except when the key fully matches an entry
1040 3a541d90 Iustin Pop
  (e.g. I{'test1'} against C{['test1', '']}).
1041 a8083063 Iustin Pop

1042 58885d79 Iustin Pop
  @type key: str
1043 58885d79 Iustin Pop
  @param key: the name to be searched
1044 58885d79 Iustin Pop
  @type name_list: list
1045 58885d79 Iustin Pop
  @param name_list: the list of strings against which to search the key
1046 256eb94b Guido Trotter
  @type case_sensitive: boolean
1047 256eb94b Guido Trotter
  @param case_sensitive: whether to provide a case-sensitive match
1048 a8083063 Iustin Pop

1049 58885d79 Iustin Pop
  @rtype: None or str
1050 58885d79 Iustin Pop
  @return: None if there is no match I{or} if there are multiple matches,
1051 58885d79 Iustin Pop
      otherwise the element from the list which matches
1052 a8083063 Iustin Pop

1053 a8083063 Iustin Pop
1054 3a541d90 Iustin Pop
  if key in name_list:
1055 3a541d90 Iustin Pop
    return key
1056 256eb94b Guido Trotter
1057 256eb94b Guido Trotter
  re_flags = 0
1058 256eb94b Guido Trotter
  if not case_sensitive:
1059 256eb94b Guido Trotter
    re_flags |= re.IGNORECASE
1060 099c52ad Iustin Pop
    key = key.upper()
1061 256eb94b Guido Trotter
  mo = re.compile("^%s(\..*)?$" % re.escape(key), re_flags)
1062 256eb94b Guido Trotter
  names_filtered = []
1063 256eb94b Guido Trotter
  string_matches = []
1064 256eb94b Guido Trotter
  for name in name_list:
1065 256eb94b Guido Trotter
    if mo.match(name) is not None:
1066 256eb94b Guido Trotter
1067 099c52ad Iustin Pop
      if not case_sensitive and key == name.upper():
1068 256eb94b Guido Trotter
1069 256eb94b Guido Trotter
1070 256eb94b Guido Trotter
  if len(string_matches) == 1:
1071 256eb94b Guido Trotter
    return string_matches[0]
1072 256eb94b Guido Trotter
  if len(names_filtered) == 1:
1073 256eb94b Guido Trotter
    return names_filtered[0]
1074 256eb94b Guido Trotter
  return None
1075 a8083063 Iustin Pop
1076 a8083063 Iustin Pop
1077 28f34048 Michael Hanselmann
def ValidateServiceName(name):
1078 28f34048 Michael Hanselmann
  """Validate the given service name.
1079 28f34048 Michael Hanselmann

1080 28f34048 Michael Hanselmann
  @type name: number or string
1081 28f34048 Michael Hanselmann
  @param name: Service name or port specification
1082 28f34048 Michael Hanselmann

1083 28f34048 Michael Hanselmann
1084 28f34048 Michael Hanselmann
1085 28f34048 Michael Hanselmann
    numport = int(name)
1086 28f34048 Michael Hanselmann
  except (ValueError, TypeError):
1087 28f34048 Michael Hanselmann
    # Non-numeric service name
1088 28f34048 Michael Hanselmann
    valid = _VALID_SERVICE_NAME_RE.match(name)
1089 28f34048 Michael Hanselmann
1090 28f34048 Michael Hanselmann
    # Numeric port (protocols other than TCP or UDP might need adjustments
1091 28f34048 Michael Hanselmann
    # here)
1092 28f34048 Michael Hanselmann
    valid = (numport >= 0 and numport < (1 << 16))
1093 28f34048 Michael Hanselmann
1094 28f34048 Michael Hanselmann
  if not valid:
1095 28f34048 Michael Hanselmann
    raise errors.OpPrereqError("Invalid service name '%s'" % name,
1096 28f34048 Michael Hanselmann
1097 28f34048 Michael Hanselmann
1098 28f34048 Michael Hanselmann
  return name
1099 28f34048 Michael Hanselmann
1100 28f34048 Michael Hanselmann
1101 a8083063 Iustin Pop
def ListVolumeGroups():
1102 a8083063 Iustin Pop
  """List volume groups and their size
1103 a8083063 Iustin Pop

1104 58885d79 Iustin Pop
  @rtype: dict
1105 58885d79 Iustin Pop
1106 58885d79 Iustin Pop
       Dictionary with keys volume name and values
1107 58885d79 Iustin Pop
       the size of the volume
1108 a8083063 Iustin Pop

1109 a8083063 Iustin Pop
1110 a8083063 Iustin Pop
  command = "vgs --noheadings --units m --nosuffix -o name,size"
1111 a8083063 Iustin Pop
  result = RunCmd(command)
1112 a8083063 Iustin Pop
  retval = {}
1113 a8083063 Iustin Pop
  if result.failed:
1114 a8083063 Iustin Pop
    return retval
1115 a8083063 Iustin Pop
1116 a8083063 Iustin Pop
  for line in result.stdout.splitlines():
1117 a8083063 Iustin Pop
1118 a8083063 Iustin Pop
      name, size = line.split()
1119 a8083063 Iustin Pop
      size = int(float(size))
1120 a8083063 Iustin Pop
    except (IndexError, ValueError), err:
1121 bb698c1f Iustin Pop
      logging.error("Invalid output from vgs (%s): %s", err, line)
1122 a8083063 Iustin Pop
1123 a8083063 Iustin Pop
1124 a8083063 Iustin Pop
    retval[name] = size
1125 a8083063 Iustin Pop
1126 a8083063 Iustin Pop
  return retval
1127 a8083063 Iustin Pop
1128 a8083063 Iustin Pop
1129 a8083063 Iustin Pop
def BridgeExists(bridge):
1130 a8083063 Iustin Pop
  """Check whether the given bridge exists in the system
1131 a8083063 Iustin Pop

1132 58885d79 Iustin Pop
  @type bridge: str
1133 58885d79 Iustin Pop
  @param bridge: the bridge name to check
1134 58885d79 Iustin Pop
  @rtype: boolean
1135 58885d79 Iustin Pop
  @return: True if it does
1136 a8083063 Iustin Pop

1137 a8083063 Iustin Pop
1138 a8083063 Iustin Pop
  return os.path.isdir("/sys/class/net/%s/bridge" % bridge)
1139 a8083063 Iustin Pop
1140 a8083063 Iustin Pop
1141 a8083063 Iustin Pop
def NiceSort(name_list):
1142 a8083063 Iustin Pop
  """Sort a list of strings based on digit and non-digit groupings.
1143 a8083063 Iustin Pop

1144 58885d79 Iustin Pop
  Given a list of names C{['a1', 'a10', 'a11', 'a2']} this function
1145 58885d79 Iustin Pop
  will sort the list in the logical order C{['a1', 'a2', 'a10',
1146 58885d79 Iustin Pop
1147 a8083063 Iustin Pop

1148 a8083063 Iustin Pop
  The sort algorithm breaks each name in groups of either only-digits
1149 a8083063 Iustin Pop
  or no-digits. Only the first eight such groups are considered, and
1150 a8083063 Iustin Pop
  after that we just use what's left of the string.
1151 a8083063 Iustin Pop

1152 58885d79 Iustin Pop
  @type name_list: list
1153 58885d79 Iustin Pop
  @param name_list: the names to be sorted
1154 58885d79 Iustin Pop
  @rtype: list
1155 58885d79 Iustin Pop
  @return: a copy of the name list sorted with our algorithm
1156 a8083063 Iustin Pop

1157 a8083063 Iustin Pop
1158 a8083063 Iustin Pop
  _SORTER_BASE = "(\D+|\d+)"
1159 a8083063 Iustin Pop
  _SORTER_FULL = "^%s%s?%s?%s?%s?%s?%s?%s?.*$" % (_SORTER_BASE, _SORTER_BASE,
1160 a8083063 Iustin Pop
                                                  _SORTER_BASE, _SORTER_BASE,
1161 a8083063 Iustin Pop
                                                  _SORTER_BASE, _SORTER_BASE,
1162 a8083063 Iustin Pop
                                                  _SORTER_BASE, _SORTER_BASE)
1163 a8083063 Iustin Pop
  _SORTER_RE = re.compile(_SORTER_FULL)
1164 a8083063 Iustin Pop
  _SORTER_NODIGIT = re.compile("^\D*$")
1165 a8083063 Iustin Pop
  def _TryInt(val):
1166 a8083063 Iustin Pop
    """Attempts to convert a variable to integer."""
1167 a8083063 Iustin Pop
    if val is None or _SORTER_NODIGIT.match(val):
1168 a8083063 Iustin Pop
      return val
1169 a8083063 Iustin Pop
    rval = int(val)
1170 a8083063 Iustin Pop
    return rval
1171 a8083063 Iustin Pop
1172 a8083063 Iustin Pop
  to_sort = [([_TryInt(grp) for grp in _SORTER_RE.match(name).groups()], name)
1173 a8083063 Iustin Pop
             for name in name_list]
1174 a8083063 Iustin Pop
1175 a8083063 Iustin Pop
  return [tup[1] for tup in to_sort]
1176 a8083063 Iustin Pop
1177 a8083063 Iustin Pop
1178 a8083063 Iustin Pop
def TryConvert(fn, val):
1179 a8083063 Iustin Pop
  """Try to convert a value ignoring errors.
1180 a8083063 Iustin Pop

1181 58885d79 Iustin Pop
  This function tries to apply function I{fn} to I{val}. If no
1182 58885d79 Iustin Pop
  C{ValueError} or C{TypeError} exceptions are raised, it will return
1183 58885d79 Iustin Pop
  the result, else it will return the original value. Any other
1184 58885d79 Iustin Pop
  exceptions are propagated to the caller.
1185 58885d79 Iustin Pop

1186 58885d79 Iustin Pop
  @type fn: callable
1187 58885d79 Iustin Pop
  @param fn: function to apply to the value
1188 58885d79 Iustin Pop
  @param val: the value to be converted
1189 58885d79 Iustin Pop
  @return: The converted value if the conversion was successful,
1190 58885d79 Iustin Pop
      otherwise the original value.
1191 a8083063 Iustin Pop

1192 a8083063 Iustin Pop
1193 a8083063 Iustin Pop
1194 a8083063 Iustin Pop
    nv = fn(val)
1195 7c4d6c7b Michael Hanselmann
  except (ValueError, TypeError):
1196 a8083063 Iustin Pop
    nv = val
1197 a8083063 Iustin Pop
  return nv
1198 a8083063 Iustin Pop
1199 a8083063 Iustin Pop
1200 a8083063 Iustin Pop
def IsValidShellParam(word):
1201 a8083063 Iustin Pop
  """Verifies is the given word is safe from the shell's p.o.v.
1202 a8083063 Iustin Pop

1203 a8083063 Iustin Pop
  This means that we can pass this to a command via the shell and be
1204 a8083063 Iustin Pop
  sure that it doesn't alter the command line and is passed as such to
1205 a8083063 Iustin Pop
  the actual command.
1206 a8083063 Iustin Pop

1207 a8083063 Iustin Pop
  Note that we are overly restrictive here, in order to be on the safe
1208 a8083063 Iustin Pop
1209 a8083063 Iustin Pop

1210 58885d79 Iustin Pop
  @type word: str
1211 58885d79 Iustin Pop
  @param word: the word to check
1212 58885d79 Iustin Pop
  @rtype: boolean
1213 58885d79 Iustin Pop
  @return: True if the word is 'safe'
1214 58885d79 Iustin Pop

1215 a8083063 Iustin Pop
1216 a8083063 Iustin Pop
  return bool(re.match("^[-a-zA-Z0-9._+/:%@]+$", word))
1217 a8083063 Iustin Pop
1218 a8083063 Iustin Pop
1219 a8083063 Iustin Pop
def BuildShellCmd(template, *args):
1220 a8083063 Iustin Pop
  """Build a safe shell command line from the given arguments.
1221 a8083063 Iustin Pop

1222 a8083063 Iustin Pop
  This function will check all arguments in the args list so that they
1223 a8083063 Iustin Pop
  are valid shell parameters (i.e. they don't contain shell
1224 5bbd3f7f Michael Hanselmann
  metacharacters). If everything is ok, it will return the result of
1225 a8083063 Iustin Pop
  template % args.
1226 a8083063 Iustin Pop

1227 58885d79 Iustin Pop
  @type template: str
1228 58885d79 Iustin Pop
  @param template: the string holding the template for the
1229 58885d79 Iustin Pop
      string formatting
1230 58885d79 Iustin Pop
  @rtype: str
1231 58885d79 Iustin Pop
  @return: the expanded command line
1232 58885d79 Iustin Pop

1233 a8083063 Iustin Pop
1234 a8083063 Iustin Pop
  for word in args:
1235 a8083063 Iustin Pop
    if not IsValidShellParam(word):
1236 3ecf6786 Iustin Pop
      raise errors.ProgrammerError("Shell argument '%s' contains"
1237 3ecf6786 Iustin Pop
                                   " invalid characters" % word)
1238 a8083063 Iustin Pop
  return template % args
1239 a8083063 Iustin Pop
1240 a8083063 Iustin Pop
1241 9fbfbb7b Iustin Pop
def FormatUnit(value, units):
1242 a8083063 Iustin Pop
  """Formats an incoming number of MiB with the appropriate unit.
1243 a8083063 Iustin Pop

1244 58885d79 Iustin Pop
  @type value: int
1245 58885d79 Iustin Pop
  @param value: integer representing the value in MiB (1048576)
1246 9fbfbb7b Iustin Pop
  @type units: char
1247 9fbfbb7b Iustin Pop
  @param units: the type of formatting we should do:
1248 9fbfbb7b Iustin Pop
      - 'h' for automatic scaling
1249 9fbfbb7b Iustin Pop
      - 'm' for MiBs
1250 9fbfbb7b Iustin Pop
      - 'g' for GiBs
1251 9fbfbb7b Iustin Pop
      - 't' for TiBs
1252 58885d79 Iustin Pop
  @rtype: str
1253 58885d79 Iustin Pop
  @return: the formatted value (with suffix)
1254 a8083063 Iustin Pop

1255 a8083063 Iustin Pop
1256 9fbfbb7b Iustin Pop
  if units not in ('m', 'g', 't', 'h'):
1257 9fbfbb7b Iustin Pop
    raise errors.ProgrammerError("Invalid unit specified '%s'" % str(units))
1258 a8083063 Iustin Pop
1259 9fbfbb7b Iustin Pop
  suffix = ''
1260 9fbfbb7b Iustin Pop
1261 9fbfbb7b Iustin Pop
  if units == 'm' or (units == 'h' and value < 1024):
1262 9fbfbb7b Iustin Pop
    if units == 'h':
1263 9fbfbb7b Iustin Pop
      suffix = 'M'
1264 9fbfbb7b Iustin Pop
    return "%d%s" % (round(value, 0), suffix)
1265 9fbfbb7b Iustin Pop
1266 9fbfbb7b Iustin Pop
  elif units == 'g' or (units == 'h' and value < (1024 * 1024)):
1267 9fbfbb7b Iustin Pop
    if units == 'h':
1268 9fbfbb7b Iustin Pop
      suffix = 'G'
1269 9fbfbb7b Iustin Pop
    return "%0.1f%s" % (round(float(value) / 1024, 1), suffix)
1270 a8083063 Iustin Pop
1271 a8083063 Iustin Pop
1272 9fbfbb7b Iustin Pop
    if units == 'h':
1273 9fbfbb7b Iustin Pop
      suffix = 'T'
1274 9fbfbb7b Iustin Pop
    return "%0.1f%s" % (round(float(value) / 1024 / 1024, 1), suffix)
1275 a8083063 Iustin Pop
1276 a8083063 Iustin Pop
1277 a8083063 Iustin Pop
def ParseUnit(input_string):
1278 a8083063 Iustin Pop
  """Tries to extract number and scale from the given string.
1279 a8083063 Iustin Pop

1280 58885d79 Iustin Pop
  Input must be in the format C{NUMBER+ [DOT NUMBER+] SPACE*
1281 58885d79 Iustin Pop
  [UNIT]}. If no unit is specified, it defaults to MiB. Return value
1282 58885d79 Iustin Pop
  is always an int in MiB.
1283 a8083063 Iustin Pop

1284 a8083063 Iustin Pop
1285 9939547b Iustin Pop
  m = re.match('^([.\d]+)\s*([a-zA-Z]+)?$', str(input_string))
1286 a8083063 Iustin Pop
  if not m:
1287 3ecf6786 Iustin Pop
    raise errors.UnitParseError("Invalid format")
1288 a8083063 Iustin Pop
1289 a8083063 Iustin Pop
  value = float(m.groups()[0])
1290 a8083063 Iustin Pop
1291 a8083063 Iustin Pop
  unit = m.groups()[1]
1292 a8083063 Iustin Pop
  if unit:
1293 a8083063 Iustin Pop
    lcunit = unit.lower()
1294 a8083063 Iustin Pop
1295 a8083063 Iustin Pop
    lcunit = 'm'
1296 a8083063 Iustin Pop
1297 a8083063 Iustin Pop
  if lcunit in ('m', 'mb', 'mib'):
1298 a8083063 Iustin Pop
    # Value already in MiB
1299 a8083063 Iustin Pop
1300 a8083063 Iustin Pop
1301 a8083063 Iustin Pop
  elif lcunit in ('g', 'gb', 'gib'):
1302 a8083063 Iustin Pop
    value *= 1024
1303 a8083063 Iustin Pop
1304 a8083063 Iustin Pop
  elif lcunit in ('t', 'tb', 'tib'):
1305 a8083063 Iustin Pop
    value *= 1024 * 1024
1306 a8083063 Iustin Pop
1307 a8083063 Iustin Pop
1308 3ecf6786 Iustin Pop
    raise errors.UnitParseError("Unknown unit: %s" % unit)
1309 a8083063 Iustin Pop
1310 a8083063 Iustin Pop
  # Make sure we round up
1311 a8083063 Iustin Pop
  if int(value) < value:
1312 a8083063 Iustin Pop
    value += 1
1313 a8083063 Iustin Pop
1314 a8083063 Iustin Pop
  # Round up to the next multiple of 4
1315 a8083063 Iustin Pop
  value = int(value)
1316 a8083063 Iustin Pop
  if value % 4:
1317 a8083063 Iustin Pop
    value += 4 - value % 4
1318 a8083063 Iustin Pop
1319 a8083063 Iustin Pop
  return value
1320 a8083063 Iustin Pop
1321 a8083063 Iustin Pop
1322 31155d60 Balazs Lecz
def ParseCpuMask(cpu_mask):
1323 31155d60 Balazs Lecz
  """Parse a CPU mask definition and return the list of CPU IDs.
1324 31155d60 Balazs Lecz

1325 31155d60 Balazs Lecz
  CPU mask format: comma-separated list of CPU IDs
1326 31155d60 Balazs Lecz
  or dash-separated ID ranges
1327 31155d60 Balazs Lecz
  Example: "0-2,5" -> "0,1,2,5"
1328 31155d60 Balazs Lecz

1329 31155d60 Balazs Lecz
  @type cpu_mask: str
1330 31155d60 Balazs Lecz
  @param cpu_mask: CPU mask definition
1331 31155d60 Balazs Lecz
  @rtype: list of int
1332 31155d60 Balazs Lecz
  @return: list of CPU IDs
1333 31155d60 Balazs Lecz

1334 31155d60 Balazs Lecz
1335 31155d60 Balazs Lecz
  if not cpu_mask:
1336 31155d60 Balazs Lecz
    return []
1337 31155d60 Balazs Lecz
  cpu_list = []
1338 31155d60 Balazs Lecz
  for range_def in cpu_mask.split(","):
1339 31155d60 Balazs Lecz
    boundaries = range_def.split("-")
1340 31155d60 Balazs Lecz
    n_elements = len(boundaries)
1341 31155d60 Balazs Lecz
    if n_elements > 2:
1342 31155d60 Balazs Lecz
      raise errors.ParseError("Invalid CPU ID range definition"
1343 31155d60 Balazs Lecz
                              " (only one hyphen allowed): %s" % range_def)
1344 31155d60 Balazs Lecz
1345 31155d60 Balazs Lecz
      lower = int(boundaries[0])
1346 31155d60 Balazs Lecz
    except (ValueError, TypeError), err:
1347 31155d60 Balazs Lecz
      raise errors.ParseError("Invalid CPU ID value for lower boundary of"
1348 31155d60 Balazs Lecz
                              " CPU ID range: %s" % str(err))
1349 31155d60 Balazs Lecz
1350 31155d60 Balazs Lecz
      higher = int(boundaries[-1])
1351 31155d60 Balazs Lecz
    except (ValueError, TypeError), err:
1352 31155d60 Balazs Lecz
      raise errors.ParseError("Invalid CPU ID value for higher boundary of"
1353 31155d60 Balazs Lecz
                              " CPU ID range: %s" % str(err))
1354 31155d60 Balazs Lecz
    if lower > higher:
1355 31155d60 Balazs Lecz
      raise errors.ParseError("Invalid CPU ID range definition"
1356 31155d60 Balazs Lecz
                              " (%d > %d): %s" % (lower, higher, range_def))
1357 31155d60 Balazs Lecz
    cpu_list.extend(range(lower, higher + 1))
1358 31155d60 Balazs Lecz
  return cpu_list
1359 31155d60 Balazs Lecz
1360 31155d60 Balazs Lecz
1361 3727671e René Nussbaumer
def AddAuthorizedKey(file_obj, key):
1362 a8083063 Iustin Pop
  """Adds an SSH public key to an authorized_keys file.
1363 a8083063 Iustin Pop

1364 3727671e René Nussbaumer
  @type file_obj: str or file handle
1365 3727671e René Nussbaumer
  @param file_obj: path to authorized_keys file
1366 58885d79 Iustin Pop
  @type key: str
1367 58885d79 Iustin Pop
  @param key: string containing key
1368 58885d79 Iustin Pop

1369 a8083063 Iustin Pop
1370 a8083063 Iustin Pop
  key_fields = key.split()
1371 a8083063 Iustin Pop
1372 3727671e René Nussbaumer
  if isinstance(file_obj, basestring):
1373 3727671e René Nussbaumer
    f = open(file_obj, 'a+')
1374 3727671e René Nussbaumer
1375 3727671e René Nussbaumer
    f = file_obj
1376 3727671e René Nussbaumer
1377 a8083063 Iustin Pop
1378 a8083063 Iustin Pop
    nl = True
1379 a8083063 Iustin Pop
    for line in f:
1380 a8083063 Iustin Pop
      # Ignore whitespace changes
1381 a8083063 Iustin Pop
      if line.split() == key_fields:
1382 a8083063 Iustin Pop
1383 a8083063 Iustin Pop
      nl = line.endswith('\n')
1384 a8083063 Iustin Pop
1385 a8083063 Iustin Pop
      if not nl:
1386 a8083063 Iustin Pop
1387 a8083063 Iustin Pop
1388 a8083063 Iustin Pop
1389 a8083063 Iustin Pop
1390 a8083063 Iustin Pop
1391 a8083063 Iustin Pop
1392 a8083063 Iustin Pop
1393 a8083063 Iustin Pop
1394 a8083063 Iustin Pop
def RemoveAuthorizedKey(file_name, key):
1395 a8083063 Iustin Pop
  """Removes an SSH public key from an authorized_keys file.
1396 a8083063 Iustin Pop

1397 58885d79 Iustin Pop
  @type file_name: str
1398 58885d79 Iustin Pop
  @param file_name: path to authorized_keys file
1399 58885d79 Iustin Pop
  @type key: str
1400 58885d79 Iustin Pop
  @param key: string containing key
1401 58885d79 Iustin Pop

1402 a8083063 Iustin Pop
1403 a8083063 Iustin Pop
  key_fields = key.split()
1404 a8083063 Iustin Pop
1405 a8083063 Iustin Pop
  fd, tmpname = tempfile.mkstemp(dir=os.path.dirname(file_name))
1406 a8083063 Iustin Pop
1407 59f82e3f Michael Hanselmann
    out = os.fdopen(fd, 'w')
1408 a8083063 Iustin Pop
1409 59f82e3f Michael Hanselmann
      f = open(file_name, 'r')
1410 59f82e3f Michael Hanselmann
1411 59f82e3f Michael Hanselmann
        for line in f:
1412 59f82e3f Michael Hanselmann
          # Ignore whitespace changes while comparing lines
1413 59f82e3f Michael Hanselmann
          if line.split() != key_fields:
1414 59f82e3f Michael Hanselmann
1415 899d2a81 Michael Hanselmann
1416 899d2a81 Michael Hanselmann
1417 899d2a81 Michael Hanselmann
        os.rename(tmpname, file_name)
1418 899d2a81 Michael Hanselmann
1419 899d2a81 Michael Hanselmann
1420 899d2a81 Michael Hanselmann
1421 899d2a81 Michael Hanselmann
1422 899d2a81 Michael Hanselmann
1423 899d2a81 Michael Hanselmann
1424 899d2a81 Michael Hanselmann
1425 899d2a81 Michael Hanselmann
1426 899d2a81 Michael Hanselmann
1427 9440aeab Michael Hanselmann
def SetEtcHostsEntry(file_name, ip, hostname, aliases):
1428 9440aeab Michael Hanselmann
  """Sets the name of an IP address and hostname in /etc/hosts.
1429 899d2a81 Michael Hanselmann

1430 58885d79 Iustin Pop
  @type file_name: str
1431 58885d79 Iustin Pop
  @param file_name: path to the file to modify (usually C{/etc/hosts})
1432 58885d79 Iustin Pop
  @type ip: str
1433 58885d79 Iustin Pop
  @param ip: the IP address
1434 58885d79 Iustin Pop
  @type hostname: str
1435 58885d79 Iustin Pop
  @param hostname: the hostname to be added
1436 58885d79 Iustin Pop
  @type aliases: list
1437 58885d79 Iustin Pop
  @param aliases: the list of aliases to add for the hostname
1438 58885d79 Iustin Pop

1439 899d2a81 Michael Hanselmann
1440 7fbb1f65 Michael Hanselmann
  # Ensure aliases are unique
1441 7fbb1f65 Michael Hanselmann
  aliases = UniqueSequence([hostname] + aliases)[1:]
1442 7fbb1f65 Michael Hanselmann
1443 edcd876b Michael Hanselmann
  def _WriteEtcHosts(fd):
1444 edcd876b Michael Hanselmann
    # Duplicating file descriptor because os.fdopen's result will automatically
1445 edcd876b Michael Hanselmann
    # close the descriptor, but we would still like to have its functionality.
1446 edcd876b Michael Hanselmann
    out = os.fdopen(os.dup(fd), "w")
1447 9440aeab Michael Hanselmann
1448 edcd876b Michael Hanselmann
      for line in ReadFile(file_name).splitlines(True):
1449 edcd876b Michael Hanselmann
        fields = line.split()
1450 edcd876b Michael Hanselmann
        if fields and not fields[0].startswith("#") and ip == fields[0]:
1451 edcd876b Michael Hanselmann
1452 edcd876b Michael Hanselmann
1453 edcd876b Michael Hanselmann
1454 edcd876b Michael Hanselmann
      out.write("%s\t%s" % (ip, hostname))
1455 edcd876b Michael Hanselmann
      if aliases:
1456 edcd876b Michael Hanselmann
        out.write(" %s" % " ".join(aliases))
1457 edcd876b Michael Hanselmann
1458 edcd876b Michael Hanselmann
1459 9440aeab Michael Hanselmann
1460 9440aeab Michael Hanselmann
1461 edcd876b Michael Hanselmann
1462 edcd876b Michael Hanselmann
  WriteFile(file_name, fn=_WriteEtcHosts, mode=0644)
1463 899d2a81 Michael Hanselmann
1464 899d2a81 Michael Hanselmann
1465 d9c02ca6 Michael Hanselmann
def AddHostToEtcHosts(hostname):
1466 d9c02ca6 Michael Hanselmann
  """Wrapper around SetEtcHostsEntry.
1467 d9c02ca6 Michael Hanselmann

1468 58885d79 Iustin Pop
  @type hostname: str
1469 58885d79 Iustin Pop
  @param hostname: a hostname that will be resolved and added to
1470 58885d79 Iustin Pop
1471 58885d79 Iustin Pop

1472 d9c02ca6 Michael Hanselmann
1473 b43dcc5a Manuel Franceschini
  SetEtcHostsEntry(constants.ETC_HOSTS, hostname.ip,,
1474 b43dcc5a Manuel Franceschini
1475 d9c02ca6 Michael Hanselmann
1476 d9c02ca6 Michael Hanselmann
1477 899d2a81 Michael Hanselmann
def RemoveEtcHostsEntry(file_name, hostname):
1478 3e1cdf9f Michael Hanselmann
  """Removes a hostname from /etc/hosts.
1479 899d2a81 Michael Hanselmann

1480 9440aeab Michael Hanselmann
  IP addresses without names are removed from the file.
1481 58885d79 Iustin Pop

1482 58885d79 Iustin Pop
  @type file_name: str
1483 58885d79 Iustin Pop
  @param file_name: path to the file to modify (usually C{/etc/hosts})
1484 58885d79 Iustin Pop
  @type hostname: str
1485 58885d79 Iustin Pop
  @param hostname: the hostname to be removed
1486 58885d79 Iustin Pop

1487 899d2a81 Michael Hanselmann
1488 edcd876b Michael Hanselmann
  def _WriteEtcHosts(fd):
1489 edcd876b Michael Hanselmann
    # Duplicating file descriptor because os.fdopen's result will automatically
1490 edcd876b Michael Hanselmann
    # close the descriptor, but we would still like to have its functionality.
1491 edcd876b Michael Hanselmann
    out = os.fdopen(os.dup(fd), "w")
1492 899d2a81 Michael Hanselmann
1493 edcd876b Michael Hanselmann
      for line in ReadFile(file_name).splitlines(True):
1494 edcd876b Michael Hanselmann
        fields = line.split()
1495 edcd876b Michael Hanselmann
        if len(fields) > 1 and not fields[0].startswith("#"):
1496 edcd876b Michael Hanselmann
          names = fields[1:]
1497 edcd876b Michael Hanselmann
          if hostname in names:
1498 edcd876b Michael Hanselmann
            while hostname in names:
1499 edcd876b Michael Hanselmann
1500 edcd876b Michael Hanselmann
            if names:
1501 edcd876b Michael Hanselmann
              out.write("%s %s\n" % (fields[0], " ".join(names)))
1502 edcd876b Michael Hanselmann
1503 59f82e3f Michael Hanselmann
1504 edcd876b Michael Hanselmann
1505 edcd876b Michael Hanselmann
1506 edcd876b Michael Hanselmann
1507 a8083063 Iustin Pop
1508 59f82e3f Michael Hanselmann
1509 edcd876b Michael Hanselmann
1510 edcd876b Michael Hanselmann
  WriteFile(file_name, fn=_WriteEtcHosts, mode=0644)
1511 a8083063 Iustin Pop
1512 a8083063 Iustin Pop
1513 d9c02ca6 Michael Hanselmann
def RemoveHostFromEtcHosts(hostname):
1514 d9c02ca6 Michael Hanselmann
  """Wrapper around RemoveEtcHostsEntry.
1515 d9c02ca6 Michael Hanselmann

1516 58885d79 Iustin Pop
  @type hostname: str
1517 58885d79 Iustin Pop
  @param hostname: hostname that will be resolved and its
1518 58885d79 Iustin Pop
      full and shot name will be removed from
1519 58885d79 Iustin Pop
1520 58885d79 Iustin Pop

1521 d9c02ca6 Michael Hanselmann
1522 b705c7a6 Manuel Franceschini
  RemoveEtcHostsEntry(constants.ETC_HOSTS, hostname)
1523 b705c7a6 Manuel Franceschini
  RemoveEtcHostsEntry(constants.ETC_HOSTS, hostname.split(".")[0])
1524 d9c02ca6 Michael Hanselmann
1525 d9c02ca6 Michael Hanselmann
1526 1d466a4f Michael Hanselmann
def TimestampForFilename():
1527 1d466a4f Michael Hanselmann
  """Returns the current time formatted for filenames.
1528 1d466a4f Michael Hanselmann

1529 1d466a4f Michael Hanselmann
  The format doesn't contain colons as some shells and applications them as
1530 1d466a4f Michael Hanselmann
1531 1d466a4f Michael Hanselmann

1532 1d466a4f Michael Hanselmann
1533 1d466a4f Michael Hanselmann
  return time.strftime("%Y-%m-%d_%H_%M_%S")
1534 1d466a4f Michael Hanselmann
1535 1d466a4f Michael Hanselmann
1536 a8083063 Iustin Pop
def CreateBackup(file_name):
1537 a8083063 Iustin Pop
  """Creates a backup of a file.
1538 a8083063 Iustin Pop

1539 58885d79 Iustin Pop
  @type file_name: str
1540 58885d79 Iustin Pop
  @param file_name: file to be backed up
1541 58885d79 Iustin Pop
  @rtype: str
1542 58885d79 Iustin Pop
  @return: the path to the newly created backup
1543 58885d79 Iustin Pop
  @raise errors.ProgrammerError: for invalid file names
1544 a8083063 Iustin Pop

1545 a8083063 Iustin Pop
1546 a8083063 Iustin Pop
  if not os.path.isfile(file_name):
1547 3ecf6786 Iustin Pop
    raise errors.ProgrammerError("Can't make a backup of a non-file '%s'" %
1548 3ecf6786 Iustin Pop
1549 a8083063 Iustin Pop
1550 1d466a4f Michael Hanselmann
  prefix = ("%s.backup-%s." %
1551 1d466a4f Michael Hanselmann
            (os.path.basename(file_name), TimestampForFilename()))
1552 65fe4693 Iustin Pop
  dir_name = os.path.dirname(file_name)
1553 081b1e69 Michael Hanselmann
1554 081b1e69 Michael Hanselmann
  fsrc = open(file_name, 'rb')
1555 081b1e69 Michael Hanselmann
1556 65fe4693 Iustin Pop
    (fd, backup_name) = tempfile.mkstemp(prefix=prefix, dir=dir_name)
1557 081b1e69 Michael Hanselmann
    fdst = os.fdopen(fd, 'wb')
1558 081b1e69 Michael Hanselmann
1559 1d466a4f Michael Hanselmann
      logging.debug("Backing up %s at %s", file_name, backup_name)
1560 081b1e69 Michael Hanselmann
      shutil.copyfileobj(fsrc, fdst)
1561 081b1e69 Michael Hanselmann
1562 081b1e69 Michael Hanselmann
1563 081b1e69 Michael Hanselmann
1564 081b1e69 Michael Hanselmann
1565 081b1e69 Michael Hanselmann
1566 a8083063 Iustin Pop
  return backup_name
1567 a8083063 Iustin Pop
1568 a8083063 Iustin Pop
1569 a8083063 Iustin Pop
def ShellQuote(value):
1570 a8083063 Iustin Pop
  """Quotes shell argument according to POSIX.
1571 3ecf6786 Iustin Pop

1572 58885d79 Iustin Pop
  @type value: str
1573 58885d79 Iustin Pop
  @param value: the argument to be quoted
1574 58885d79 Iustin Pop
  @rtype: str
1575 58885d79 Iustin Pop
  @return: the quoted value
1576 58885d79 Iustin Pop

1577 a8083063 Iustin Pop
1578 a8083063 Iustin Pop
  if _re_shell_unquoted.match(value):
1579 a8083063 Iustin Pop
    return value
1580 a8083063 Iustin Pop
1581 a8083063 Iustin Pop
    return "'%s'" % value.replace("'", "'\\''")
1582 a8083063 Iustin Pop
1583 a8083063 Iustin Pop
1584 a8083063 Iustin Pop
def ShellQuoteArgs(args):
1585 58885d79 Iustin Pop
  """Quotes a list of shell arguments.
1586 58885d79 Iustin Pop

1587 58885d79 Iustin Pop
  @type args: list
1588 58885d79 Iustin Pop
  @param args: list of arguments to be quoted
1589 58885d79 Iustin Pop
  @rtype: str
1590 5bbd3f7f Michael Hanselmann
  @return: the quoted arguments concatenated with spaces
1591 a8083063 Iustin Pop

1592 a8083063 Iustin Pop
1593 a8083063 Iustin Pop
  return ' '.join([ShellQuote(i) for i in args])
1594 88d14415 Michael Hanselmann
1595 88d14415 Michael Hanselmann
1596 858905fb Michael Hanselmann
class ShellWriter:
1597 858905fb Michael Hanselmann
  """Helper class to write scripts with indentation.
1598 858905fb Michael Hanselmann

1599 858905fb Michael Hanselmann
1600 858905fb Michael Hanselmann
  INDENT_STR = "  "
1601 858905fb Michael Hanselmann
1602 858905fb Michael Hanselmann
  def __init__(self, fh):
1603 858905fb Michael Hanselmann
    """Initializes this class.
1604 858905fb Michael Hanselmann

1605 858905fb Michael Hanselmann
1606 858905fb Michael Hanselmann
    self._fh = fh
1607 858905fb Michael Hanselmann
    self._indent = 0
1608 858905fb Michael Hanselmann
1609 858905fb Michael Hanselmann
  def IncIndent(self):
1610 858905fb Michael Hanselmann
    """Increase indentation level by 1.
1611 858905fb Michael Hanselmann

1612 858905fb Michael Hanselmann
1613 858905fb Michael Hanselmann
    self._indent += 1
1614 858905fb Michael Hanselmann
1615 858905fb Michael Hanselmann
  def DecIndent(self):
1616 858905fb Michael Hanselmann
    """Decrease indentation level by 1.
1617 858905fb Michael Hanselmann

1618 858905fb Michael Hanselmann
1619 858905fb Michael Hanselmann
    assert self._indent > 0
1620 858905fb Michael Hanselmann
    self._indent -= 1
1621 858905fb Michael Hanselmann
1622 858905fb Michael Hanselmann
  def Write(self, txt, *args):
1623 858905fb Michael Hanselmann
    """Write line to output file.
1624 858905fb Michael Hanselmann

1625 858905fb Michael Hanselmann
1626 858905fb Michael Hanselmann
    assert self._indent >= 0
1627 858905fb Michael Hanselmann
1628 858905fb Michael Hanselmann
    self._fh.write(self._indent * self.INDENT_STR)
1629 858905fb Michael Hanselmann
1630 858905fb Michael Hanselmann
    if args:
1631 858905fb Michael Hanselmann
      self._fh.write(txt % args)
1632 858905fb Michael Hanselmann
1633 858905fb Michael Hanselmann
1634 858905fb Michael Hanselmann
1635 858905fb Michael Hanselmann
1636 858905fb Michael Hanselmann
1637 858905fb Michael Hanselmann
1638 b5b8309d Guido Trotter
def ListVisibleFiles(path):
1639 58885d79 Iustin Pop
  """Returns a list of visible files in a directory.
1640 58885d79 Iustin Pop

1641 58885d79 Iustin Pop
  @type path: str
1642 58885d79 Iustin Pop
  @param path: the directory to enumerate
1643 58885d79 Iustin Pop
  @rtype: list
1644 58885d79 Iustin Pop
  @return: the list of all files not starting with a dot
1645 04a69a18 Iustin Pop
  @raise ProgrammerError: if L{path} is not an absolue and normalized path
1646 eedbda4b Michael Hanselmann

1647 eedbda4b Michael Hanselmann
1648 04a69a18 Iustin Pop
  if not IsNormAbsPath(path):
1649 04a69a18 Iustin Pop
    raise errors.ProgrammerError("Path passed to ListVisibleFiles is not"
1650 04a69a18 Iustin Pop
                                 " absolute/normalized: '%s'" % path)
1651 f3299a07 Michael Hanselmann
  files = [i for i in os.listdir(path) if not i.startswith(".")]
1652 f3299a07 Michael Hanselmann
  return files
1653 2f8b60b3 Iustin Pop
1654 2f8b60b3 Iustin Pop
1655 257f4c0a Iustin Pop
def GetHomeDir(user, default=None):
1656 257f4c0a Iustin Pop
  """Try to get the homedir of the given user.
1657 257f4c0a Iustin Pop

1658 257f4c0a Iustin Pop
  The user can be passed either as a string (denoting the name) or as
1659 257f4c0a Iustin Pop
  an integer (denoting the user id). If the user is not found, the
1660 257f4c0a Iustin Pop
  'default' argument is returned, which defaults to None.
1661 2f8b60b3 Iustin Pop

1662 2f8b60b3 Iustin Pop
1663 2f8b60b3 Iustin Pop
1664 257f4c0a Iustin Pop
    if isinstance(user, basestring):
1665 257f4c0a Iustin Pop
      result = pwd.getpwnam(user)
1666 257f4c0a Iustin Pop
    elif isinstance(user, (int, long)):
1667 257f4c0a Iustin Pop
      result = pwd.getpwuid(user)
1668 257f4c0a Iustin Pop
1669 257f4c0a Iustin Pop
      raise errors.ProgrammerError("Invalid type passed to GetHomeDir (%s)" %
1670 257f4c0a Iustin Pop
1671 2f8b60b3 Iustin Pop
  except KeyError:
1672 2f8b60b3 Iustin Pop
    return default
1673 2f8b60b3 Iustin Pop
  return result.pw_dir
1674 59072e7e Michael Hanselmann
1675 59072e7e Michael Hanselmann
1676 24818e8f Michael Hanselmann
def NewUUID():
1677 59072e7e Michael Hanselmann
  """Returns a random UUID.
1678 59072e7e Michael Hanselmann

1679 58885d79 Iustin Pop
  @note: This is a Linux-specific method as it uses the /proc
1680 58885d79 Iustin Pop
1681 58885d79 Iustin Pop
  @rtype: str
1682 58885d79 Iustin Pop

1683 59072e7e Michael Hanselmann
1684 13998ef2 Michael Hanselmann
  return ReadFile(_RANDOM_UUID_FILE, size=128).rstrip("\n")
1685 087b34fe Iustin Pop
1686 087b34fe Iustin Pop
1687 ec2c2bc4 Luca Bigliardi
def GenerateSecret(numbytes=20):
1688 33081d90 Iustin Pop
  """Generates a random secret.
1689 33081d90 Iustin Pop

1690 ec2c2bc4 Luca Bigliardi
  This will generate a pseudo-random secret returning an hex string
1691 33081d90 Iustin Pop
  (so that it can be used where an ASCII string is needed).
1692 33081d90 Iustin Pop

1693 ec2c2bc4 Luca Bigliardi
  @param numbytes: the number of bytes which will be represented by the returned
1694 ec2c2bc4 Luca Bigliardi
      string (defaulting to 20, the length of a SHA1 hash)
1695 58885d79 Iustin Pop
  @rtype: str
1696 ec2c2bc4 Luca Bigliardi
  @return: an hex representation of the pseudo-random sequence
1697 58885d79 Iustin Pop

1698 33081d90 Iustin Pop
1699 ec2c2bc4 Luca Bigliardi
  return os.urandom(numbytes).encode('hex')
1700 33081d90 Iustin Pop
1701 33081d90 Iustin Pop
1702 9dae41ad Guido Trotter
def EnsureDirs(dirs):
1703 9dae41ad Guido Trotter
  """Make required directories, if they don't exist.
1704 9dae41ad Guido Trotter

1705 9dae41ad Guido Trotter
  @param dirs: list of tuples (dir_name, dir_mode)
1706 9dae41ad Guido Trotter
  @type dirs: list of (string, integer)
1707 9dae41ad Guido Trotter

1708 9dae41ad Guido Trotter
1709 9dae41ad Guido Trotter
  for dir_name, dir_mode in dirs:
1710 9dae41ad Guido Trotter
1711 1b2c8f85 Iustin Pop
      os.mkdir(dir_name, dir_mode)
1712 9dae41ad Guido Trotter
    except EnvironmentError, err:
1713 9dae41ad Guido Trotter
      if err.errno != errno.EEXIST:
1714 9dae41ad Guido Trotter
        raise errors.GenericError("Cannot create needed directory"
1715 1b2c8f85 Iustin Pop
                                  " '%s': %s" % (dir_name, err))
1716 b73360e3 Balazs Lecz
1717 b73360e3 Balazs Lecz
      os.chmod(dir_name, dir_mode)
1718 b73360e3 Balazs Lecz
    except EnvironmentError, err:
1719 b73360e3 Balazs Lecz
      raise errors.GenericError("Cannot change directory permissions on"
1720 b73360e3 Balazs Lecz
                                " '%s': %s" % (dir_name, err))
1721 9dae41ad Guido Trotter
    if not os.path.isdir(dir_name):
1722 9dae41ad Guido Trotter
      raise errors.GenericError("%s is not a directory" % dir_name)
1723 9dae41ad Guido Trotter
1724 9dae41ad Guido Trotter
1725 582ed043 Guido Trotter
def ReadFile(file_name, size=-1):
1726 ca0aa6d0 Michael Hanselmann
  """Reads a file.
1727 ca0aa6d0 Michael Hanselmann

1728 016308cb Iustin Pop
  @type size: int
1729 016308cb Iustin Pop
  @param size: Read at most size bytes (if negative, entire file)
1730 58885d79 Iustin Pop
  @rtype: str
1731 5bbd3f7f Michael Hanselmann
  @return: the (possibly partial) content of the file
1732 ca0aa6d0 Michael Hanselmann

1733 ca0aa6d0 Michael Hanselmann
1734 ca0aa6d0 Michael Hanselmann
  f = open(file_name, "r")
1735 ca0aa6d0 Michael Hanselmann
1736 582ed043 Guido Trotter
1737 ca0aa6d0 Michael Hanselmann
1738 ca0aa6d0 Michael Hanselmann
1739 ca0aa6d0 Michael Hanselmann
1740 ca0aa6d0 Michael Hanselmann
1741 087b34fe Iustin Pop
def WriteFile(file_name, fn=None, data=None,
1742 087b34fe Iustin Pop
              mode=None, uid=-1, gid=-1,
1743 71714516 Michael Hanselmann
              atime=None, mtime=None, close=True,
1744 04a8d789 Michael Hanselmann
              dry_run=False, backup=False,
1745 71714516 Michael Hanselmann
              prewrite=None, postwrite=None):
1746 087b34fe Iustin Pop
  """(Over)write a file atomically.
1747 087b34fe Iustin Pop

1748 087b34fe Iustin Pop
  The file_name and either fn (a function taking one argument, the
1749 087b34fe Iustin Pop
  file descriptor, and which should write the data to it) or data (the
1750 087b34fe Iustin Pop
  contents of the file) must be passed. The other arguments are
1751 087b34fe Iustin Pop
  optional and allow setting the file mode, owner and group, and the
1752 087b34fe Iustin Pop
  mtime/atime of the file.
1753 087b34fe Iustin Pop

1754 087b34fe Iustin Pop
  If the function doesn't raise an exception, it has succeeded and the
1755 69efe319 Michael Hanselmann
  target file has the new contents. If the function has raised an
1756 087b34fe Iustin Pop
  exception, an existing target file should be unmodified and the
1757 087b34fe Iustin Pop
  temporary file should be removed.
1758 087b34fe Iustin Pop

1759 58885d79 Iustin Pop
  @type file_name: str
1760 58885d79 Iustin Pop
  @param file_name: the target filename
1761 58885d79 Iustin Pop
  @type fn: callable
1762 58885d79 Iustin Pop
  @param fn: content writing function, called with
1763 58885d79 Iustin Pop
      file descriptor as parameter
1764 69efe319 Michael Hanselmann
  @type data: str
1765 58885d79 Iustin Pop
  @param data: contents of the file
1766 58885d79 Iustin Pop
  @type mode: int
1767 58885d79 Iustin Pop
  @param mode: file mode
1768 58885d79 Iustin Pop
  @type uid: int
1769 58885d79 Iustin Pop
  @param uid: the owner of the file
1770 58885d79 Iustin Pop
  @type gid: int
1771 58885d79 Iustin Pop
  @param gid: the group of the file
1772 58885d79 Iustin Pop
  @type atime: int
1773 58885d79 Iustin Pop
  @param atime: a custom access time to be set on the file
1774 58885d79 Iustin Pop
  @type mtime: int
1775 58885d79 Iustin Pop
  @param mtime: a custom modification time to be set on the file
1776 58885d79 Iustin Pop
  @type close: boolean
1777 58885d79 Iustin Pop
  @param close: whether to close file after writing it
1778 58885d79 Iustin Pop
  @type prewrite: callable
1779 58885d79 Iustin Pop
  @param prewrite: function to be called before writing content
1780 58885d79 Iustin Pop
  @type postwrite: callable
1781 58885d79 Iustin Pop
  @param postwrite: function to be called after writing content
1782 58885d79 Iustin Pop

1783 58885d79 Iustin Pop
  @rtype: None or int
1784 58885d79 Iustin Pop
  @return: None if the 'close' parameter evaluates to True,
1785 58885d79 Iustin Pop
      otherwise the file descriptor
1786 58885d79 Iustin Pop

1787 69efe319 Michael Hanselmann
  @raise errors.ProgrammerError: if any of the arguments are not valid
1788 71714516 Michael Hanselmann

1789 087b34fe Iustin Pop
1790 04a8d789 Michael Hanselmann
  if not os.path.isabs(file_name):
1791 087b34fe Iustin Pop
    raise errors.ProgrammerError("Path passed to WriteFile is not"
1792 087b34fe Iustin Pop
                                 " absolute: '%s'" % file_name)
1793 087b34fe Iustin Pop
1794 087b34fe Iustin Pop
  if [fn, data].count(None) != 1:
1795 087b34fe Iustin Pop
    raise errors.ProgrammerError("fn or data required")
1796 087b34fe Iustin Pop
1797 087b34fe Iustin Pop
  if [atime, mtime].count(None) == 1:
1798 087b34fe Iustin Pop
    raise errors.ProgrammerError("Both atime and mtime must be either"
1799 087b34fe Iustin Pop
                                 " set or None")
1800 087b34fe Iustin Pop
1801 70f4497c Michael Hanselmann
  if backup and not dry_run and os.path.isfile(file_name):
1802 70f4497c Michael Hanselmann
1803 087b34fe Iustin Pop
1804 087b34fe Iustin Pop
  dir_name, base_name = os.path.split(file_name)
1805 087b34fe Iustin Pop
  fd, new_name = tempfile.mkstemp('.new', base_name, dir_name)
1806 81b7354c Iustin Pop
  do_remove = True
1807 087b34fe Iustin Pop
  # here we need to make sure we remove the temp file, if any error
1808 087b34fe Iustin Pop
  # leaves it in place
1809 087b34fe Iustin Pop
1810 087b34fe Iustin Pop
    if uid != -1 or gid != -1:
1811 087b34fe Iustin Pop
      os.chown(new_name, uid, gid)
1812 087b34fe Iustin Pop
    if mode:
1813 087b34fe Iustin Pop
      os.chmod(new_name, mode)
1814 71714516 Michael Hanselmann
    if callable(prewrite):
1815 71714516 Michael Hanselmann
1816 087b34fe Iustin Pop
    if data is not None:
1817 087b34fe Iustin Pop
      os.write(fd, data)
1818 087b34fe Iustin Pop
1819 087b34fe Iustin Pop
1820 71714516 Michael Hanselmann
    if callable(postwrite):
1821 71714516 Michael Hanselmann
1822 087b34fe Iustin Pop
1823 087b34fe Iustin Pop
    if atime is not None and mtime is not None:
1824 087b34fe Iustin Pop
      os.utime(new_name, (atime, mtime))
1825 70f4497c Michael Hanselmann
    if not dry_run:
1826 70f4497c Michael Hanselmann
      os.rename(new_name, file_name)
1827 81b7354c Iustin Pop
      do_remove = False
1828 087b34fe Iustin Pop
1829 71714516 Michael Hanselmann
    if close:
1830 71714516 Michael Hanselmann
1831 71714516 Michael Hanselmann
      result = None
1832 71714516 Michael Hanselmann
1833 71714516 Michael Hanselmann
      result = fd
1834 81b7354c Iustin Pop
    if do_remove:
1835 81b7354c Iustin Pop
1836 78feb6fb Guido Trotter
1837 71714516 Michael Hanselmann
  return result
1838 71714516 Michael Hanselmann
1839 78feb6fb Guido Trotter
1840 e587b46a Guido Trotter
def ReadOneLineFile(file_name, strict=False):
1841 e587b46a Guido Trotter
  """Return the first non-empty line from a file.
1842 e587b46a Guido Trotter

1843 e587b46a Guido Trotter
  @type strict: boolean
1844 e587b46a Guido Trotter
  @param strict: if True, abort if the file has more than one
1845 e587b46a Guido Trotter
      non-empty line
1846 e587b46a Guido Trotter

1847 e587b46a Guido Trotter
1848 e587b46a Guido Trotter
  file_lines = ReadFile(file_name).splitlines()
1849 e587b46a Guido Trotter
  full_lines = filter(bool, file_lines)
1850 e587b46a Guido Trotter
  if not file_lines or not full_lines:
1851 e587b46a Guido Trotter
    raise errors.GenericError("No data in one-liner file %s" % file_name)
1852 e587b46a Guido Trotter
  elif strict and len(full_lines) > 1:
1853 e587b46a Guido Trotter
    raise errors.GenericError("Too many lines in one-liner file %s" %
1854 e587b46a Guido Trotter
1855 e587b46a Guido Trotter
  return full_lines[0]
1856 e587b46a Guido Trotter
1857 e587b46a Guido Trotter
1858 7b4126b7 Iustin Pop
def FirstFree(seq, base=0):
1859 7b4126b7 Iustin Pop
  """Returns the first non-existing integer from seq.
1860 7b4126b7 Iustin Pop

1861 7b4126b7 Iustin Pop
  The seq argument should be a sorted list of positive integers. The
1862 7b4126b7 Iustin Pop
  first time the index of an element is smaller than the element
1863 7b4126b7 Iustin Pop
  value, the index will be returned.
1864 7b4126b7 Iustin Pop

1865 7b4126b7 Iustin Pop
  The base argument is used to start at a different offset,
1866 58885d79 Iustin Pop
  i.e. C{[3, 4, 6]} with I{offset=3} will return 5.
1867 58885d79 Iustin Pop

1868 58885d79 Iustin Pop
  Example: C{[0, 1, 3]} will return I{2}.
1869 7b4126b7 Iustin Pop

1870 58885d79 Iustin Pop
  @type seq: sequence
1871 58885d79 Iustin Pop
  @param seq: the sequence to be analyzed.
1872 58885d79 Iustin Pop
  @type base: int
1873 58885d79 Iustin Pop
  @param base: use this value as the base index of the sequence
1874 58885d79 Iustin Pop
  @rtype: int
1875 58885d79 Iustin Pop
  @return: the first non-used index in the sequence
1876 7b4126b7 Iustin Pop

1877 7b4126b7 Iustin Pop
1878 7b4126b7 Iustin Pop
  for idx, elem in enumerate(seq):
1879 7b4126b7 Iustin Pop
    assert elem >= base, "Passed element is higher than base offset"
1880 7b4126b7 Iustin Pop
    if elem > idx + base:
1881 7b4126b7 Iustin Pop
      # idx is not used
1882 7b4126b7 Iustin Pop
      return idx + base
1883 7b4126b7 Iustin Pop
  return None
1884 7b4126b7 Iustin Pop
1885 7b4126b7 Iustin Pop
1886 dfdc4060 Guido Trotter
def SingleWaitForFdCondition(fdobj, event, timeout):
1887 dcd511c8 Guido Trotter
  """Waits for a condition to occur on the socket.
1888 dcd511c8 Guido Trotter

1889 dfdc4060 Guido Trotter
  Immediately returns at the first interruption.
1890 dfdc4060 Guido Trotter

1891 dfdc4060 Guido Trotter
  @type fdobj: integer or object supporting a fileno() method
1892 dfdc4060 Guido Trotter
  @param fdobj: entity to wait for events on
1893 dfdc4060 Guido Trotter
  @type event: integer
1894 dcd511c8 Guido Trotter
  @param event: ORed condition (see select module)
1895 dcd511c8 Guido Trotter
  @type timeout: float or None
1896 dcd511c8 Guido Trotter
  @param timeout: Timeout in seconds
1897 dcd511c8 Guido Trotter
  @rtype: int or None
1898 dcd511c8 Guido Trotter
  @return: None for timeout, otherwise occured conditions
1899 dcd511c8 Guido Trotter

1900 dcd511c8 Guido Trotter
1901 dcd511c8 Guido Trotter
  check = (event | select.POLLPRI |
1902 dcd511c8 Guido Trotter
           select.POLLNVAL | select.POLLHUP | select.POLLERR)
1903 dcd511c8 Guido Trotter
1904 dcd511c8 Guido Trotter
  if timeout is not None:
1905 dcd511c8 Guido Trotter
    # Poller object expects milliseconds
1906 dcd511c8 Guido Trotter
    timeout *= 1000
1907 dcd511c8 Guido Trotter
1908 dcd511c8 Guido Trotter
  poller = select.poll()
1909 dfdc4060 Guido Trotter
  poller.register(fdobj, event)
1910 dcd511c8 Guido Trotter
1911 dfdc4060 Guido Trotter
    # TODO: If the main thread receives a signal and we have no timeout, we
1912 dfdc4060 Guido Trotter
    # could wait forever. This should check a global "quit" flag or something
1913 dfdc4060 Guido Trotter
    # every so often.
1914 dfdc4060 Guido Trotter
    io_events = poller.poll(timeout)
1915 dfdc4060 Guido Trotter
  except select.error, err:
1916 dfdc4060 Guido Trotter
    if err[0] != errno.EINTR:
1917 dfdc4060 Guido Trotter
1918 dfdc4060 Guido Trotter
    io_events = []
1919 dfdc4060 Guido Trotter
  if io_events and io_events[0][1] & check:
1920 dfdc4060 Guido Trotter
    return io_events[0][1]
1921 dfdc4060 Guido Trotter
1922 dfdc4060 Guido Trotter
    return None
1923 dfdc4060 Guido Trotter
1924 dfdc4060 Guido Trotter
1925 dfdc4060 Guido Trotter
class FdConditionWaiterHelper(object):
1926 dfdc4060 Guido Trotter
  """Retry helper for WaitForFdCondition.
1927 dfdc4060 Guido Trotter

1928 dfdc4060 Guido Trotter
  This class contains the retried and wait functions that make sure
1929 dfdc4060 Guido Trotter
  WaitForFdCondition can continue waiting until the timeout is actually
1930 dfdc4060 Guido Trotter
1931 dfdc4060 Guido Trotter

1932 dfdc4060 Guido Trotter
1933 dfdc4060 Guido Trotter
1934 dfdc4060 Guido Trotter
  def __init__(self, timeout):
1935 dfdc4060 Guido Trotter
    self.timeout = timeout
1936 dfdc4060 Guido Trotter
1937 dfdc4060 Guido Trotter
  def Poll(self, fdobj, event):
1938 dfdc4060 Guido Trotter
    result = SingleWaitForFdCondition(fdobj, event, self.timeout)
1939 dfdc4060 Guido Trotter
    if result is None:
1940 dfdc4060 Guido Trotter
      raise RetryAgain()
1941 dfdc4060 Guido Trotter
1942 dfdc4060 Guido Trotter
      return result
1943 dfdc4060 Guido Trotter
1944 dfdc4060 Guido Trotter
  def UpdateTimeout(self, timeout):
1945 dfdc4060 Guido Trotter
    self.timeout = timeout
1946 dfdc4060 Guido Trotter
1947 dfdc4060 Guido Trotter
1948 dfdc4060 Guido Trotter
def WaitForFdCondition(fdobj, event, timeout):
1949 dfdc4060 Guido Trotter
  """Waits for a condition to occur on the socket.
1950 dfdc4060 Guido Trotter

1951 dfdc4060 Guido Trotter
  Retries until the timeout is expired, even if interrupted.
1952 dfdc4060 Guido Trotter

1953 dfdc4060 Guido Trotter
  @type fdobj: integer or object supporting a fileno() method
1954 dfdc4060 Guido Trotter
  @param fdobj: entity to wait for events on
1955 dfdc4060 Guido Trotter
  @type event: integer
1956 dfdc4060 Guido Trotter
  @param event: ORed condition (see select module)
1957 dfdc4060 Guido Trotter
  @type timeout: float or None
1958 dfdc4060 Guido Trotter
  @param timeout: Timeout in seconds
1959 dfdc4060 Guido Trotter
  @rtype: int or None
1960 dfdc4060 Guido Trotter
  @return: None for timeout, otherwise occured conditions
1961 dfdc4060 Guido Trotter

1962 dfdc4060 Guido Trotter
1963 dfdc4060 Guido Trotter
  if timeout is not None:
1964 dfdc4060 Guido Trotter
    retrywaiter = FdConditionWaiterHelper(timeout)
1965 1b429e2a Iustin Pop
1966 1b429e2a Iustin Pop
      result = Retry(retrywaiter.Poll, RETRY_REMAINING_TIME, timeout,
1967 1b429e2a Iustin Pop
                     args=(fdobj, event), wait_fn=retrywaiter.UpdateTimeout)
1968 1b429e2a Iustin Pop
    except RetryTimeout:
1969 1b429e2a Iustin Pop
      result = None
1970 dfdc4060 Guido Trotter
1971 dfdc4060 Guido Trotter
    result = None
1972 dfdc4060 Guido Trotter
    while result is None:
1973 dfdc4060 Guido Trotter
      result = SingleWaitForFdCondition(fdobj, event, timeout)
1974 dfdc4060 Guido Trotter
  return result
1975 2de64672 Iustin Pop
1976 2de64672 Iustin Pop
1977 f7414041 Michael Hanselmann
def UniqueSequence(seq):
1978 f7414041 Michael Hanselmann
  """Returns a list with unique elements.
1979 f7414041 Michael Hanselmann

1980 f7414041 Michael Hanselmann
  Element order is preserved.
1981 58885d79 Iustin Pop

1982 58885d79 Iustin Pop
  @type seq: sequence
1983 5bbd3f7f Michael Hanselmann
  @param seq: the sequence with the source elements
1984 58885d79 Iustin Pop
  @rtype: list
1985 58885d79 Iustin Pop
  @return: list of unique elements from seq
1986 58885d79 Iustin Pop

1987 f7414041 Michael Hanselmann
1988 f7414041 Michael Hanselmann
  seen = set()
1989 f7414041 Michael Hanselmann
  return [i for i in seq if i not in seen and not seen.add(i)]
1990 1862d460 Alexander Schreiber
1991 1862d460 Alexander Schreiber
1992 82187135 René Nussbaumer
def NormalizeAndValidateMac(mac):
1993 82187135 René Nussbaumer
  """Normalizes and check if a MAC address is valid.
1994 1862d460 Alexander Schreiber

1995 5bbd3f7f Michael Hanselmann
  Checks whether the supplied MAC address is formally correct, only
1996 82187135 René Nussbaumer
  accepts colon separated format. Normalize it to all lower.
1997 58885d79 Iustin Pop

1998 58885d79 Iustin Pop
  @type mac: str
1999 58885d79 Iustin Pop
  @param mac: the MAC to be validated
2000 82187135 René Nussbaumer
  @rtype: str
2001 82187135 René Nussbaumer
  @return: returns the normalized and validated MAC.
2002 82187135 René Nussbaumer

2003 82187135 René Nussbaumer
  @raise errors.OpPrereqError: If the MAC isn't valid
2004 58885d79 Iustin Pop

2005 1862d460 Alexander Schreiber
2006 82187135 René Nussbaumer
  mac_check = re.compile("^([0-9a-f]{2}(:|$)){6}$", re.I)
2007 82187135 René Nussbaumer
  if not mac_check.match(mac):
2008 82187135 René Nussbaumer
    raise errors.OpPrereqError("Invalid MAC address specified: %s" %
2009 82187135 René Nussbaumer
                               mac, errors.ECODE_INVAL)
2010 82187135 René Nussbaumer
2011 82187135 René Nussbaumer
  return mac.lower()
2012 06009e27 Iustin Pop
2013 06009e27 Iustin Pop
2014 06009e27 Iustin Pop
def TestDelay(duration):
2015 06009e27 Iustin Pop
  """Sleep for a fixed amount of time.
2016 06009e27 Iustin Pop

2017 58885d79 Iustin Pop
  @type duration: float
2018 58885d79 Iustin Pop
  @param duration: the sleep duration
2019 58885d79 Iustin Pop
  @rtype: boolean
2020 58885d79 Iustin Pop
  @return: False for negative value, True otherwise
2021 58885d79 Iustin Pop

2022 06009e27 Iustin Pop
2023 06009e27 Iustin Pop
  if duration < 0:
2024 38ea42a1 Iustin Pop
    return False, "Invalid sleep duration"
2025 06009e27 Iustin Pop
2026 38ea42a1 Iustin Pop
  return True, None
2027 8f765069 Iustin Pop
2028 8f765069 Iustin Pop
2029 7d88772a Iustin Pop
def _CloseFDNoErr(fd, retries=5):
2030 7d88772a Iustin Pop
  """Close a file descriptor ignoring errors.
2031 8f765069 Iustin Pop

2032 7d88772a Iustin Pop
  @type fd: int
2033 7d88772a Iustin Pop
  @param fd: the file descriptor
2034 7d88772a Iustin Pop
  @type retries: int
2035 7d88772a Iustin Pop
  @param retries: how many retries to make, in case we get any
2036 7d88772a Iustin Pop
      other error than EBADF
2037 7d88772a Iustin Pop

2038 7d88772a Iustin Pop
2039 7d88772a Iustin Pop
2040 7d88772a Iustin Pop
2041 7d88772a Iustin Pop
  except OSError, err:
2042 7d88772a Iustin Pop
    if err.errno != errno.EBADF:
2043 7d88772a Iustin Pop
      if retries > 0:
2044 7d88772a Iustin Pop
        _CloseFDNoErr(fd, retries - 1)
2045 7d88772a Iustin Pop
    # else either it's closed already or we're out of retries, so we
2046 7d88772a Iustin Pop
    # ignore this and go on
2047 7d88772a Iustin Pop
2048 7d88772a Iustin Pop
2049 7d88772a Iustin Pop
def CloseFDs(noclose_fds=None):
2050 7d88772a Iustin Pop
  """Close file descriptors.
2051 7d88772a Iustin Pop

2052 7d88772a Iustin Pop
  This closes all file descriptors above 2 (i.e. except
2053 7d88772a Iustin Pop
2054 8f765069 Iustin Pop

2055 58885d79 Iustin Pop
  @type noclose_fds: list or None
2056 58885d79 Iustin Pop
  @param noclose_fds: if given, it denotes a list of file descriptor
2057 58885d79 Iustin Pop
      that should not be closed
2058 58885d79 Iustin Pop

2059 8f765069 Iustin Pop
2060 8f765069 Iustin Pop
  # Default maximum for the number of available file descriptors.
2061 8f765069 Iustin Pop
  if 'SC_OPEN_MAX' in os.sysconf_names:
2062 8f765069 Iustin Pop
2063 8f765069 Iustin Pop
      MAXFD = os.sysconf('SC_OPEN_MAX')
2064 8f765069 Iustin Pop
      if MAXFD < 0:
2065 8f765069 Iustin Pop
        MAXFD = 1024
2066 8f765069 Iustin Pop
    except OSError:
2067 8f765069 Iustin Pop
      MAXFD = 1024
2068 8f765069 Iustin Pop
2069 8f765069 Iustin Pop
    MAXFD = 1024
2070 7d88772a Iustin Pop
  maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1]
2071 7d88772a Iustin Pop
  if (maxfd == resource.RLIM_INFINITY):
2072 7d88772a Iustin Pop
    maxfd = MAXFD
2073 7d88772a Iustin Pop
2074 7d88772a Iustin Pop
  # Iterate through and close all file descriptors (except the standard ones)
2075 7d88772a Iustin Pop
  for fd in range(3, maxfd):
2076 7d88772a Iustin Pop
    if noclose_fds and fd in noclose_fds:
2077 7d88772a Iustin Pop
2078 7d88772a Iustin Pop
2079 7d88772a Iustin Pop
2080 7d88772a Iustin Pop
2081 4c32a8bd Luca Bigliardi
def Mlockall(_ctypes=ctypes):
2082 4b6fa0bf Luca Bigliardi
  """Lock current process' virtual address space into RAM.
2083 4b6fa0bf Luca Bigliardi

2084 4b6fa0bf Luca Bigliardi
  This is equivalent to the C call mlockall(MCL_CURRENT|MCL_FUTURE),
2085 4b6fa0bf Luca Bigliardi
  see mlock(2) for more details. This function requires ctypes module.
2086 4b6fa0bf Luca Bigliardi

2087 4c32a8bd Luca Bigliardi
  @raises errors.NoCtypesError: if ctypes module is not found
2088 4c32a8bd Luca Bigliardi

2089 4b6fa0bf Luca Bigliardi
2090 4c32a8bd Luca Bigliardi
  if _ctypes is None: