Statistics
| Branch: | Tag: | Revision:

root / daemons / import-export @ 8e2ed2e8

History | View | Annotate | Download (19.6 kB)

1 2d76b580 Michael Hanselmann
#!/usr/bin/python
2 2d76b580 Michael Hanselmann
#
3 2d76b580 Michael Hanselmann
4 2d76b580 Michael Hanselmann
# Copyright (C) 2010 Google Inc.
5 2d76b580 Michael Hanselmann
#
6 2d76b580 Michael Hanselmann
# This program is free software; you can redistribute it and/or modify
7 2d76b580 Michael Hanselmann
# it under the terms of the GNU General Public License as published by
8 2d76b580 Michael Hanselmann
# the Free Software Foundation; either version 2 of the License, or
9 2d76b580 Michael Hanselmann
# (at your option) any later version.
10 2d76b580 Michael Hanselmann
#
11 2d76b580 Michael Hanselmann
# This program is distributed in the hope that it will be useful, but
12 2d76b580 Michael Hanselmann
# WITHOUT ANY WARRANTY; without even the implied warranty of
13 2d76b580 Michael Hanselmann
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14 2d76b580 Michael Hanselmann
# General Public License for more details.
15 2d76b580 Michael Hanselmann
#
16 2d76b580 Michael Hanselmann
# You should have received a copy of the GNU General Public License
17 2d76b580 Michael Hanselmann
# along with this program; if not, write to the Free Software
18 2d76b580 Michael Hanselmann
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
19 2d76b580 Michael Hanselmann
# 02110-1301, USA.
20 2d76b580 Michael Hanselmann
21 2d76b580 Michael Hanselmann
22 2d76b580 Michael Hanselmann
"""Import/export daemon.
23 2d76b580 Michael Hanselmann
24 2d76b580 Michael Hanselmann
"""
25 2d76b580 Michael Hanselmann
26 b459a848 Andrea Spadaccini
# pylint: disable=C0103
27 2d76b580 Michael Hanselmann
# C0103: Invalid name import-export
28 2d76b580 Michael Hanselmann
29 2d76b580 Michael Hanselmann
import errno
30 2d76b580 Michael Hanselmann
import logging
31 2d76b580 Michael Hanselmann
import optparse
32 2d76b580 Michael Hanselmann
import os
33 2d76b580 Michael Hanselmann
import select
34 2d76b580 Michael Hanselmann
import signal
35 2d76b580 Michael Hanselmann
import subprocess
36 2d76b580 Michael Hanselmann
import sys
37 2d76b580 Michael Hanselmann
import time
38 c08d76f5 Michael Hanselmann
import math
39 2d76b580 Michael Hanselmann
40 2d76b580 Michael Hanselmann
from ganeti import constants
41 2d76b580 Michael Hanselmann
from ganeti import cli
42 2d76b580 Michael Hanselmann
from ganeti import utils
43 acd65a16 Michael Hanselmann
from ganeti import errors
44 2d76b580 Michael Hanselmann
from ganeti import serializer
45 2d76b580 Michael Hanselmann
from ganeti import objects
46 bb44b1ae Michael Hanselmann
from ganeti import impexpd
47 a744b676 Manuel Franceschini
from ganeti import netutils
48 2d76b580 Michael Hanselmann
49 2d76b580 Michael Hanselmann
50 2d76b580 Michael Hanselmann
#: How many lines to keep in the status file
51 2d76b580 Michael Hanselmann
MAX_RECENT_OUTPUT_LINES = 20
52 2d76b580 Michael Hanselmann
53 2d76b580 Michael Hanselmann
#: Don't update status file more than once every 5 seconds (unless forced)
54 2d76b580 Michael Hanselmann
MIN_UPDATE_INTERVAL = 5.0
55 2d76b580 Michael Hanselmann
56 043f2292 Michael Hanselmann
#: How long to wait for a connection to be established
57 043f2292 Michael Hanselmann
DEFAULT_CONNECT_TIMEOUT = 60
58 043f2292 Michael Hanselmann
59 c08d76f5 Michael Hanselmann
#: Get dd(1) statistics every few seconds
60 c08d76f5 Michael Hanselmann
DD_STATISTICS_INTERVAL = 5.0
61 c08d76f5 Michael Hanselmann
62 c08d76f5 Michael Hanselmann
#: Seconds for throughput calculation
63 c08d76f5 Michael Hanselmann
DD_THROUGHPUT_INTERVAL = 60.0
64 c08d76f5 Michael Hanselmann
65 c08d76f5 Michael Hanselmann
#: Number of samples for throughput calculation
66 c08d76f5 Michael Hanselmann
DD_THROUGHPUT_SAMPLES = int(math.ceil(float(DD_THROUGHPUT_INTERVAL) /
67 c08d76f5 Michael Hanselmann
                                      DD_STATISTICS_INTERVAL))
68 c08d76f5 Michael Hanselmann
69 2d76b580 Michael Hanselmann
70 2d76b580 Michael Hanselmann
# Global variable for options
71 2d76b580 Michael Hanselmann
options = None
72 2d76b580 Michael Hanselmann
73 2d76b580 Michael Hanselmann
74 2d76b580 Michael Hanselmann
def SetupLogging():
75 2d76b580 Michael Hanselmann
  """Configures the logging module.
76 2d76b580 Michael Hanselmann
77 2d76b580 Michael Hanselmann
  """
78 2d76b580 Michael Hanselmann
  formatter = logging.Formatter("%(asctime)s: %(message)s")
79 2d76b580 Michael Hanselmann
80 2d76b580 Michael Hanselmann
  stderr_handler = logging.StreamHandler()
81 2d76b580 Michael Hanselmann
  stderr_handler.setFormatter(formatter)
82 2d76b580 Michael Hanselmann
  stderr_handler.setLevel(logging.NOTSET)
83 2d76b580 Michael Hanselmann
84 2d76b580 Michael Hanselmann
  root_logger = logging.getLogger("")
85 2d76b580 Michael Hanselmann
  root_logger.addHandler(stderr_handler)
86 2d76b580 Michael Hanselmann
87 2d76b580 Michael Hanselmann
  if options.debug:
88 2d76b580 Michael Hanselmann
    root_logger.setLevel(logging.NOTSET)
89 2d76b580 Michael Hanselmann
  elif options.verbose:
90 2d76b580 Michael Hanselmann
    root_logger.setLevel(logging.INFO)
91 2d76b580 Michael Hanselmann
  else:
92 2d76b580 Michael Hanselmann
    root_logger.setLevel(logging.ERROR)
93 2d76b580 Michael Hanselmann
94 2d76b580 Michael Hanselmann
  # Create special logger for child process output
95 2d76b580 Michael Hanselmann
  child_logger = logging.Logger("child output")
96 2d76b580 Michael Hanselmann
  child_logger.addHandler(stderr_handler)
97 2d76b580 Michael Hanselmann
  child_logger.setLevel(logging.NOTSET)
98 2d76b580 Michael Hanselmann
99 2d76b580 Michael Hanselmann
  return child_logger
100 2d76b580 Michael Hanselmann
101 2d76b580 Michael Hanselmann
102 2d76b580 Michael Hanselmann
class StatusFile:
103 2d76b580 Michael Hanselmann
  """Status file manager.
104 2d76b580 Michael Hanselmann
105 2d76b580 Michael Hanselmann
  """
106 2d76b580 Michael Hanselmann
  def __init__(self, path):
107 2d76b580 Michael Hanselmann
    """Initializes class.
108 2d76b580 Michael Hanselmann
109 2d76b580 Michael Hanselmann
    """
110 2d76b580 Michael Hanselmann
    self._path = path
111 2d76b580 Michael Hanselmann
    self._data = objects.ImportExportStatus(ctime=time.time(),
112 2d76b580 Michael Hanselmann
                                            mtime=None,
113 2d76b580 Michael Hanselmann
                                            recent_output=[])
114 2d76b580 Michael Hanselmann
115 2d76b580 Michael Hanselmann
  def AddRecentOutput(self, line):
116 2d76b580 Michael Hanselmann
    """Adds a new line of recent output.
117 2d76b580 Michael Hanselmann
118 2d76b580 Michael Hanselmann
    """
119 2d76b580 Michael Hanselmann
    self._data.recent_output.append(line)
120 2d76b580 Michael Hanselmann
121 2d76b580 Michael Hanselmann
    # Remove old lines
122 2d76b580 Michael Hanselmann
    del self._data.recent_output[:-MAX_RECENT_OUTPUT_LINES]
123 2d76b580 Michael Hanselmann
124 2d76b580 Michael Hanselmann
  def SetListenPort(self, port):
125 2d76b580 Michael Hanselmann
    """Sets the port the daemon is listening on.
126 2d76b580 Michael Hanselmann
127 2d76b580 Michael Hanselmann
    @type port: int
128 2d76b580 Michael Hanselmann
    @param port: TCP/UDP port
129 2d76b580 Michael Hanselmann
130 2d76b580 Michael Hanselmann
    """
131 e687ec01 Michael Hanselmann
    assert isinstance(port, (int, long)) and 0 < port < (2 ** 16)
132 2d76b580 Michael Hanselmann
    self._data.listen_port = port
133 2d76b580 Michael Hanselmann
134 2d76b580 Michael Hanselmann
  def GetListenPort(self):
135 2d76b580 Michael Hanselmann
    """Returns the port the daemon is listening on.
136 2d76b580 Michael Hanselmann
137 2d76b580 Michael Hanselmann
    """
138 2d76b580 Michael Hanselmann
    return self._data.listen_port
139 2d76b580 Michael Hanselmann
140 2d76b580 Michael Hanselmann
  def SetConnected(self):
141 2d76b580 Michael Hanselmann
    """Sets the connected flag.
142 2d76b580 Michael Hanselmann
143 2d76b580 Michael Hanselmann
    """
144 2d76b580 Michael Hanselmann
    self._data.connected = True
145 2d76b580 Michael Hanselmann
146 043f2292 Michael Hanselmann
  def GetConnected(self):
147 043f2292 Michael Hanselmann
    """Determines whether the daemon is connected.
148 043f2292 Michael Hanselmann
149 043f2292 Michael Hanselmann
    """
150 043f2292 Michael Hanselmann
    return self._data.connected
151 043f2292 Michael Hanselmann
152 c08d76f5 Michael Hanselmann
  def SetProgress(self, mbytes, throughput, percent, eta):
153 c08d76f5 Michael Hanselmann
    """Sets how much data has been transferred so far.
154 c08d76f5 Michael Hanselmann
155 c08d76f5 Michael Hanselmann
    @type mbytes: number
156 c08d76f5 Michael Hanselmann
    @param mbytes: Transferred amount of data in MiB.
157 c08d76f5 Michael Hanselmann
    @type throughput: float
158 c08d76f5 Michael Hanselmann
    @param throughput: MiB/second
159 c08d76f5 Michael Hanselmann
    @type percent: number
160 c08d76f5 Michael Hanselmann
    @param percent: Percent processed
161 c08d76f5 Michael Hanselmann
    @type eta: number
162 c08d76f5 Michael Hanselmann
    @param eta: Expected number of seconds until done
163 c08d76f5 Michael Hanselmann
164 c08d76f5 Michael Hanselmann
    """
165 c08d76f5 Michael Hanselmann
    self._data.progress_mbytes = mbytes
166 c08d76f5 Michael Hanselmann
    self._data.progress_throughput = throughput
167 c08d76f5 Michael Hanselmann
    self._data.progress_percent = percent
168 c08d76f5 Michael Hanselmann
    self._data.progress_eta = eta
169 c08d76f5 Michael Hanselmann
170 2d76b580 Michael Hanselmann
  def SetExitStatus(self, exit_status, error_message):
171 2d76b580 Michael Hanselmann
    """Sets the exit status and an error message.
172 2d76b580 Michael Hanselmann
173 2d76b580 Michael Hanselmann
    """
174 2d76b580 Michael Hanselmann
    # Require error message when status isn't 0
175 2d76b580 Michael Hanselmann
    assert exit_status == 0 or error_message
176 2d76b580 Michael Hanselmann
177 2d76b580 Michael Hanselmann
    self._data.exit_status = exit_status
178 2d76b580 Michael Hanselmann
    self._data.error_message = error_message
179 2d76b580 Michael Hanselmann
180 2d76b580 Michael Hanselmann
  def ExitStatusIsSuccess(self):
181 2d76b580 Michael Hanselmann
    """Returns whether the exit status means "success".
182 2d76b580 Michael Hanselmann
183 2d76b580 Michael Hanselmann
    """
184 2d76b580 Michael Hanselmann
    return not bool(self._data.error_message)
185 2d76b580 Michael Hanselmann
186 2d76b580 Michael Hanselmann
  def Update(self, force):
187 2d76b580 Michael Hanselmann
    """Updates the status file.
188 2d76b580 Michael Hanselmann
189 2d76b580 Michael Hanselmann
    @type force: bool
190 2d76b580 Michael Hanselmann
    @param force: Write status file in any case, not only when minimum interval
191 2d76b580 Michael Hanselmann
                  is expired
192 2d76b580 Michael Hanselmann
193 2d76b580 Michael Hanselmann
    """
194 2d76b580 Michael Hanselmann
    if not (force or
195 2d76b580 Michael Hanselmann
            self._data.mtime is None or
196 2d76b580 Michael Hanselmann
            time.time() > (self._data.mtime + MIN_UPDATE_INTERVAL)):
197 2d76b580 Michael Hanselmann
      return
198 2d76b580 Michael Hanselmann
199 2d76b580 Michael Hanselmann
    logging.debug("Updating status file %s", self._path)
200 2d76b580 Michael Hanselmann
201 2d76b580 Michael Hanselmann
    self._data.mtime = time.time()
202 2d76b580 Michael Hanselmann
    utils.WriteFile(self._path,
203 2d76b580 Michael Hanselmann
                    data=serializer.DumpJson(self._data.ToDict(), indent=True),
204 2d76b580 Michael Hanselmann
                    mode=0400)
205 2d76b580 Michael Hanselmann
206 2d76b580 Michael Hanselmann
207 c08d76f5 Michael Hanselmann
def ProcessChildIO(child, socat_stderr_read_fd, dd_stderr_read_fd,
208 f9323011 Michael Hanselmann
                   dd_pid_read_fd, exp_size_read_fd, status_file, child_logger,
209 043f2292 Michael Hanselmann
                   signal_notify, signal_handler, mode):
210 2d76b580 Michael Hanselmann
  """Handles the child processes' output.
211 2d76b580 Michael Hanselmann
212 2d76b580 Michael Hanselmann
  """
213 29da446a Michael Hanselmann
  assert not (signal_handler.signum - set([signal.SIGTERM, signal.SIGINT])), \
214 29da446a Michael Hanselmann
         "Other signals are not handled in this function"
215 29da446a Michael Hanselmann
216 29da446a Michael Hanselmann
  # Buffer size 0 is important, otherwise .read() with a specified length
217 29da446a Michael Hanselmann
  # might buffer data while poll(2) won't mark its file descriptor as
218 29da446a Michael Hanselmann
  # readable again.
219 29da446a Michael Hanselmann
  socat_stderr_read = os.fdopen(socat_stderr_read_fd, "r", 0)
220 c08d76f5 Michael Hanselmann
  dd_stderr_read = os.fdopen(dd_stderr_read_fd, "r", 0)
221 c08d76f5 Michael Hanselmann
  dd_pid_read = os.fdopen(dd_pid_read_fd, "r", 0)
222 f9323011 Michael Hanselmann
  exp_size_read = os.fdopen(exp_size_read_fd, "r", 0)
223 c08d76f5 Michael Hanselmann
224 c08d76f5 Michael Hanselmann
  tp_samples = DD_THROUGHPUT_SAMPLES
225 2d76b580 Michael Hanselmann
226 f9323011 Michael Hanselmann
  if options.exp_size == constants.IE_CUSTOM_SIZE:
227 f9323011 Michael Hanselmann
    exp_size = None
228 f9323011 Michael Hanselmann
  else:
229 f9323011 Michael Hanselmann
    exp_size = options.exp_size
230 f9323011 Michael Hanselmann
231 34c9ee7b Michael Hanselmann
  child_io_proc = impexpd.ChildIOProcessor(options.debug, status_file,
232 f9323011 Michael Hanselmann
                                           child_logger, tp_samples,
233 f9323011 Michael Hanselmann
                                           exp_size)
234 2d76b580 Michael Hanselmann
  try:
235 34c9ee7b Michael Hanselmann
    fdmap = {
236 34c9ee7b Michael Hanselmann
      child.stderr.fileno():
237 34c9ee7b Michael Hanselmann
        (child.stderr, child_io_proc.GetLineSplitter(impexpd.PROG_OTHER)),
238 34c9ee7b Michael Hanselmann
      socat_stderr_read.fileno():
239 34c9ee7b Michael Hanselmann
        (socat_stderr_read, child_io_proc.GetLineSplitter(impexpd.PROG_SOCAT)),
240 c08d76f5 Michael Hanselmann
      dd_pid_read.fileno():
241 c08d76f5 Michael Hanselmann
        (dd_pid_read, child_io_proc.GetLineSplitter(impexpd.PROG_DD_PID)),
242 c08d76f5 Michael Hanselmann
      dd_stderr_read.fileno():
243 c08d76f5 Michael Hanselmann
        (dd_stderr_read, child_io_proc.GetLineSplitter(impexpd.PROG_DD)),
244 f9323011 Michael Hanselmann
      exp_size_read.fileno():
245 f9323011 Michael Hanselmann
        (exp_size_read, child_io_proc.GetLineSplitter(impexpd.PROG_EXP_SIZE)),
246 34c9ee7b Michael Hanselmann
      signal_notify.fileno(): (signal_notify, None),
247 34c9ee7b Michael Hanselmann
      }
248 34c9ee7b Michael Hanselmann
249 34c9ee7b Michael Hanselmann
    poller = select.poll()
250 34c9ee7b Michael Hanselmann
    for fd in fdmap:
251 34c9ee7b Michael Hanselmann
      utils.SetNonblockFlag(fd, True)
252 34c9ee7b Michael Hanselmann
      poller.register(fd, select.POLLIN)
253 34c9ee7b Michael Hanselmann
254 34c9ee7b Michael Hanselmann
    if options.connect_timeout and mode == constants.IEM_IMPORT:
255 557838c1 René Nussbaumer
      listen_timeout = utils.RunningTimeout(options.connect_timeout, True)
256 34c9ee7b Michael Hanselmann
    else:
257 34c9ee7b Michael Hanselmann
      listen_timeout = None
258 34c9ee7b Michael Hanselmann
259 34c9ee7b Michael Hanselmann
    exit_timeout = None
260 c08d76f5 Michael Hanselmann
    dd_stats_timeout = None
261 34c9ee7b Michael Hanselmann
262 34c9ee7b Michael Hanselmann
    while True:
263 34c9ee7b Michael Hanselmann
      # Break out of loop if only signal notify FD is left
264 34c9ee7b Michael Hanselmann
      if len(fdmap) == 1 and signal_notify.fileno() in fdmap:
265 34c9ee7b Michael Hanselmann
        break
266 34c9ee7b Michael Hanselmann
267 34c9ee7b Michael Hanselmann
      timeout = None
268 34c9ee7b Michael Hanselmann
269 34c9ee7b Michael Hanselmann
      if listen_timeout and not exit_timeout:
270 bd275a93 Michael Hanselmann
        assert mode == constants.IEM_IMPORT and options.connect_timeout
271 34c9ee7b Michael Hanselmann
        if status_file.GetConnected():
272 34c9ee7b Michael Hanselmann
          listen_timeout = None
273 34c9ee7b Michael Hanselmann
        elif listen_timeout.Remaining() < 0:
274 bd275a93 Michael Hanselmann
          errmsg = ("Child process didn't establish connection in time"
275 bd275a93 Michael Hanselmann
                    " (%0.0fs), sending SIGTERM" % options.connect_timeout)
276 bd275a93 Michael Hanselmann
          logging.error(errmsg)
277 bd275a93 Michael Hanselmann
          status_file.AddRecentOutput(errmsg)
278 bd275a93 Michael Hanselmann
          status_file.Update(True)
279 bd275a93 Michael Hanselmann
280 34c9ee7b Michael Hanselmann
          child.Kill(signal.SIGTERM)
281 34c9ee7b Michael Hanselmann
          exit_timeout = \
282 c74cda62 René Nussbaumer
            utils.RunningTimeout(constants.CHILD_LINGER_TIMEOUT, True)
283 34c9ee7b Michael Hanselmann
          # Next block will calculate timeout
284 34c9ee7b Michael Hanselmann
        else:
285 34c9ee7b Michael Hanselmann
          # Not yet connected, check again in a second
286 34c9ee7b Michael Hanselmann
          timeout = 1000
287 34c9ee7b Michael Hanselmann
288 34c9ee7b Michael Hanselmann
      if exit_timeout:
289 34c9ee7b Michael Hanselmann
        timeout = exit_timeout.Remaining() * 1000
290 34c9ee7b Michael Hanselmann
        if timeout < 0:
291 34c9ee7b Michael Hanselmann
          logging.info("Child process didn't exit in time")
292 2d76b580 Michael Hanselmann
          break
293 2d76b580 Michael Hanselmann
294 c08d76f5 Michael Hanselmann
      if (not dd_stats_timeout) or dd_stats_timeout.Remaining() < 0:
295 c08d76f5 Michael Hanselmann
        notify_status = child_io_proc.NotifyDd()
296 c08d76f5 Michael Hanselmann
        if notify_status:
297 c08d76f5 Michael Hanselmann
          # Schedule next notification
298 557838c1 René Nussbaumer
          dd_stats_timeout = utils.RunningTimeout(DD_STATISTICS_INTERVAL, True)
299 c08d76f5 Michael Hanselmann
        else:
300 c08d76f5 Michael Hanselmann
          # Try again soon (dd isn't ready yet)
301 557838c1 René Nussbaumer
          dd_stats_timeout = utils.RunningTimeout(1.0, True)
302 c08d76f5 Michael Hanselmann
303 c08d76f5 Michael Hanselmann
      if dd_stats_timeout:
304 c08d76f5 Michael Hanselmann
        dd_timeout = max(0, dd_stats_timeout.Remaining() * 1000)
305 c08d76f5 Michael Hanselmann
306 c08d76f5 Michael Hanselmann
        if timeout is None:
307 c08d76f5 Michael Hanselmann
          timeout = dd_timeout
308 c08d76f5 Michael Hanselmann
        else:
309 c08d76f5 Michael Hanselmann
          timeout = min(timeout, dd_timeout)
310 c08d76f5 Michael Hanselmann
311 34c9ee7b Michael Hanselmann
      for fd, event in utils.RetryOnSignal(poller.poll, timeout):
312 34c9ee7b Michael Hanselmann
        if event & (select.POLLIN | event & select.POLLPRI):
313 34c9ee7b Michael Hanselmann
          (from_, to) = fdmap[fd]
314 34c9ee7b Michael Hanselmann
315 34c9ee7b Michael Hanselmann
          # Read up to 1 KB of data
316 34c9ee7b Michael Hanselmann
          data = from_.read(1024)
317 34c9ee7b Michael Hanselmann
          if data:
318 34c9ee7b Michael Hanselmann
            if to:
319 34c9ee7b Michael Hanselmann
              to.write(data)
320 34c9ee7b Michael Hanselmann
            elif fd == signal_notify.fileno():
321 34c9ee7b Michael Hanselmann
              # Signal handling
322 34c9ee7b Michael Hanselmann
              if signal_handler.called:
323 34c9ee7b Michael Hanselmann
                signal_handler.Clear()
324 34c9ee7b Michael Hanselmann
                if exit_timeout:
325 34c9ee7b Michael Hanselmann
                  logging.info("Child process still has about %0.2f seconds"
326 34c9ee7b Michael Hanselmann
                               " to exit", exit_timeout.Remaining())
327 34c9ee7b Michael Hanselmann
                else:
328 34c9ee7b Michael Hanselmann
                  logging.info("Giving child process %0.2f seconds to exit",
329 c74cda62 René Nussbaumer
                               constants.CHILD_LINGER_TIMEOUT)
330 34c9ee7b Michael Hanselmann
                  exit_timeout = \
331 c74cda62 René Nussbaumer
                    utils.RunningTimeout(constants.CHILD_LINGER_TIMEOUT, True)
332 043f2292 Michael Hanselmann
          else:
333 2d76b580 Michael Hanselmann
            poller.unregister(fd)
334 2d76b580 Michael Hanselmann
            del fdmap[fd]
335 2d76b580 Michael Hanselmann
336 34c9ee7b Michael Hanselmann
        elif event & (select.POLLNVAL | select.POLLHUP |
337 34c9ee7b Michael Hanselmann
                      select.POLLERR):
338 34c9ee7b Michael Hanselmann
          poller.unregister(fd)
339 34c9ee7b Michael Hanselmann
          del fdmap[fd]
340 34c9ee7b Michael Hanselmann
341 34c9ee7b Michael Hanselmann
      child_io_proc.FlushAll()
342 2d76b580 Michael Hanselmann
343 34c9ee7b Michael Hanselmann
    # If there was a timeout calculator, we were waiting for the child to
344 34c9ee7b Michael Hanselmann
    # finish, e.g. due to a signal
345 34c9ee7b Michael Hanselmann
    return not bool(exit_timeout)
346 2d76b580 Michael Hanselmann
  finally:
347 34c9ee7b Michael Hanselmann
    child_io_proc.CloseAll()
348 2d76b580 Michael Hanselmann
349 2d76b580 Michael Hanselmann
350 2d76b580 Michael Hanselmann
def ParseOptions():
351 2d76b580 Michael Hanselmann
  """Parses the options passed to the program.
352 2d76b580 Michael Hanselmann
353 2d76b580 Michael Hanselmann
  @return: Arguments to program
354 2d76b580 Michael Hanselmann
355 2d76b580 Michael Hanselmann
  """
356 b459a848 Andrea Spadaccini
  global options # pylint: disable=W0603
357 2d76b580 Michael Hanselmann
358 2d76b580 Michael Hanselmann
  parser = optparse.OptionParser(usage=("%%prog <status-file> {%s|%s}" %
359 2d76b580 Michael Hanselmann
                                        (constants.IEM_IMPORT,
360 2d76b580 Michael Hanselmann
                                         constants.IEM_EXPORT)))
361 2d76b580 Michael Hanselmann
  parser.add_option(cli.DEBUG_OPT)
362 2d76b580 Michael Hanselmann
  parser.add_option(cli.VERBOSE_OPT)
363 2d76b580 Michael Hanselmann
  parser.add_option("--key", dest="key", action="store", type="string",
364 2d76b580 Michael Hanselmann
                    help="RSA key file")
365 2d76b580 Michael Hanselmann
  parser.add_option("--cert", dest="cert", action="store", type="string",
366 2d76b580 Michael Hanselmann
                    help="X509 certificate file")
367 2d76b580 Michael Hanselmann
  parser.add_option("--ca", dest="ca", action="store", type="string",
368 2d76b580 Michael Hanselmann
                    help="X509 CA file")
369 2d76b580 Michael Hanselmann
  parser.add_option("--bind", dest="bind", action="store", type="string",
370 2d76b580 Michael Hanselmann
                    help="Bind address")
371 58bb385c Michael Hanselmann
  parser.add_option("--ipv4", dest="ipv4", action="store_true",
372 58bb385c Michael Hanselmann
                    help="Use IPv4 only")
373 58bb385c Michael Hanselmann
  parser.add_option("--ipv6", dest="ipv6", action="store_true",
374 58bb385c Michael Hanselmann
                    help="Use IPv6 only")
375 2d76b580 Michael Hanselmann
  parser.add_option("--host", dest="host", action="store", type="string",
376 2d76b580 Michael Hanselmann
                    help="Remote hostname")
377 2d76b580 Michael Hanselmann
  parser.add_option("--port", dest="port", action="store", type="int",
378 2d76b580 Michael Hanselmann
                    help="Remote port")
379 043f2292 Michael Hanselmann
  parser.add_option("--connect-retries", dest="connect_retries", action="store",
380 043f2292 Michael Hanselmann
                    type="int", default=0,
381 043f2292 Michael Hanselmann
                    help=("How many times the connection should be retried"
382 043f2292 Michael Hanselmann
                          " (export only)"))
383 043f2292 Michael Hanselmann
  parser.add_option("--connect-timeout", dest="connect_timeout", action="store",
384 043f2292 Michael Hanselmann
                    type="int", default=DEFAULT_CONNECT_TIMEOUT,
385 043f2292 Michael Hanselmann
                    help="Timeout for connection to be established (seconds)")
386 7e3c1da6 Michael Hanselmann
  parser.add_option("--compress", dest="compress", action="store",
387 7e3c1da6 Michael Hanselmann
                    type="choice", help="Compression method",
388 7e3c1da6 Michael Hanselmann
                    metavar="[%s]" % "|".join(constants.IEC_ALL),
389 7e3c1da6 Michael Hanselmann
                    choices=list(constants.IEC_ALL), default=constants.IEC_GZIP)
390 f9323011 Michael Hanselmann
  parser.add_option("--expected-size", dest="exp_size", action="store",
391 f9323011 Michael Hanselmann
                    type="string", default=None,
392 f9323011 Michael Hanselmann
                    help="Expected import/export size (MiB)")
393 1d3dfa29 Michael Hanselmann
  parser.add_option("--magic", dest="magic", action="store",
394 1d3dfa29 Michael Hanselmann
                    type="string", default=None, help="Magic string")
395 2d76b580 Michael Hanselmann
  parser.add_option("--cmd-prefix", dest="cmd_prefix", action="store",
396 2d76b580 Michael Hanselmann
                    type="string", help="Command prefix")
397 2d76b580 Michael Hanselmann
  parser.add_option("--cmd-suffix", dest="cmd_suffix", action="store",
398 2d76b580 Michael Hanselmann
                    type="string", help="Command suffix")
399 2d76b580 Michael Hanselmann
400 2d76b580 Michael Hanselmann
  (options, args) = parser.parse_args()
401 2d76b580 Michael Hanselmann
402 2d76b580 Michael Hanselmann
  if len(args) != 2:
403 2d76b580 Michael Hanselmann
    # Won't return
404 2d76b580 Michael Hanselmann
    parser.error("Expected exactly two arguments")
405 2d76b580 Michael Hanselmann
406 2d76b580 Michael Hanselmann
  (status_file_path, mode) = args
407 2d76b580 Michael Hanselmann
408 2d76b580 Michael Hanselmann
  if mode not in (constants.IEM_IMPORT,
409 2d76b580 Michael Hanselmann
                  constants.IEM_EXPORT):
410 2d76b580 Michael Hanselmann
    # Won't return
411 2d76b580 Michael Hanselmann
    parser.error("Invalid mode: %s" % mode)
412 2d76b580 Michael Hanselmann
413 acd65a16 Michael Hanselmann
  # Normalize and check parameters
414 58bb385c Michael Hanselmann
  if options.host is not None and not netutils.IPAddress.IsValid(options.host):
415 acd65a16 Michael Hanselmann
    try:
416 b705c7a6 Manuel Franceschini
      options.host = netutils.Hostname.GetNormalizedName(options.host)
417 acd65a16 Michael Hanselmann
    except errors.OpPrereqError, err:
418 acd65a16 Michael Hanselmann
      parser.error("Invalid hostname '%s': %s" % (options.host, err))
419 acd65a16 Michael Hanselmann
420 acd65a16 Michael Hanselmann
  if options.port is not None:
421 acd65a16 Michael Hanselmann
    options.port = utils.ValidateServiceName(options.port)
422 acd65a16 Michael Hanselmann
423 f9323011 Michael Hanselmann
  if (options.exp_size is not None and
424 f9323011 Michael Hanselmann
      options.exp_size != constants.IE_CUSTOM_SIZE):
425 f9323011 Michael Hanselmann
    try:
426 f9323011 Michael Hanselmann
      options.exp_size = int(options.exp_size)
427 f9323011 Michael Hanselmann
    except (ValueError, TypeError), err:
428 f9323011 Michael Hanselmann
      # Won't return
429 f9323011 Michael Hanselmann
      parser.error("Invalid value for --expected-size: %s (%s)" %
430 f9323011 Michael Hanselmann
                   (options.exp_size, err))
431 f9323011 Michael Hanselmann
432 1d3dfa29 Michael Hanselmann
  if not (options.magic is None or constants.IE_MAGIC_RE.match(options.magic)):
433 1d3dfa29 Michael Hanselmann
    parser.error("Magic must match regular expression %s" %
434 1d3dfa29 Michael Hanselmann
                 constants.IE_MAGIC_RE.pattern)
435 1d3dfa29 Michael Hanselmann
436 58bb385c Michael Hanselmann
  if options.ipv4 and options.ipv6:
437 58bb385c Michael Hanselmann
    parser.error("Can only use one of --ipv4 and --ipv6")
438 58bb385c Michael Hanselmann
439 2d76b580 Michael Hanselmann
  return (status_file_path, mode)
440 2d76b580 Michael Hanselmann
441 2d76b580 Michael Hanselmann
442 4ca693ca Michael Hanselmann
class ChildProcess(subprocess.Popen):
443 f9323011 Michael Hanselmann
  def __init__(self, env, cmd, noclose_fds):
444 4ca693ca Michael Hanselmann
    """Initializes this class.
445 4ca693ca Michael Hanselmann
446 4ca693ca Michael Hanselmann
    """
447 4ca693ca Michael Hanselmann
    self._noclose_fds = noclose_fds
448 4ca693ca Michael Hanselmann
449 4ca693ca Michael Hanselmann
    # Not using close_fds because doing so would also close the socat stderr
450 4ca693ca Michael Hanselmann
    # pipe, which we still need.
451 f9323011 Michael Hanselmann
    subprocess.Popen.__init__(self, cmd, env=env, shell=False, close_fds=False,
452 4ca693ca Michael Hanselmann
                              stderr=subprocess.PIPE, stdout=None, stdin=None,
453 4ca693ca Michael Hanselmann
                              preexec_fn=self._ChildPreexec)
454 4ca693ca Michael Hanselmann
    self._SetProcessGroup()
455 4ca693ca Michael Hanselmann
456 4ca693ca Michael Hanselmann
  def _ChildPreexec(self):
457 4ca693ca Michael Hanselmann
    """Called before child executable is execve'd.
458 4ca693ca Michael Hanselmann
459 4ca693ca Michael Hanselmann
    """
460 4ca693ca Michael Hanselmann
    # Move to separate process group. By sending a signal to its process group
461 4ca693ca Michael Hanselmann
    # we can kill the child process and all grandchildren.
462 4ca693ca Michael Hanselmann
    os.setpgid(0, 0)
463 4ca693ca Michael Hanselmann
464 4ca693ca Michael Hanselmann
    # Close almost all file descriptors
465 4ca693ca Michael Hanselmann
    utils.CloseFDs(noclose_fds=self._noclose_fds)
466 4ca693ca Michael Hanselmann
467 4ca693ca Michael Hanselmann
  def _SetProcessGroup(self):
468 4ca693ca Michael Hanselmann
    """Sets the child's process group.
469 4ca693ca Michael Hanselmann
470 4ca693ca Michael Hanselmann
    """
471 4ca693ca Michael Hanselmann
    assert self.pid, "Can't be called in child process"
472 4ca693ca Michael Hanselmann
473 4ca693ca Michael Hanselmann
    # Avoid race condition by setting child's process group (as good as
474 4ca693ca Michael Hanselmann
    # possible in Python) before sending signals to child. For an
475 4ca693ca Michael Hanselmann
    # explanation, see preexec function for child.
476 4ca693ca Michael Hanselmann
    try:
477 4ca693ca Michael Hanselmann
      os.setpgid(self.pid, self.pid)
478 4ca693ca Michael Hanselmann
    except EnvironmentError, err:
479 4ca693ca Michael Hanselmann
      # If the child process was faster we receive EPERM or EACCES
480 4ca693ca Michael Hanselmann
      if err.errno not in (errno.EPERM, errno.EACCES):
481 4ca693ca Michael Hanselmann
        raise
482 4ca693ca Michael Hanselmann
483 4ca693ca Michael Hanselmann
  def Kill(self, signum):
484 4ca693ca Michael Hanselmann
    """Sends signal to child process.
485 4ca693ca Michael Hanselmann
486 4ca693ca Michael Hanselmann
    """
487 4ca693ca Michael Hanselmann
    logging.info("Sending signal %s to child process", signum)
488 560cbec1 Michael Hanselmann
    utils.IgnoreProcessNotFound(os.killpg, self.pid, signum)
489 4ca693ca Michael Hanselmann
490 4ca693ca Michael Hanselmann
  def ForceQuit(self):
491 4ca693ca Michael Hanselmann
    """Ensure child process is no longer running.
492 4ca693ca Michael Hanselmann
493 4ca693ca Michael Hanselmann
    """
494 4ca693ca Michael Hanselmann
    # Final check if child process is still alive
495 4ca693ca Michael Hanselmann
    if utils.RetryOnSignal(self.poll) is None:
496 4ca693ca Michael Hanselmann
      logging.error("Child process still alive, sending SIGKILL")
497 4ca693ca Michael Hanselmann
      self.Kill(signal.SIGKILL)
498 4ca693ca Michael Hanselmann
      utils.RetryOnSignal(self.wait)
499 4ca693ca Michael Hanselmann
500 4ca693ca Michael Hanselmann
501 2d76b580 Michael Hanselmann
def main():
502 2d76b580 Michael Hanselmann
  """Main function.
503 2d76b580 Michael Hanselmann
504 2d76b580 Michael Hanselmann
  """
505 2d76b580 Michael Hanselmann
  # Option parsing
506 2d76b580 Michael Hanselmann
  (status_file_path, mode) = ParseOptions()
507 2d76b580 Michael Hanselmann
508 2d76b580 Michael Hanselmann
  # Configure logging
509 2d76b580 Michael Hanselmann
  child_logger = SetupLogging()
510 2d76b580 Michael Hanselmann
511 2d76b580 Michael Hanselmann
  status_file = StatusFile(status_file_path)
512 2d76b580 Michael Hanselmann
  try:
513 2d76b580 Michael Hanselmann
    try:
514 2d76b580 Michael Hanselmann
      # Pipe to receive socat's stderr output
515 2d76b580 Michael Hanselmann
      (socat_stderr_read_fd, socat_stderr_write_fd) = os.pipe()
516 2d76b580 Michael Hanselmann
517 c08d76f5 Michael Hanselmann
      # Pipe to receive dd's stderr output
518 c08d76f5 Michael Hanselmann
      (dd_stderr_read_fd, dd_stderr_write_fd) = os.pipe()
519 c08d76f5 Michael Hanselmann
520 c08d76f5 Michael Hanselmann
      # Pipe to receive dd's PID
521 c08d76f5 Michael Hanselmann
      (dd_pid_read_fd, dd_pid_write_fd) = os.pipe()
522 c08d76f5 Michael Hanselmann
523 f9323011 Michael Hanselmann
      # Pipe to receive size predicted by export script
524 f9323011 Michael Hanselmann
      (exp_size_read_fd, exp_size_write_fd) = os.pipe()
525 f9323011 Michael Hanselmann
526 2d76b580 Michael Hanselmann
      # Get child process command
527 c08d76f5 Michael Hanselmann
      cmd_builder = impexpd.CommandBuilder(mode, options, socat_stderr_write_fd,
528 c08d76f5 Michael Hanselmann
                                           dd_stderr_write_fd, dd_pid_write_fd)
529 bb44b1ae Michael Hanselmann
      cmd = cmd_builder.GetCommand()
530 2d76b580 Michael Hanselmann
531 f9323011 Michael Hanselmann
      # Prepare command environment
532 f9323011 Michael Hanselmann
      cmd_env = os.environ.copy()
533 f9323011 Michael Hanselmann
534 f9323011 Michael Hanselmann
      if options.exp_size == constants.IE_CUSTOM_SIZE:
535 f9323011 Michael Hanselmann
        cmd_env["EXP_SIZE_FD"] = str(exp_size_write_fd)
536 f9323011 Michael Hanselmann
537 2d76b580 Michael Hanselmann
      logging.debug("Starting command %r", cmd)
538 2d76b580 Michael Hanselmann
539 4ca693ca Michael Hanselmann
      # Start child process
540 f9323011 Michael Hanselmann
      child = ChildProcess(cmd_env, cmd,
541 f9323011 Michael Hanselmann
                           [socat_stderr_write_fd, dd_stderr_write_fd,
542 f9323011 Michael Hanselmann
                            dd_pid_write_fd, exp_size_write_fd])
543 2d76b580 Michael Hanselmann
      try:
544 2d76b580 Michael Hanselmann
        def _ForwardSignal(signum, _):
545 29da446a Michael Hanselmann
          """Forwards signals to child process.
546 2d76b580 Michael Hanselmann
547 29da446a Michael Hanselmann
          """
548 4ca693ca Michael Hanselmann
          child.Kill(signum)
549 2d76b580 Michael Hanselmann
550 29da446a Michael Hanselmann
        signal_wakeup = utils.SignalWakeupFd()
551 2d76b580 Michael Hanselmann
        try:
552 29da446a Michael Hanselmann
          # TODO: There is a race condition between starting the child and
553 29da446a Michael Hanselmann
          # handling the signals here. While there might be a way to work around
554 29da446a Michael Hanselmann
          # it by registering the handlers before starting the child and
555 29da446a Michael Hanselmann
          # deferring sent signals until the child is available, doing so can be
556 29da446a Michael Hanselmann
          # complicated.
557 29da446a Michael Hanselmann
          signal_handler = utils.SignalHandler([signal.SIGTERM, signal.SIGINT],
558 29da446a Michael Hanselmann
                                               handler_fn=_ForwardSignal,
559 29da446a Michael Hanselmann
                                               wakeup=signal_wakeup)
560 29da446a Michael Hanselmann
          try:
561 29da446a Michael Hanselmann
            # Close child's side
562 29da446a Michael Hanselmann
            utils.RetryOnSignal(os.close, socat_stderr_write_fd)
563 c08d76f5 Michael Hanselmann
            utils.RetryOnSignal(os.close, dd_stderr_write_fd)
564 c08d76f5 Michael Hanselmann
            utils.RetryOnSignal(os.close, dd_pid_write_fd)
565 f9323011 Michael Hanselmann
            utils.RetryOnSignal(os.close, exp_size_write_fd)
566 29da446a Michael Hanselmann
567 c08d76f5 Michael Hanselmann
            if ProcessChildIO(child, socat_stderr_read_fd, dd_stderr_read_fd,
568 f9323011 Michael Hanselmann
                              dd_pid_read_fd, exp_size_read_fd,
569 f9323011 Michael Hanselmann
                              status_file, child_logger,
570 c08d76f5 Michael Hanselmann
                              signal_wakeup, signal_handler, mode):
571 29da446a Michael Hanselmann
              # The child closed all its file descriptors and there was no
572 29da446a Michael Hanselmann
              # signal
573 29da446a Michael Hanselmann
              # TODO: Implement timeout instead of waiting indefinitely
574 29da446a Michael Hanselmann
              utils.RetryOnSignal(child.wait)
575 29da446a Michael Hanselmann
          finally:
576 29da446a Michael Hanselmann
            signal_handler.Reset()
577 2d76b580 Michael Hanselmann
        finally:
578 29da446a Michael Hanselmann
          signal_wakeup.Reset()
579 2d76b580 Michael Hanselmann
      finally:
580 4ca693ca Michael Hanselmann
        child.ForceQuit()
581 2d76b580 Michael Hanselmann
582 2d76b580 Michael Hanselmann
      if child.returncode == 0:
583 2d76b580 Michael Hanselmann
        errmsg = None
584 2d76b580 Michael Hanselmann
      elif child.returncode < 0:
585 2d76b580 Michael Hanselmann
        errmsg = "Exited due to signal %s" % (-child.returncode, )
586 2d76b580 Michael Hanselmann
      else:
587 2d76b580 Michael Hanselmann
        errmsg = "Exited with status %s" % (child.returncode, )
588 2d76b580 Michael Hanselmann
589 2d76b580 Michael Hanselmann
      status_file.SetExitStatus(child.returncode, errmsg)
590 b459a848 Andrea Spadaccini
    except Exception, err: # pylint: disable=W0703
591 2d76b580 Michael Hanselmann
      logging.exception("Unhandled error occurred")
592 2d76b580 Michael Hanselmann
      status_file.SetExitStatus(constants.EXIT_FAILURE,
593 2d76b580 Michael Hanselmann
                                "Unhandled error occurred: %s" % (err, ))
594 2d76b580 Michael Hanselmann
595 2d76b580 Michael Hanselmann
    if status_file.ExitStatusIsSuccess():
596 2d76b580 Michael Hanselmann
      sys.exit(constants.EXIT_SUCCESS)
597 2d76b580 Michael Hanselmann
598 2d76b580 Michael Hanselmann
    sys.exit(constants.EXIT_FAILURE)
599 2d76b580 Michael Hanselmann
  finally:
600 2d76b580 Michael Hanselmann
    status_file.Update(True)
601 2d76b580 Michael Hanselmann
602 2d76b580 Michael Hanselmann
603 2d76b580 Michael Hanselmann
if __name__ == "__main__":
604 2d76b580 Michael Hanselmann
  main()