Statistics
| Branch: | Tag: | Revision:

root / tools / cluster-merge @ 691e9563

History | View | Annotate | Download (17 kB)

1 1f7d3f7d René Nussbaumer
#!/usr/bin/python
2 1f7d3f7d René Nussbaumer
#
3 1f7d3f7d René Nussbaumer
4 1f7d3f7d René Nussbaumer
# Copyright (C) 2010 Google Inc.
5 1f7d3f7d René Nussbaumer
#
6 1f7d3f7d René Nussbaumer
# This program is free software; you can redistribute it and/or modify
7 1f7d3f7d René Nussbaumer
# it under the terms of the GNU General Public License as published by
8 1f7d3f7d René Nussbaumer
# the Free Software Foundation; either version 2 of the License, or
9 1f7d3f7d René Nussbaumer
# (at your option) any later version.
10 1f7d3f7d René Nussbaumer
#
11 1f7d3f7d René Nussbaumer
# This program is distributed in the hope that it will be useful, but
12 1f7d3f7d René Nussbaumer
# WITHOUT ANY WARRANTY; without even the implied warranty of
13 1f7d3f7d René Nussbaumer
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14 1f7d3f7d René Nussbaumer
# General Public License for more details.
15 1f7d3f7d René Nussbaumer
#
16 1f7d3f7d René Nussbaumer
# You should have received a copy of the GNU General Public License
17 1f7d3f7d René Nussbaumer
# along with this program; if not, write to the Free Software
18 1f7d3f7d René Nussbaumer
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
19 1f7d3f7d René Nussbaumer
# 02110-1301, USA.
20 1f7d3f7d René Nussbaumer
21 1f7d3f7d René Nussbaumer
"""Tool to merge two or more clusters together.
22 1f7d3f7d René Nussbaumer
23 1f7d3f7d René Nussbaumer
The clusters have to run the same version of Ganeti!
24 1f7d3f7d René Nussbaumer
25 1f7d3f7d René Nussbaumer
"""
26 1f7d3f7d René Nussbaumer
27 1f7d3f7d René Nussbaumer
# pylint: disable-msg=C0103
28 1f7d3f7d René Nussbaumer
# C0103: Invalid name cluster-merge
29 1f7d3f7d René Nussbaumer
30 1f7d3f7d René Nussbaumer
import logging
31 1f7d3f7d René Nussbaumer
import os
32 1f7d3f7d René Nussbaumer
import optparse
33 1f7d3f7d René Nussbaumer
import shutil
34 1f7d3f7d René Nussbaumer
import sys
35 1f7d3f7d René Nussbaumer
import tempfile
36 1f7d3f7d René Nussbaumer
37 1f7d3f7d René Nussbaumer
from ganeti import cli
38 1f7d3f7d René Nussbaumer
from ganeti import config
39 1f7d3f7d René Nussbaumer
from ganeti import constants
40 1f7d3f7d René Nussbaumer
from ganeti import errors
41 1f7d3f7d René Nussbaumer
from ganeti import ssh
42 1f7d3f7d René Nussbaumer
from ganeti import utils
43 1f7d3f7d René Nussbaumer
44 1f7d3f7d René Nussbaumer
45 1f7d3f7d René Nussbaumer
PAUSE_PERIOD_OPT = cli.cli_option("-p", "--watcher-pause-period", default=1800,
46 1f7d3f7d René Nussbaumer
                                  action="store", type="int",
47 1f7d3f7d René Nussbaumer
                                  dest="pause_period",
48 1f7d3f7d René Nussbaumer
                                  help=("Amount of time in seconds watcher"
49 1f7d3f7d René Nussbaumer
                                        " should be suspended from running"))
50 1f7d3f7d René Nussbaumer
51 1f7d3f7d René Nussbaumer
52 1f7d3f7d René Nussbaumer
def Flatten(unflatten_list):
53 1f7d3f7d René Nussbaumer
  """Flattens a list.
54 1f7d3f7d René Nussbaumer
55 1f7d3f7d René Nussbaumer
  @param unflatten_list: A list of unflatten list objects.
56 1f7d3f7d René Nussbaumer
  @return: A flatten list
57 1f7d3f7d René Nussbaumer
58 1f7d3f7d René Nussbaumer
  """
59 1f7d3f7d René Nussbaumer
  flatten_list = []
60 1f7d3f7d René Nussbaumer
61 1f7d3f7d René Nussbaumer
  for item in unflatten_list:
62 1f7d3f7d René Nussbaumer
    if isinstance(item, list):
63 1f7d3f7d René Nussbaumer
      flatten_list.extend(Flatten(item))
64 1f7d3f7d René Nussbaumer
    else:
65 1f7d3f7d René Nussbaumer
      flatten_list.append(item)
66 1f7d3f7d René Nussbaumer
  return flatten_list
67 1f7d3f7d René Nussbaumer
68 1f7d3f7d René Nussbaumer
69 1f7d3f7d René Nussbaumer
class MergerData(object):
70 1f7d3f7d René Nussbaumer
  """Container class to hold data used for merger.
71 1f7d3f7d René Nussbaumer
72 1f7d3f7d René Nussbaumer
  """
73 1f7d3f7d René Nussbaumer
  def __init__(self, cluster, key_path, nodes, instances, config_path=None):
74 1f7d3f7d René Nussbaumer
    """Initialize the container.
75 1f7d3f7d René Nussbaumer
76 1f7d3f7d René Nussbaumer
    @param cluster: The name of the cluster
77 1f7d3f7d René Nussbaumer
    @param key_path: Path to the ssh private key used for authentication
78 1f7d3f7d René Nussbaumer
    @param config_path: Path to the merging cluster config
79 1f7d3f7d René Nussbaumer
    @param nodes: List of nodes in the merging cluster
80 1f7d3f7d René Nussbaumer
    @param instances: List of instances running on merging cluster
81 1f7d3f7d René Nussbaumer
82 1f7d3f7d René Nussbaumer
    """
83 1f7d3f7d René Nussbaumer
    self.cluster = cluster
84 1f7d3f7d René Nussbaumer
    self.key_path = key_path
85 1f7d3f7d René Nussbaumer
    self.config_path = config_path
86 1f7d3f7d René Nussbaumer
    self.instances = instances
87 1f7d3f7d René Nussbaumer
    self.nodes = nodes
88 1f7d3f7d René Nussbaumer
89 1f7d3f7d René Nussbaumer
90 1f7d3f7d René Nussbaumer
class Merger(object):
91 1f7d3f7d René Nussbaumer
  """Handling the merge.
92 1f7d3f7d René Nussbaumer
93 1f7d3f7d René Nussbaumer
  """
94 1f7d3f7d René Nussbaumer
  def __init__(self, clusters, pause_period):
95 1f7d3f7d René Nussbaumer
    """Initialize object with sane defaults and infos required.
96 1f7d3f7d René Nussbaumer
97 1f7d3f7d René Nussbaumer
    @param clusters: The list of clusters to merge in
98 1f7d3f7d René Nussbaumer
    @param pause_period: The time watcher shall be disabled for
99 1f7d3f7d René Nussbaumer
100 1f7d3f7d René Nussbaumer
    """
101 1f7d3f7d René Nussbaumer
    self.merger_data = []
102 1f7d3f7d René Nussbaumer
    self.clusters = clusters
103 1f7d3f7d René Nussbaumer
    self.pause_period = pause_period
104 1f7d3f7d René Nussbaumer
    self.work_dir = tempfile.mkdtemp(suffix="cluster-merger")
105 1f7d3f7d René Nussbaumer
    self.cluster_name = cli.GetClient().QueryConfigValues(["cluster_name"])
106 1f7d3f7d René Nussbaumer
    self.ssh_runner = ssh.SshRunner(self.cluster_name)
107 1f7d3f7d René Nussbaumer
108 1f7d3f7d René Nussbaumer
  def Setup(self):
109 1f7d3f7d René Nussbaumer
    """Sets up our end so we can do the merger.
110 1f7d3f7d René Nussbaumer
111 1f7d3f7d René Nussbaumer
    This method is setting us up as a preparation for the merger.
112 1f7d3f7d René Nussbaumer
    It makes the initial contact and gathers information needed.
113 1f7d3f7d René Nussbaumer
114 1f7d3f7d René Nussbaumer
    @raise errors.RemoteError: for errors in communication/grabbing
115 1f7d3f7d René Nussbaumer
116 1f7d3f7d René Nussbaumer
    """
117 1f7d3f7d René Nussbaumer
    (remote_path, _, _) = ssh.GetUserFiles("root")
118 1f7d3f7d René Nussbaumer
119 1f7d3f7d René Nussbaumer
    # Fetch remotes private key
120 1f7d3f7d René Nussbaumer
    for cluster in self.clusters:
121 1f7d3f7d René Nussbaumer
      result = self._RunCmd(cluster, "cat %s" % remote_path, batch=False,
122 1f7d3f7d René Nussbaumer
                            ask_key=False)
123 1f7d3f7d René Nussbaumer
      if result.failed:
124 1f7d3f7d René Nussbaumer
        raise errors.RemoteError("There was an error while grabbing ssh private"
125 1f7d3f7d René Nussbaumer
                                 " key from %s. Fail reason: %s; output: %s" %
126 1f7d3f7d René Nussbaumer
                                 (cluster, result.fail_reason, result.output))
127 1f7d3f7d René Nussbaumer
128 c4feafe8 Iustin Pop
      key_path = utils.PathJoin(self.work_dir, cluster)
129 1f7d3f7d René Nussbaumer
      utils.WriteFile(key_path, mode=0600, data=result.stdout)
130 1f7d3f7d René Nussbaumer
131 1f7d3f7d René Nussbaumer
      result = self._RunCmd(cluster, "gnt-node list -o name --no-header",
132 1f7d3f7d René Nussbaumer
                            private_key=key_path)
133 1f7d3f7d René Nussbaumer
      if result.failed:
134 1f7d3f7d René Nussbaumer
        raise errors.RemoteError("Unable to retrieve list of nodes from %s."
135 1f7d3f7d René Nussbaumer
                                 " Fail reason: %s; output: %s" %
136 1f7d3f7d René Nussbaumer
                                 (cluster, result.fail_reason, result.output))
137 1f7d3f7d René Nussbaumer
      nodes = result.stdout.splitlines()
138 1f7d3f7d René Nussbaumer
139 1f7d3f7d René Nussbaumer
      result = self._RunCmd(cluster, "gnt-instance list -o name --no-header",
140 1f7d3f7d René Nussbaumer
                            private_key=key_path)
141 1f7d3f7d René Nussbaumer
      if result.failed:
142 1f7d3f7d René Nussbaumer
        raise errors.RemoteError("Unable to retrieve list of instances from"
143 1f7d3f7d René Nussbaumer
                                 " %s. Fail reason: %s; output: %s" %
144 1f7d3f7d René Nussbaumer
                                 (cluster, result.fail_reason, result.output))
145 1f7d3f7d René Nussbaumer
      instances = result.stdout.splitlines()
146 1f7d3f7d René Nussbaumer
147 1f7d3f7d René Nussbaumer
      self.merger_data.append(MergerData(cluster, key_path, nodes, instances))
148 1f7d3f7d René Nussbaumer
149 1f7d3f7d René Nussbaumer
  def _PrepareAuthorizedKeys(self):
150 1f7d3f7d René Nussbaumer
    """Prepare the authorized_keys on every merging node.
151 1f7d3f7d René Nussbaumer
152 1f7d3f7d René Nussbaumer
    This method add our public key to remotes authorized_key for further
153 1f7d3f7d René Nussbaumer
    communication.
154 1f7d3f7d René Nussbaumer
155 1f7d3f7d René Nussbaumer
    """
156 1f7d3f7d René Nussbaumer
    (_, pub_key_file, auth_keys) = ssh.GetUserFiles("root")
157 1f7d3f7d René Nussbaumer
    pub_key = utils.ReadFile(pub_key_file)
158 1f7d3f7d René Nussbaumer
159 1f7d3f7d René Nussbaumer
    for data in self.merger_data:
160 1f7d3f7d René Nussbaumer
      for node in data.nodes:
161 1f7d3f7d René Nussbaumer
        result = self._RunCmd(node, ("cat >> %s << '!EOF.'\n%s!EOF.\n" %
162 1f7d3f7d René Nussbaumer
                                     (auth_keys, pub_key)),
163 1f7d3f7d René Nussbaumer
                              private_key=data.key_path)
164 1f7d3f7d René Nussbaumer
165 1f7d3f7d René Nussbaumer
        if result.failed:
166 1f7d3f7d René Nussbaumer
          raise errors.RemoteError("Unable to add our public key to %s in %s."
167 1f7d3f7d René Nussbaumer
                                   " Fail reason: %s; output: %s" %
168 1f7d3f7d René Nussbaumer
                                   (node, data.cluster, result.fail_reason,
169 1f7d3f7d René Nussbaumer
                                    result.output))
170 1f7d3f7d René Nussbaumer
171 1f7d3f7d René Nussbaumer
  def _RunCmd(self, hostname, command, user="root", use_cluster_key=False,
172 1f7d3f7d René Nussbaumer
              strict_host_check=False, private_key=None, batch=True,
173 1f7d3f7d René Nussbaumer
              ask_key=False):
174 1f7d3f7d René Nussbaumer
    """Wrapping SshRunner.Run with default parameters.
175 1f7d3f7d René Nussbaumer
176 1f7d3f7d René Nussbaumer
    For explanation of parameters see L{ssh.SshRunner.Run}.
177 1f7d3f7d René Nussbaumer
178 1f7d3f7d René Nussbaumer
    """
179 1f7d3f7d René Nussbaumer
    return self.ssh_runner.Run(hostname=hostname, command=command, user=user,
180 1f7d3f7d René Nussbaumer
                               use_cluster_key=use_cluster_key,
181 1f7d3f7d René Nussbaumer
                               strict_host_check=strict_host_check,
182 1f7d3f7d René Nussbaumer
                               private_key=private_key, batch=batch,
183 1f7d3f7d René Nussbaumer
                               ask_key=ask_key)
184 1f7d3f7d René Nussbaumer
185 1f7d3f7d René Nussbaumer
  def _StopMergingInstances(self):
186 1f7d3f7d René Nussbaumer
    """Stop instances on merging clusters.
187 1f7d3f7d René Nussbaumer
188 1f7d3f7d René Nussbaumer
    """
189 1f7d3f7d René Nussbaumer
    for cluster in self.clusters:
190 1f7d3f7d René Nussbaumer
      result = self._RunCmd(cluster, "gnt-instance shutdown --all"
191 1f7d3f7d René Nussbaumer
                                     " --force-multiple")
192 1f7d3f7d René Nussbaumer
193 1f7d3f7d René Nussbaumer
      if result.failed:
194 1f7d3f7d René Nussbaumer
        raise errors.RemoteError("Unable to stop instances on %s."
195 1f7d3f7d René Nussbaumer
                                 " Fail reason: %s; output: %s" %
196 1f7d3f7d René Nussbaumer
                                 (cluster, result.fail_reason, result.output))
197 1f7d3f7d René Nussbaumer
198 1f7d3f7d René Nussbaumer
  def _DisableWatcher(self):
199 1f7d3f7d René Nussbaumer
    """Disable watch on all merging clusters, including ourself.
200 1f7d3f7d René Nussbaumer
201 1f7d3f7d René Nussbaumer
    """
202 1f7d3f7d René Nussbaumer
    for cluster in ["localhost"] + self.clusters:
203 1f7d3f7d René Nussbaumer
      result = self._RunCmd(cluster, "gnt-cluster watcher pause %d" %
204 1f7d3f7d René Nussbaumer
                                     self.pause_period)
205 1f7d3f7d René Nussbaumer
206 1f7d3f7d René Nussbaumer
      if result.failed:
207 1f7d3f7d René Nussbaumer
        raise errors.RemoteError("Unable to pause watcher on %s."
208 1f7d3f7d René Nussbaumer
                                 " Fail reason: %s; output: %s" %
209 1f7d3f7d René Nussbaumer
                                 (cluster, result.fail_reason, result.output))
210 1f7d3f7d René Nussbaumer
211 1f7d3f7d René Nussbaumer
212 1f7d3f7d René Nussbaumer
  # R0201: Method could be a function
213 1f7d3f7d René Nussbaumer
  def _EnableWatcher(self): # pylint: disable-msg=R0201
214 1f7d3f7d René Nussbaumer
    """Reenable watcher (locally).
215 1f7d3f7d René Nussbaumer
216 1f7d3f7d René Nussbaumer
    """
217 1f7d3f7d René Nussbaumer
    result = utils.RunCmd(["gnt-cluster", "watcher", "continue"])
218 1f7d3f7d René Nussbaumer
219 1f7d3f7d René Nussbaumer
    if result.failed:
220 1f7d3f7d René Nussbaumer
      logging.warning("Unable to continue watcher. Fail reason: %s;"
221 3f1cf151 René Nussbaumer
                      " output: %s", result.fail_reason, result.output)
222 1f7d3f7d René Nussbaumer
223 1f7d3f7d René Nussbaumer
  def _StopDaemons(self):
224 1f7d3f7d René Nussbaumer
    """Stop all daemons on merging nodes.
225 1f7d3f7d René Nussbaumer
226 1f7d3f7d René Nussbaumer
    """
227 1f7d3f7d René Nussbaumer
    # FIXME: Worth to put this into constants?
228 1f7d3f7d René Nussbaumer
    cmds = []
229 1f7d3f7d René Nussbaumer
    for daemon in (constants.RAPI, constants.MASTERD,
230 1f7d3f7d René Nussbaumer
                   constants.NODED, constants.CONFD):
231 1f7d3f7d René Nussbaumer
      cmds.append("%s stop %s" % (constants.DAEMON_UTIL, daemon))
232 1f7d3f7d René Nussbaumer
    for data in self.merger_data:
233 1f7d3f7d René Nussbaumer
      for node in data.nodes:
234 1f7d3f7d René Nussbaumer
        result = self._RunCmd(node, " && ".join(cmds))
235 1f7d3f7d René Nussbaumer
236 1f7d3f7d René Nussbaumer
        if result.failed:
237 1f7d3f7d René Nussbaumer
          raise errors.RemoteError("Unable to stop daemons on %s."
238 1f7d3f7d René Nussbaumer
                                   " Fail reason: %s; output: %s." %
239 1f7d3f7d René Nussbaumer
                                   (node, result.fail_reason, result.output))
240 1f7d3f7d René Nussbaumer
241 1f7d3f7d René Nussbaumer
  def _FetchRemoteConfig(self):
242 1f7d3f7d René Nussbaumer
    """Fetches and stores remote cluster config from the master.
243 1f7d3f7d René Nussbaumer
244 1f7d3f7d René Nussbaumer
    This step is needed before we can merge the config.
245 1f7d3f7d René Nussbaumer
246 1f7d3f7d René Nussbaumer
    """
247 1f7d3f7d René Nussbaumer
    for data in self.merger_data:
248 1f7d3f7d René Nussbaumer
      result = self._RunCmd(data.cluster, "cat %s" %
249 1f7d3f7d René Nussbaumer
                                          constants.CLUSTER_CONF_FILE)
250 1f7d3f7d René Nussbaumer
251 1f7d3f7d René Nussbaumer
      if result.failed:
252 1f7d3f7d René Nussbaumer
        raise errors.RemoteError("Unable to retrieve remote config on %s."
253 1f7d3f7d René Nussbaumer
                                 " Fail reason: %s; output %s" %
254 1f7d3f7d René Nussbaumer
                                 (data.cluster, result.fail_reason,
255 1f7d3f7d René Nussbaumer
                                  result.output))
256 1f7d3f7d René Nussbaumer
257 c4feafe8 Iustin Pop
      data.config_path = utils.PathJoin(self.work_dir, "%s_config.data" %
258 c4feafe8 Iustin Pop
                                        data.cluster)
259 1f7d3f7d René Nussbaumer
      utils.WriteFile(data.config_path, data=result.stdout)
260 1f7d3f7d René Nussbaumer
261 1f7d3f7d René Nussbaumer
  # R0201: Method could be a function
262 1f7d3f7d René Nussbaumer
  def _KillMasterDaemon(self): # pylint: disable-msg=R0201
263 1f7d3f7d René Nussbaumer
    """Kills the local master daemon.
264 1f7d3f7d René Nussbaumer
265 1f7d3f7d René Nussbaumer
    @raise errors.CommandError: If unable to kill
266 1f7d3f7d René Nussbaumer
267 1f7d3f7d René Nussbaumer
    """
268 1f7d3f7d René Nussbaumer
    result = utils.RunCmd([constants.DAEMON_UTIL, "stop-master"])
269 1f7d3f7d René Nussbaumer
    if result.failed:
270 1f7d3f7d René Nussbaumer
      raise errors.CommandError("Unable to stop master daemons."
271 1f7d3f7d René Nussbaumer
                                " Fail reason: %s; output: %s" %
272 1f7d3f7d René Nussbaumer
                                (result.fail_reason, result.output))
273 1f7d3f7d René Nussbaumer
274 1f7d3f7d René Nussbaumer
  def _MergeConfig(self):
275 1f7d3f7d René Nussbaumer
    """Merges all foreign config into our own config.
276 1f7d3f7d René Nussbaumer
277 1f7d3f7d René Nussbaumer
    """
278 1f7d3f7d René Nussbaumer
    my_config = config.ConfigWriter(offline=True)
279 1f7d3f7d René Nussbaumer
    fake_ec_id = 0 # Needs to be uniq over the whole config merge
280 1f7d3f7d René Nussbaumer
281 1f7d3f7d René Nussbaumer
    for data in self.merger_data:
282 1f7d3f7d René Nussbaumer
      other_config = config.ConfigWriter(data.config_path)
283 1f7d3f7d René Nussbaumer
284 1f7d3f7d René Nussbaumer
      for node in other_config.GetNodeList():
285 1f7d3f7d René Nussbaumer
        node_info = other_config.GetNodeInfo(node)
286 1f7d3f7d René Nussbaumer
        node_info.master_candidate = False
287 1f7d3f7d René Nussbaumer
        my_config.AddNode(node_info, str(fake_ec_id))
288 1f7d3f7d René Nussbaumer
        fake_ec_id += 1
289 1f7d3f7d René Nussbaumer
290 1f7d3f7d René Nussbaumer
      for instance in other_config.GetInstanceList():
291 1f7d3f7d René Nussbaumer
        instance_info = other_config.GetInstanceInfo(instance)
292 1f7d3f7d René Nussbaumer
293 1f7d3f7d René Nussbaumer
        # Update the DRBD port assignments
294 1f7d3f7d René Nussbaumer
        # This is a little bit hackish
295 1f7d3f7d René Nussbaumer
        for dsk in instance_info.disks:
296 1f7d3f7d René Nussbaumer
          if dsk.dev_type in constants.LDS_DRBD:
297 1f7d3f7d René Nussbaumer
            port = my_config.AllocatePort()
298 1f7d3f7d René Nussbaumer
299 1f7d3f7d René Nussbaumer
            logical_id = list(dsk.logical_id)
300 1f7d3f7d René Nussbaumer
            logical_id[2] = port
301 1f7d3f7d René Nussbaumer
            dsk.logical_id = tuple(logical_id)
302 1f7d3f7d René Nussbaumer
303 1f7d3f7d René Nussbaumer
            physical_id = list(dsk.physical_id)
304 1f7d3f7d René Nussbaumer
            physical_id[1] = physical_id[3] = port
305 1f7d3f7d René Nussbaumer
            dsk.physical_id = tuple(physical_id)
306 1f7d3f7d René Nussbaumer
307 1f7d3f7d René Nussbaumer
        my_config.AddInstance(instance_info, str(fake_ec_id))
308 1f7d3f7d René Nussbaumer
        fake_ec_id += 1
309 1f7d3f7d René Nussbaumer
310 1f7d3f7d René Nussbaumer
  # R0201: Method could be a function
311 1f7d3f7d René Nussbaumer
  def _StartMasterDaemon(self, no_vote=False): # pylint: disable-msg=R0201
312 1f7d3f7d René Nussbaumer
    """Starts the local master daemon.
313 1f7d3f7d René Nussbaumer
314 1f7d3f7d René Nussbaumer
    @param no_vote: Should the masterd started without voting? default: False
315 1f7d3f7d René Nussbaumer
    @raise errors.CommandError: If unable to start daemon.
316 1f7d3f7d René Nussbaumer
317 1f7d3f7d René Nussbaumer
    """
318 1f7d3f7d René Nussbaumer
    env = {}
319 1f7d3f7d René Nussbaumer
    if no_vote:
320 1f7d3f7d René Nussbaumer
      env["EXTRA_MASTERD_ARGS"] = "--no-voting --yes-do-it"
321 1f7d3f7d René Nussbaumer
322 1f7d3f7d René Nussbaumer
    result = utils.RunCmd([constants.DAEMON_UTIL, "start-master"], env=env)
323 1f7d3f7d René Nussbaumer
    if result.failed:
324 1f7d3f7d René Nussbaumer
      raise errors.CommandError("Couldn't start ganeti master."
325 1f7d3f7d René Nussbaumer
                                " Fail reason: %s; output: %s" %
326 1f7d3f7d René Nussbaumer
                                (result.fail_reason, result.output))
327 1f7d3f7d René Nussbaumer
328 1f7d3f7d René Nussbaumer
  def _ReaddMergedNodesAndRedist(self):
329 1f7d3f7d René Nussbaumer
    """Readds all merging nodes and make sure their config is up-to-date.
330 1f7d3f7d René Nussbaumer
331 1f7d3f7d René Nussbaumer
    @raise errors.CommandError: If anything fails.
332 1f7d3f7d René Nussbaumer
333 1f7d3f7d René Nussbaumer
    """
334 1f7d3f7d René Nussbaumer
    for data in self.merger_data:
335 1f7d3f7d René Nussbaumer
      for node in data.nodes:
336 1f7d3f7d René Nussbaumer
        result = utils.RunCmd(["gnt-node", "add", "--readd",
337 1f7d3f7d René Nussbaumer
                               "--no-ssh-key-check", node])
338 1f7d3f7d René Nussbaumer
        if result.failed:
339 1f7d3f7d René Nussbaumer
          raise errors.CommandError("Couldn't readd node %s. Fail reason: %s;"
340 1f7d3f7d René Nussbaumer
                                    " output: %s" % (node, result.fail_reason,
341 1f7d3f7d René Nussbaumer
                                                     result.output))
342 1f7d3f7d René Nussbaumer
343 1f7d3f7d René Nussbaumer
    result = utils.RunCmd(["gnt-cluster", "redist-conf"])
344 1f7d3f7d René Nussbaumer
    if result.failed:
345 1f7d3f7d René Nussbaumer
      raise errors.CommandError("Redistribution failed. Fail reason: %s;"
346 1f7d3f7d René Nussbaumer
                                " output: %s" % (result.fail_reason,
347 1f7d3f7d René Nussbaumer
                                                result.output))
348 1f7d3f7d René Nussbaumer
349 1f7d3f7d René Nussbaumer
  # R0201: Method could be a function
350 1f7d3f7d René Nussbaumer
  def _StartupAllInstances(self): # pylint: disable-msg=R0201
351 1f7d3f7d René Nussbaumer
    """Starts up all instances (locally).
352 1f7d3f7d René Nussbaumer
353 1f7d3f7d René Nussbaumer
    @raise errors.CommandError: If unable to start clusters
354 1f7d3f7d René Nussbaumer
355 1f7d3f7d René Nussbaumer
    """
356 1f7d3f7d René Nussbaumer
    result = utils.RunCmd(["gnt-instance", "startup", "--all",
357 1f7d3f7d René Nussbaumer
                           "--force-multiple"])
358 1f7d3f7d René Nussbaumer
    if result.failed:
359 1f7d3f7d René Nussbaumer
      raise errors.CommandError("Unable to start all instances."
360 1f7d3f7d René Nussbaumer
                                " Fail reason: %s; output: %s" %
361 1f7d3f7d René Nussbaumer
                                (result.fail_reason, result.output))
362 1f7d3f7d René Nussbaumer
363 1f7d3f7d René Nussbaumer
  # R0201: Method could be a function
364 1f7d3f7d René Nussbaumer
  def _VerifyCluster(self): # pylint: disable-msg=R0201
365 1f7d3f7d René Nussbaumer
    """Runs gnt-cluster verify to verify the health.
366 1f7d3f7d René Nussbaumer
367 1f7d3f7d René Nussbaumer
    @raise errors.ProgrammError: If cluster fails on verification
368 1f7d3f7d René Nussbaumer
369 1f7d3f7d René Nussbaumer
    """
370 1f7d3f7d René Nussbaumer
    result = utils.RunCmd(["gnt-cluster", "verify"])
371 1f7d3f7d René Nussbaumer
    if result.failed:
372 1f7d3f7d René Nussbaumer
      raise errors.CommandError("Verification of cluster failed."
373 1f7d3f7d René Nussbaumer
                                " Fail reason: %s; output: %s" %
374 1f7d3f7d René Nussbaumer
                                (result.fail_reason, result.output))
375 1f7d3f7d René Nussbaumer
376 1f7d3f7d René Nussbaumer
  def Merge(self):
377 1f7d3f7d René Nussbaumer
    """Does the actual merge.
378 1f7d3f7d René Nussbaumer
379 1f7d3f7d René Nussbaumer
    It runs all the steps in the right order and updates the user about steps
380 1f7d3f7d René Nussbaumer
    taken. Also it keeps track of rollback_steps to undo everything.
381 1f7d3f7d René Nussbaumer
382 1f7d3f7d René Nussbaumer
    """
383 1f7d3f7d René Nussbaumer
    rbsteps = []
384 1f7d3f7d René Nussbaumer
    try:
385 1f7d3f7d René Nussbaumer
      logging.info("Pre cluster verification")
386 1f7d3f7d René Nussbaumer
      self._VerifyCluster()
387 1f7d3f7d René Nussbaumer
388 1f7d3f7d René Nussbaumer
      logging.info("Prepare authorized_keys")
389 1f7d3f7d René Nussbaumer
      rbsteps.append("Remove our key from authorized_keys on nodes:"
390 1f7d3f7d René Nussbaumer
                     " %(nodes)s")
391 1f7d3f7d René Nussbaumer
      self._PrepareAuthorizedKeys()
392 1f7d3f7d René Nussbaumer
393 1f7d3f7d René Nussbaumer
      rbsteps.append("Start all instances again on the merging"
394 1f7d3f7d René Nussbaumer
                     " clusters: %(clusters)s")
395 1f7d3f7d René Nussbaumer
      logging.info("Stopping merging instances (takes a while)")
396 1f7d3f7d René Nussbaumer
      self._StopMergingInstances()
397 1f7d3f7d René Nussbaumer
398 1f7d3f7d René Nussbaumer
      logging.info("Disable watcher")
399 1f7d3f7d René Nussbaumer
      self._DisableWatcher()
400 1f7d3f7d René Nussbaumer
      logging.info("Stop daemons on merging nodes")
401 1f7d3f7d René Nussbaumer
      self._StopDaemons()
402 1f7d3f7d René Nussbaumer
      logging.info("Merging config")
403 1f7d3f7d René Nussbaumer
      self._FetchRemoteConfig()
404 1f7d3f7d René Nussbaumer
      self._KillMasterDaemon()
405 1f7d3f7d René Nussbaumer
406 1f7d3f7d René Nussbaumer
      rbsteps.append("Restore %s from another master candidate" %
407 1f7d3f7d René Nussbaumer
                     constants.CLUSTER_CONF_FILE)
408 1f7d3f7d René Nussbaumer
      self._MergeConfig()
409 1f7d3f7d René Nussbaumer
      self._StartMasterDaemon(no_vote=True)
410 1f7d3f7d René Nussbaumer
411 1f7d3f7d René Nussbaumer
      # Point of no return, delete rbsteps
412 1f7d3f7d René Nussbaumer
      del rbsteps[:]
413 1f7d3f7d René Nussbaumer
414 1f7d3f7d René Nussbaumer
      logging.warning("We are at the point of no return. Merge can not easily"
415 1f7d3f7d René Nussbaumer
                      " be undone after this point.")
416 1f7d3f7d René Nussbaumer
      logging.info("Readd nodes and redistribute config")
417 1f7d3f7d René Nussbaumer
      self._ReaddMergedNodesAndRedist()
418 1f7d3f7d René Nussbaumer
      self._KillMasterDaemon()
419 1f7d3f7d René Nussbaumer
      self._StartMasterDaemon()
420 1f7d3f7d René Nussbaumer
      logging.info("Starting instances again")
421 1f7d3f7d René Nussbaumer
      self._StartupAllInstances()
422 1f7d3f7d René Nussbaumer
      logging.info("Post cluster verification")
423 1f7d3f7d René Nussbaumer
      self._VerifyCluster()
424 1f7d3f7d René Nussbaumer
    except errors.GenericError, e:
425 1f7d3f7d René Nussbaumer
      logging.exception(e)
426 1f7d3f7d René Nussbaumer
427 1f7d3f7d René Nussbaumer
      if rbsteps:
428 1f7d3f7d René Nussbaumer
        nodes = Flatten([data.nodes for data in self.merger_data])
429 1f7d3f7d René Nussbaumer
        info = {
430 1f7d3f7d René Nussbaumer
          "clusters": self.clusters,
431 1f7d3f7d René Nussbaumer
          "nodes": nodes,
432 1f7d3f7d René Nussbaumer
          }
433 1f7d3f7d René Nussbaumer
        logging.critical("In order to rollback do the following:")
434 1f7d3f7d René Nussbaumer
        for step in rbsteps:
435 3f1cf151 René Nussbaumer
          logging.critical("  * %s", step % info)
436 1f7d3f7d René Nussbaumer
      else:
437 1f7d3f7d René Nussbaumer
        logging.critical("Nothing to rollback.")
438 1f7d3f7d René Nussbaumer
439 1f7d3f7d René Nussbaumer
      # TODO: Keep track of steps done for a flawless resume?
440 1f7d3f7d René Nussbaumer
441 1f7d3f7d René Nussbaumer
  def Cleanup(self):
442 1f7d3f7d René Nussbaumer
    """Clean up our environment.
443 1f7d3f7d René Nussbaumer
444 1f7d3f7d René Nussbaumer
    This cleans up remote private keys and configs and after that
445 1f7d3f7d René Nussbaumer
    deletes the temporary directory.
446 1f7d3f7d René Nussbaumer
447 1f7d3f7d René Nussbaumer
    """
448 1f7d3f7d René Nussbaumer
    shutil.rmtree(self.work_dir)
449 1f7d3f7d René Nussbaumer
450 1f7d3f7d René Nussbaumer
451 1f7d3f7d René Nussbaumer
def SetupLogging(options):
452 1f7d3f7d René Nussbaumer
  """Setting up logging infrastructure.
453 1f7d3f7d René Nussbaumer
454 1f7d3f7d René Nussbaumer
  @param options: Parsed command line options
455 1f7d3f7d René Nussbaumer
456 1f7d3f7d René Nussbaumer
  """
457 1f7d3f7d René Nussbaumer
  formatter = logging.Formatter("%(asctime)s: %(levelname)s %(message)s")
458 1f7d3f7d René Nussbaumer
459 1f7d3f7d René Nussbaumer
  stderr_handler = logging.StreamHandler()
460 1f7d3f7d René Nussbaumer
  stderr_handler.setFormatter(formatter)
461 1f7d3f7d René Nussbaumer
  if options.debug:
462 1f7d3f7d René Nussbaumer
    stderr_handler.setLevel(logging.NOTSET)
463 1f7d3f7d René Nussbaumer
  elif options.verbose:
464 1f7d3f7d René Nussbaumer
    stderr_handler.setLevel(logging.INFO)
465 1f7d3f7d René Nussbaumer
  else:
466 1f7d3f7d René Nussbaumer
    stderr_handler.setLevel(logging.ERROR)
467 1f7d3f7d René Nussbaumer
468 1f7d3f7d René Nussbaumer
  root_logger = logging.getLogger("")
469 1f7d3f7d René Nussbaumer
  root_logger.setLevel(logging.NOTSET)
470 1f7d3f7d René Nussbaumer
  root_logger.addHandler(stderr_handler)
471 1f7d3f7d René Nussbaumer
472 1f7d3f7d René Nussbaumer
473 1f7d3f7d René Nussbaumer
def main():
474 1f7d3f7d René Nussbaumer
  """Main routine.
475 1f7d3f7d René Nussbaumer
476 1f7d3f7d René Nussbaumer
  """
477 1f7d3f7d René Nussbaumer
  program = os.path.basename(sys.argv[0])
478 1f7d3f7d René Nussbaumer
479 1f7d3f7d René Nussbaumer
  parser = optparse.OptionParser(usage=("%prog [--debug|--verbose]"
480 1f7d3f7d René Nussbaumer
                                        " [--watcher-pause-period SECONDS]"
481 1f7d3f7d René Nussbaumer
                                        " <cluster> <cluster...>"),
482 1f7d3f7d René Nussbaumer
                                        prog=program)
483 1f7d3f7d René Nussbaumer
  parser.add_option(cli.DEBUG_OPT)
484 1f7d3f7d René Nussbaumer
  parser.add_option(cli.VERBOSE_OPT)
485 1f7d3f7d René Nussbaumer
  parser.add_option(PAUSE_PERIOD_OPT)
486 1f7d3f7d René Nussbaumer
487 1f7d3f7d René Nussbaumer
  (options, args) = parser.parse_args()
488 1f7d3f7d René Nussbaumer
489 1f7d3f7d René Nussbaumer
  SetupLogging(options)
490 1f7d3f7d René Nussbaumer
491 1f7d3f7d René Nussbaumer
  if not args:
492 1f7d3f7d René Nussbaumer
    parser.error("No clusters specified")
493 1f7d3f7d René Nussbaumer
494 1f7d3f7d René Nussbaumer
  cluster_merger = Merger(utils.UniqueSequence(args), options.pause_period)
495 1f7d3f7d René Nussbaumer
  try:
496 1f7d3f7d René Nussbaumer
    try:
497 1f7d3f7d René Nussbaumer
      cluster_merger.Setup()
498 1f7d3f7d René Nussbaumer
      cluster_merger.Merge()
499 1f7d3f7d René Nussbaumer
    except errors.GenericError, e:
500 1f7d3f7d René Nussbaumer
      logging.exception(e)
501 1f7d3f7d René Nussbaumer
      return constants.EXIT_FAILURE
502 1f7d3f7d René Nussbaumer
  finally:
503 1f7d3f7d René Nussbaumer
    cluster_merger.Cleanup()
504 1f7d3f7d René Nussbaumer
505 1f7d3f7d René Nussbaumer
  return constants.EXIT_SUCCESS
506 1f7d3f7d René Nussbaumer
507 1f7d3f7d René Nussbaumer
508 1f7d3f7d René Nussbaumer
if __name__ == "__main__":
509 1f7d3f7d René Nussbaumer
  sys.exit(main())