root / tools / cluster-merge @ 99a11adc
History | View | Annotate | Download (17.3 kB)
1 | 1f7d3f7d | René Nussbaumer | #!/usr/bin/python |
---|---|---|---|
2 | 1f7d3f7d | René Nussbaumer | # |
3 | 1f7d3f7d | René Nussbaumer | |
4 | 3d8f154f | Stephen Shirley | # Copyright (C) 2010 Google Inc. |
5 | 1f7d3f7d | René Nussbaumer | # |
6 | 1f7d3f7d | René Nussbaumer | # This program is free software; you can redistribute it and/or modify |
7 | 1f7d3f7d | René Nussbaumer | # it under the terms of the GNU General Public License as published by |
8 | 1f7d3f7d | René Nussbaumer | # the Free Software Foundation; either version 2 of the License, or |
9 | 1f7d3f7d | René Nussbaumer | # (at your option) any later version. |
10 | 1f7d3f7d | René Nussbaumer | # |
11 | 1f7d3f7d | René Nussbaumer | # This program is distributed in the hope that it will be useful, but |
12 | 1f7d3f7d | René Nussbaumer | # WITHOUT ANY WARRANTY; without even the implied warranty of |
13 | 1f7d3f7d | René Nussbaumer | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
14 | 1f7d3f7d | René Nussbaumer | # General Public License for more details. |
15 | 1f7d3f7d | René Nussbaumer | # |
16 | 1f7d3f7d | René Nussbaumer | # You should have received a copy of the GNU General Public License |
17 | 1f7d3f7d | René Nussbaumer | # along with this program; if not, write to the Free Software |
18 | 1f7d3f7d | René Nussbaumer | # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA |
19 | 1f7d3f7d | René Nussbaumer | # 02110-1301, USA. |
20 | 1f7d3f7d | René Nussbaumer | |
21 | 1f7d3f7d | René Nussbaumer | """Tool to merge two or more clusters together. |
22 | 1f7d3f7d | René Nussbaumer | |
23 | 1f7d3f7d | René Nussbaumer | The clusters have to run the same version of Ganeti! |
24 | 1f7d3f7d | René Nussbaumer | |
25 | 1f7d3f7d | René Nussbaumer | """ |
26 | 1f7d3f7d | René Nussbaumer | |
27 | 1f7d3f7d | René Nussbaumer | # pylint: disable-msg=C0103 |
28 | 1f7d3f7d | René Nussbaumer | # C0103: Invalid name cluster-merge |
29 | 1f7d3f7d | René Nussbaumer | |
30 | 1f7d3f7d | René Nussbaumer | import logging |
31 | 1f7d3f7d | René Nussbaumer | import os |
32 | 1f7d3f7d | René Nussbaumer | import optparse |
33 | 1f7d3f7d | René Nussbaumer | import shutil |
34 | 1f7d3f7d | René Nussbaumer | import sys |
35 | 1f7d3f7d | René Nussbaumer | import tempfile |
36 | 1f7d3f7d | René Nussbaumer | |
37 | 1f7d3f7d | René Nussbaumer | from ganeti import cli |
38 | 1f7d3f7d | René Nussbaumer | from ganeti import config |
39 | 1f7d3f7d | René Nussbaumer | from ganeti import constants |
40 | 1f7d3f7d | René Nussbaumer | from ganeti import errors |
41 | 1f7d3f7d | René Nussbaumer | from ganeti import ssh |
42 | 1f7d3f7d | René Nussbaumer | from ganeti import utils |
43 | 1f7d3f7d | René Nussbaumer | |
44 | 1f7d3f7d | René Nussbaumer | |
45 | 1f7d3f7d | René Nussbaumer | PAUSE_PERIOD_OPT = cli.cli_option("-p", "--watcher-pause-period", default=1800, |
46 | 1f7d3f7d | René Nussbaumer | action="store", type="int", |
47 | 1f7d3f7d | René Nussbaumer | dest="pause_period", |
48 | 1f7d3f7d | René Nussbaumer | help=("Amount of time in seconds watcher" |
49 | 1f7d3f7d | René Nussbaumer | " should be suspended from running")) |
50 | 8f44674f | Stephen Shirley | _CLUSTERMERGE_ECID = "clustermerge-ecid" |
51 | 1f7d3f7d | René Nussbaumer | |
52 | 1f7d3f7d | René Nussbaumer | |
53 | 71bbe910 | Stephen Shirley | def Flatten(unflattened_list): |
54 | 1f7d3f7d | René Nussbaumer | """Flattens a list. |
55 | 1f7d3f7d | René Nussbaumer | |
56 | 71bbe910 | Stephen Shirley | @param unflattened_list: A list of unflattened list objects. |
57 | 71bbe910 | Stephen Shirley | @return: A flattened list |
58 | 1f7d3f7d | René Nussbaumer | |
59 | 1f7d3f7d | René Nussbaumer | """ |
60 | 71bbe910 | Stephen Shirley | flattened_list = [] |
61 | 1f7d3f7d | René Nussbaumer | |
62 | 71bbe910 | Stephen Shirley | for item in unflattened_list: |
63 | 1f7d3f7d | René Nussbaumer | if isinstance(item, list): |
64 | 71bbe910 | Stephen Shirley | flattened_list.extend(Flatten(item)) |
65 | 1f7d3f7d | René Nussbaumer | else: |
66 | 71bbe910 | Stephen Shirley | flattened_list.append(item) |
67 | 71bbe910 | Stephen Shirley | return flattened_list |
68 | 1f7d3f7d | René Nussbaumer | |
69 | 1f7d3f7d | René Nussbaumer | |
70 | 1f7d3f7d | René Nussbaumer | class MergerData(object): |
71 | 1f7d3f7d | René Nussbaumer | """Container class to hold data used for merger. |
72 | 1f7d3f7d | René Nussbaumer | |
73 | 1f7d3f7d | René Nussbaumer | """ |
74 | 1f7d3f7d | René Nussbaumer | def __init__(self, cluster, key_path, nodes, instances, config_path=None): |
75 | 1f7d3f7d | René Nussbaumer | """Initialize the container. |
76 | 1f7d3f7d | René Nussbaumer | |
77 | 1f7d3f7d | René Nussbaumer | @param cluster: The name of the cluster |
78 | 1f7d3f7d | René Nussbaumer | @param key_path: Path to the ssh private key used for authentication |
79 | 1f7d3f7d | René Nussbaumer | @param nodes: List of nodes in the merging cluster |
80 | 1f7d3f7d | René Nussbaumer | @param instances: List of instances running on merging cluster |
81 | 3d2e7a27 | Stephen Shirley | @param config_path: Path to the merging cluster config |
82 | 1f7d3f7d | René Nussbaumer | |
83 | 1f7d3f7d | René Nussbaumer | """ |
84 | 1f7d3f7d | René Nussbaumer | self.cluster = cluster |
85 | 1f7d3f7d | René Nussbaumer | self.key_path = key_path |
86 | 1f7d3f7d | René Nussbaumer | self.nodes = nodes |
87 | 3d2e7a27 | Stephen Shirley | self.instances = instances |
88 | 3d2e7a27 | Stephen Shirley | self.config_path = config_path |
89 | 1f7d3f7d | René Nussbaumer | |
90 | 1f7d3f7d | René Nussbaumer | |
91 | 1f7d3f7d | René Nussbaumer | class Merger(object): |
92 | 1f7d3f7d | René Nussbaumer | """Handling the merge. |
93 | 1f7d3f7d | René Nussbaumer | |
94 | 1f7d3f7d | René Nussbaumer | """ |
95 | 1f7d3f7d | René Nussbaumer | def __init__(self, clusters, pause_period): |
96 | 1f7d3f7d | René Nussbaumer | """Initialize object with sane defaults and infos required. |
97 | 1f7d3f7d | René Nussbaumer | |
98 | 1f7d3f7d | René Nussbaumer | @param clusters: The list of clusters to merge in |
99 | 1f7d3f7d | René Nussbaumer | @param pause_period: The time watcher shall be disabled for |
100 | 1f7d3f7d | René Nussbaumer | |
101 | 1f7d3f7d | René Nussbaumer | """ |
102 | 1f7d3f7d | René Nussbaumer | self.merger_data = [] |
103 | 1f7d3f7d | René Nussbaumer | self.clusters = clusters |
104 | 1f7d3f7d | René Nussbaumer | self.pause_period = pause_period |
105 | 1f7d3f7d | René Nussbaumer | self.work_dir = tempfile.mkdtemp(suffix="cluster-merger") |
106 | be8aecab | Stephen Shirley | (self.cluster_name, ) = cli.GetClient().QueryConfigValues(["cluster_name"]) |
107 | 1f7d3f7d | René Nussbaumer | self.ssh_runner = ssh.SshRunner(self.cluster_name) |
108 | 1f7d3f7d | René Nussbaumer | |
109 | 1f7d3f7d | René Nussbaumer | def Setup(self): |
110 | 1f7d3f7d | René Nussbaumer | """Sets up our end so we can do the merger. |
111 | 1f7d3f7d | René Nussbaumer | |
112 | 1f7d3f7d | René Nussbaumer | This method is setting us up as a preparation for the merger. |
113 | 1f7d3f7d | René Nussbaumer | It makes the initial contact and gathers information needed. |
114 | 1f7d3f7d | René Nussbaumer | |
115 | 1f7d3f7d | René Nussbaumer | @raise errors.RemoteError: for errors in communication/grabbing |
116 | 1f7d3f7d | René Nussbaumer | |
117 | 1f7d3f7d | René Nussbaumer | """ |
118 | 1f7d3f7d | René Nussbaumer | (remote_path, _, _) = ssh.GetUserFiles("root") |
119 | 1f7d3f7d | René Nussbaumer | |
120 | be8aecab | Stephen Shirley | if self.cluster_name in self.clusters: |
121 | be8aecab | Stephen Shirley | raise errors.CommandError("Cannot merge cluster %s with itself" % |
122 | be8aecab | Stephen Shirley | self.cluster_name) |
123 | be8aecab | Stephen Shirley | |
124 | 1f7d3f7d | René Nussbaumer | # Fetch remotes private key |
125 | 1f7d3f7d | René Nussbaumer | for cluster in self.clusters: |
126 | 1f7d3f7d | René Nussbaumer | result = self._RunCmd(cluster, "cat %s" % remote_path, batch=False, |
127 | 1f7d3f7d | René Nussbaumer | ask_key=False) |
128 | 1f7d3f7d | René Nussbaumer | if result.failed: |
129 | 1f7d3f7d | René Nussbaumer | raise errors.RemoteError("There was an error while grabbing ssh private" |
130 | 1f7d3f7d | René Nussbaumer | " key from %s. Fail reason: %s; output: %s" % |
131 | 1f7d3f7d | René Nussbaumer | (cluster, result.fail_reason, result.output)) |
132 | 1f7d3f7d | René Nussbaumer | |
133 | c4feafe8 | Iustin Pop | key_path = utils.PathJoin(self.work_dir, cluster) |
134 | 1f7d3f7d | René Nussbaumer | utils.WriteFile(key_path, mode=0600, data=result.stdout) |
135 | 1f7d3f7d | René Nussbaumer | |
136 | 1f7d3f7d | René Nussbaumer | result = self._RunCmd(cluster, "gnt-node list -o name --no-header", |
137 | 1f7d3f7d | René Nussbaumer | private_key=key_path) |
138 | 1f7d3f7d | René Nussbaumer | if result.failed: |
139 | 1f7d3f7d | René Nussbaumer | raise errors.RemoteError("Unable to retrieve list of nodes from %s." |
140 | 1f7d3f7d | René Nussbaumer | " Fail reason: %s; output: %s" % |
141 | 1f7d3f7d | René Nussbaumer | (cluster, result.fail_reason, result.output)) |
142 | 1f7d3f7d | René Nussbaumer | nodes = result.stdout.splitlines() |
143 | 1f7d3f7d | René Nussbaumer | |
144 | 1f7d3f7d | René Nussbaumer | result = self._RunCmd(cluster, "gnt-instance list -o name --no-header", |
145 | 1f7d3f7d | René Nussbaumer | private_key=key_path) |
146 | 1f7d3f7d | René Nussbaumer | if result.failed: |
147 | 1f7d3f7d | René Nussbaumer | raise errors.RemoteError("Unable to retrieve list of instances from" |
148 | 1f7d3f7d | René Nussbaumer | " %s. Fail reason: %s; output: %s" % |
149 | 1f7d3f7d | René Nussbaumer | (cluster, result.fail_reason, result.output)) |
150 | 1f7d3f7d | René Nussbaumer | instances = result.stdout.splitlines() |
151 | 1f7d3f7d | René Nussbaumer | |
152 | 1f7d3f7d | René Nussbaumer | self.merger_data.append(MergerData(cluster, key_path, nodes, instances)) |
153 | 1f7d3f7d | René Nussbaumer | |
154 | 1f7d3f7d | René Nussbaumer | def _PrepareAuthorizedKeys(self): |
155 | 1f7d3f7d | René Nussbaumer | """Prepare the authorized_keys on every merging node. |
156 | 1f7d3f7d | René Nussbaumer | |
157 | 1f7d3f7d | René Nussbaumer | This method add our public key to remotes authorized_key for further |
158 | 1f7d3f7d | René Nussbaumer | communication. |
159 | 1f7d3f7d | René Nussbaumer | |
160 | 1f7d3f7d | René Nussbaumer | """ |
161 | 1f7d3f7d | René Nussbaumer | (_, pub_key_file, auth_keys) = ssh.GetUserFiles("root") |
162 | 1f7d3f7d | René Nussbaumer | pub_key = utils.ReadFile(pub_key_file) |
163 | 1f7d3f7d | René Nussbaumer | |
164 | 1f7d3f7d | René Nussbaumer | for data in self.merger_data: |
165 | 1f7d3f7d | René Nussbaumer | for node in data.nodes: |
166 | 1f7d3f7d | René Nussbaumer | result = self._RunCmd(node, ("cat >> %s << '!EOF.'\n%s!EOF.\n" % |
167 | 1f7d3f7d | René Nussbaumer | (auth_keys, pub_key)), |
168 | 1f7d3f7d | René Nussbaumer | private_key=data.key_path) |
169 | 1f7d3f7d | René Nussbaumer | |
170 | 1f7d3f7d | René Nussbaumer | if result.failed: |
171 | 1f7d3f7d | René Nussbaumer | raise errors.RemoteError("Unable to add our public key to %s in %s." |
172 | 1f7d3f7d | René Nussbaumer | " Fail reason: %s; output: %s" % |
173 | 1f7d3f7d | René Nussbaumer | (node, data.cluster, result.fail_reason, |
174 | 1f7d3f7d | René Nussbaumer | result.output)) |
175 | 1f7d3f7d | René Nussbaumer | |
176 | 1f7d3f7d | René Nussbaumer | def _RunCmd(self, hostname, command, user="root", use_cluster_key=False, |
177 | 1f7d3f7d | René Nussbaumer | strict_host_check=False, private_key=None, batch=True, |
178 | 1f7d3f7d | René Nussbaumer | ask_key=False): |
179 | 1f7d3f7d | René Nussbaumer | """Wrapping SshRunner.Run with default parameters. |
180 | 1f7d3f7d | René Nussbaumer | |
181 | 454723b5 | Iustin Pop | For explanation of parameters see L{ganeti.ssh.SshRunner.Run}. |
182 | 1f7d3f7d | René Nussbaumer | |
183 | 1f7d3f7d | René Nussbaumer | """ |
184 | 1f7d3f7d | René Nussbaumer | return self.ssh_runner.Run(hostname=hostname, command=command, user=user, |
185 | 1f7d3f7d | René Nussbaumer | use_cluster_key=use_cluster_key, |
186 | 1f7d3f7d | René Nussbaumer | strict_host_check=strict_host_check, |
187 | 1f7d3f7d | René Nussbaumer | private_key=private_key, batch=batch, |
188 | 1f7d3f7d | René Nussbaumer | ask_key=ask_key) |
189 | 1f7d3f7d | René Nussbaumer | |
190 | 1f7d3f7d | René Nussbaumer | def _StopMergingInstances(self): |
191 | 1f7d3f7d | René Nussbaumer | """Stop instances on merging clusters. |
192 | 1f7d3f7d | René Nussbaumer | |
193 | 1f7d3f7d | René Nussbaumer | """ |
194 | 1f7d3f7d | René Nussbaumer | for cluster in self.clusters: |
195 | 1f7d3f7d | René Nussbaumer | result = self._RunCmd(cluster, "gnt-instance shutdown --all" |
196 | 1f7d3f7d | René Nussbaumer | " --force-multiple") |
197 | 1f7d3f7d | René Nussbaumer | |
198 | 1f7d3f7d | René Nussbaumer | if result.failed: |
199 | 1f7d3f7d | René Nussbaumer | raise errors.RemoteError("Unable to stop instances on %s." |
200 | 1f7d3f7d | René Nussbaumer | " Fail reason: %s; output: %s" % |
201 | 1f7d3f7d | René Nussbaumer | (cluster, result.fail_reason, result.output)) |
202 | 1f7d3f7d | René Nussbaumer | |
203 | 1f7d3f7d | René Nussbaumer | def _DisableWatcher(self): |
204 | 1f7d3f7d | René Nussbaumer | """Disable watch on all merging clusters, including ourself. |
205 | 1f7d3f7d | René Nussbaumer | |
206 | 1f7d3f7d | René Nussbaumer | """ |
207 | 1f7d3f7d | René Nussbaumer | for cluster in ["localhost"] + self.clusters: |
208 | 1f7d3f7d | René Nussbaumer | result = self._RunCmd(cluster, "gnt-cluster watcher pause %d" % |
209 | 1f7d3f7d | René Nussbaumer | self.pause_period) |
210 | 1f7d3f7d | René Nussbaumer | |
211 | 1f7d3f7d | René Nussbaumer | if result.failed: |
212 | 1f7d3f7d | René Nussbaumer | raise errors.RemoteError("Unable to pause watcher on %s." |
213 | 1f7d3f7d | René Nussbaumer | " Fail reason: %s; output: %s" % |
214 | 1f7d3f7d | René Nussbaumer | (cluster, result.fail_reason, result.output)) |
215 | 1f7d3f7d | René Nussbaumer | |
216 | 1f7d3f7d | René Nussbaumer | def _StopDaemons(self): |
217 | 1f7d3f7d | René Nussbaumer | """Stop all daemons on merging nodes. |
218 | 1f7d3f7d | René Nussbaumer | |
219 | 1f7d3f7d | René Nussbaumer | """ |
220 | d8aab233 | René Nussbaumer | cmd = "%s stop-all" % constants.DAEMON_UTIL |
221 | 1f7d3f7d | René Nussbaumer | for data in self.merger_data: |
222 | 1f7d3f7d | René Nussbaumer | for node in data.nodes: |
223 | d8aab233 | René Nussbaumer | result = self._RunCmd(node, cmd) |
224 | 1f7d3f7d | René Nussbaumer | |
225 | 1f7d3f7d | René Nussbaumer | if result.failed: |
226 | 1f7d3f7d | René Nussbaumer | raise errors.RemoteError("Unable to stop daemons on %s." |
227 | 1f7d3f7d | René Nussbaumer | " Fail reason: %s; output: %s." % |
228 | 1f7d3f7d | René Nussbaumer | (node, result.fail_reason, result.output)) |
229 | 1f7d3f7d | René Nussbaumer | |
230 | 1f7d3f7d | René Nussbaumer | def _FetchRemoteConfig(self): |
231 | 1f7d3f7d | René Nussbaumer | """Fetches and stores remote cluster config from the master. |
232 | 1f7d3f7d | René Nussbaumer | |
233 | 1f7d3f7d | René Nussbaumer | This step is needed before we can merge the config. |
234 | 1f7d3f7d | René Nussbaumer | |
235 | 1f7d3f7d | René Nussbaumer | """ |
236 | 1f7d3f7d | René Nussbaumer | for data in self.merger_data: |
237 | 1f7d3f7d | René Nussbaumer | result = self._RunCmd(data.cluster, "cat %s" % |
238 | 1f7d3f7d | René Nussbaumer | constants.CLUSTER_CONF_FILE) |
239 | 1f7d3f7d | René Nussbaumer | |
240 | 1f7d3f7d | René Nussbaumer | if result.failed: |
241 | 1f7d3f7d | René Nussbaumer | raise errors.RemoteError("Unable to retrieve remote config on %s." |
242 | 1f7d3f7d | René Nussbaumer | " Fail reason: %s; output %s" % |
243 | 1f7d3f7d | René Nussbaumer | (data.cluster, result.fail_reason, |
244 | 1f7d3f7d | René Nussbaumer | result.output)) |
245 | 1f7d3f7d | René Nussbaumer | |
246 | c4feafe8 | Iustin Pop | data.config_path = utils.PathJoin(self.work_dir, "%s_config.data" % |
247 | c4feafe8 | Iustin Pop | data.cluster) |
248 | 1f7d3f7d | René Nussbaumer | utils.WriteFile(data.config_path, data=result.stdout) |
249 | 1f7d3f7d | René Nussbaumer | |
250 | 1f7d3f7d | René Nussbaumer | # R0201: Method could be a function |
251 | 1f7d3f7d | René Nussbaumer | def _KillMasterDaemon(self): # pylint: disable-msg=R0201 |
252 | 1f7d3f7d | René Nussbaumer | """Kills the local master daemon. |
253 | 1f7d3f7d | René Nussbaumer | |
254 | 1f7d3f7d | René Nussbaumer | @raise errors.CommandError: If unable to kill |
255 | 1f7d3f7d | René Nussbaumer | |
256 | 1f7d3f7d | René Nussbaumer | """ |
257 | 1f7d3f7d | René Nussbaumer | result = utils.RunCmd([constants.DAEMON_UTIL, "stop-master"]) |
258 | 1f7d3f7d | René Nussbaumer | if result.failed: |
259 | 1f7d3f7d | René Nussbaumer | raise errors.CommandError("Unable to stop master daemons." |
260 | 1f7d3f7d | René Nussbaumer | " Fail reason: %s; output: %s" % |
261 | 1f7d3f7d | René Nussbaumer | (result.fail_reason, result.output)) |
262 | 1f7d3f7d | René Nussbaumer | |
263 | 1f7d3f7d | René Nussbaumer | def _MergeConfig(self): |
264 | 1f7d3f7d | René Nussbaumer | """Merges all foreign config into our own config. |
265 | 1f7d3f7d | René Nussbaumer | |
266 | 1f7d3f7d | René Nussbaumer | """ |
267 | 1f7d3f7d | René Nussbaumer | my_config = config.ConfigWriter(offline=True) |
268 | 1f7d3f7d | René Nussbaumer | fake_ec_id = 0 # Needs to be uniq over the whole config merge |
269 | 1f7d3f7d | René Nussbaumer | |
270 | 1f7d3f7d | René Nussbaumer | for data in self.merger_data: |
271 | e1ab08db | Stephen Shirley | other_config = config.ConfigWriter(data.config_path, accept_foreign=True) |
272 | 8f44674f | Stephen Shirley | self._MergeNodeGroups(my_config, other_config) |
273 | 1f7d3f7d | René Nussbaumer | |
274 | 1f7d3f7d | René Nussbaumer | for node in other_config.GetNodeList(): |
275 | 1f7d3f7d | René Nussbaumer | node_info = other_config.GetNodeInfo(node) |
276 | 1f7d3f7d | René Nussbaumer | node_info.master_candidate = False |
277 | 1f7d3f7d | René Nussbaumer | my_config.AddNode(node_info, str(fake_ec_id)) |
278 | 1f7d3f7d | René Nussbaumer | fake_ec_id += 1 |
279 | 1f7d3f7d | René Nussbaumer | |
280 | 1f7d3f7d | René Nussbaumer | for instance in other_config.GetInstanceList(): |
281 | 1f7d3f7d | René Nussbaumer | instance_info = other_config.GetInstanceInfo(instance) |
282 | 1f7d3f7d | René Nussbaumer | |
283 | 1f7d3f7d | René Nussbaumer | # Update the DRBD port assignments |
284 | 1f7d3f7d | René Nussbaumer | # This is a little bit hackish |
285 | 1f7d3f7d | René Nussbaumer | for dsk in instance_info.disks: |
286 | 1f7d3f7d | René Nussbaumer | if dsk.dev_type in constants.LDS_DRBD: |
287 | 1f7d3f7d | René Nussbaumer | port = my_config.AllocatePort() |
288 | 1f7d3f7d | René Nussbaumer | |
289 | 1f7d3f7d | René Nussbaumer | logical_id = list(dsk.logical_id) |
290 | 1f7d3f7d | René Nussbaumer | logical_id[2] = port |
291 | 1f7d3f7d | René Nussbaumer | dsk.logical_id = tuple(logical_id) |
292 | 1f7d3f7d | René Nussbaumer | |
293 | 1f7d3f7d | René Nussbaumer | physical_id = list(dsk.physical_id) |
294 | 1f7d3f7d | René Nussbaumer | physical_id[1] = physical_id[3] = port |
295 | 1f7d3f7d | René Nussbaumer | dsk.physical_id = tuple(physical_id) |
296 | 1f7d3f7d | René Nussbaumer | |
297 | 1f7d3f7d | René Nussbaumer | my_config.AddInstance(instance_info, str(fake_ec_id)) |
298 | 1f7d3f7d | René Nussbaumer | fake_ec_id += 1 |
299 | 1f7d3f7d | René Nussbaumer | |
300 | 1f7d3f7d | René Nussbaumer | # R0201: Method could be a function |
301 | 8f44674f | Stephen Shirley | def _MergeNodeGroups(self, my_config, other_config): |
302 | 8f44674f | Stephen Shirley | """Adds foreign node groups |
303 | 8f44674f | Stephen Shirley | |
304 | 8f44674f | Stephen Shirley | ConfigWriter.AddNodeGroup takes care of making sure there are no conflicts. |
305 | 8f44674f | Stephen Shirley | """ |
306 | 8f44674f | Stephen Shirley | # pylint: disable-msg=R0201 |
307 | 6323e232 | Stephen Shirley | for grp in other_config.GetAllNodeGroupsInfo().values(): |
308 | 8f44674f | Stephen Shirley | #TODO: handle node group conflicts |
309 | 8f44674f | Stephen Shirley | my_config.AddNodeGroup(grp, _CLUSTERMERGE_ECID) |
310 | 8f44674f | Stephen Shirley | |
311 | 8f44674f | Stephen Shirley | # R0201: Method could be a function |
312 | 1f7d3f7d | René Nussbaumer | def _StartMasterDaemon(self, no_vote=False): # pylint: disable-msg=R0201 |
313 | 1f7d3f7d | René Nussbaumer | """Starts the local master daemon. |
314 | 1f7d3f7d | René Nussbaumer | |
315 | 1f7d3f7d | René Nussbaumer | @param no_vote: Should the masterd started without voting? default: False |
316 | 1f7d3f7d | René Nussbaumer | @raise errors.CommandError: If unable to start daemon. |
317 | 1f7d3f7d | René Nussbaumer | |
318 | 1f7d3f7d | René Nussbaumer | """ |
319 | 1f7d3f7d | René Nussbaumer | env = {} |
320 | 1f7d3f7d | René Nussbaumer | if no_vote: |
321 | 1f7d3f7d | René Nussbaumer | env["EXTRA_MASTERD_ARGS"] = "--no-voting --yes-do-it" |
322 | 1f7d3f7d | René Nussbaumer | |
323 | 1f7d3f7d | René Nussbaumer | result = utils.RunCmd([constants.DAEMON_UTIL, "start-master"], env=env) |
324 | 1f7d3f7d | René Nussbaumer | if result.failed: |
325 | 1f7d3f7d | René Nussbaumer | raise errors.CommandError("Couldn't start ganeti master." |
326 | 1f7d3f7d | René Nussbaumer | " Fail reason: %s; output: %s" % |
327 | 1f7d3f7d | René Nussbaumer | (result.fail_reason, result.output)) |
328 | 1f7d3f7d | René Nussbaumer | |
329 | 1f7d3f7d | René Nussbaumer | def _ReaddMergedNodesAndRedist(self): |
330 | 1f7d3f7d | René Nussbaumer | """Readds all merging nodes and make sure their config is up-to-date. |
331 | 1f7d3f7d | René Nussbaumer | |
332 | 1f7d3f7d | René Nussbaumer | @raise errors.CommandError: If anything fails. |
333 | 1f7d3f7d | René Nussbaumer | |
334 | 1f7d3f7d | René Nussbaumer | """ |
335 | 1f7d3f7d | René Nussbaumer | for data in self.merger_data: |
336 | 1f7d3f7d | René Nussbaumer | for node in data.nodes: |
337 | 1f7d3f7d | René Nussbaumer | result = utils.RunCmd(["gnt-node", "add", "--readd", |
338 | 53991408 | Stephen Shirley | "--no-ssh-key-check", "--force-join", node]) |
339 | 1f7d3f7d | René Nussbaumer | if result.failed: |
340 | 1f7d3f7d | René Nussbaumer | raise errors.CommandError("Couldn't readd node %s. Fail reason: %s;" |
341 | 1f7d3f7d | René Nussbaumer | " output: %s" % (node, result.fail_reason, |
342 | 1f7d3f7d | René Nussbaumer | result.output)) |
343 | 1f7d3f7d | René Nussbaumer | |
344 | 1f7d3f7d | René Nussbaumer | result = utils.RunCmd(["gnt-cluster", "redist-conf"]) |
345 | 1f7d3f7d | René Nussbaumer | if result.failed: |
346 | 1f7d3f7d | René Nussbaumer | raise errors.CommandError("Redistribution failed. Fail reason: %s;" |
347 | 1f7d3f7d | René Nussbaumer | " output: %s" % (result.fail_reason, |
348 | 1f7d3f7d | René Nussbaumer | result.output)) |
349 | 1f7d3f7d | René Nussbaumer | |
350 | 1f7d3f7d | René Nussbaumer | # R0201: Method could be a function |
351 | 1f7d3f7d | René Nussbaumer | def _StartupAllInstances(self): # pylint: disable-msg=R0201 |
352 | 1f7d3f7d | René Nussbaumer | """Starts up all instances (locally). |
353 | 1f7d3f7d | René Nussbaumer | |
354 | 1f7d3f7d | René Nussbaumer | @raise errors.CommandError: If unable to start clusters |
355 | 1f7d3f7d | René Nussbaumer | |
356 | 1f7d3f7d | René Nussbaumer | """ |
357 | 1f7d3f7d | René Nussbaumer | result = utils.RunCmd(["gnt-instance", "startup", "--all", |
358 | 1f7d3f7d | René Nussbaumer | "--force-multiple"]) |
359 | 1f7d3f7d | René Nussbaumer | if result.failed: |
360 | 1f7d3f7d | René Nussbaumer | raise errors.CommandError("Unable to start all instances." |
361 | 1f7d3f7d | René Nussbaumer | " Fail reason: %s; output: %s" % |
362 | 1f7d3f7d | René Nussbaumer | (result.fail_reason, result.output)) |
363 | 1f7d3f7d | René Nussbaumer | |
364 | 1f7d3f7d | René Nussbaumer | # R0201: Method could be a function |
365 | 1f7d3f7d | René Nussbaumer | def _VerifyCluster(self): # pylint: disable-msg=R0201 |
366 | 1f7d3f7d | René Nussbaumer | """Runs gnt-cluster verify to verify the health. |
367 | 1f7d3f7d | René Nussbaumer | |
368 | 1f7d3f7d | René Nussbaumer | @raise errors.ProgrammError: If cluster fails on verification |
369 | 1f7d3f7d | René Nussbaumer | |
370 | 1f7d3f7d | René Nussbaumer | """ |
371 | 1f7d3f7d | René Nussbaumer | result = utils.RunCmd(["gnt-cluster", "verify"]) |
372 | 1f7d3f7d | René Nussbaumer | if result.failed: |
373 | 1f7d3f7d | René Nussbaumer | raise errors.CommandError("Verification of cluster failed." |
374 | 1f7d3f7d | René Nussbaumer | " Fail reason: %s; output: %s" % |
375 | 1f7d3f7d | René Nussbaumer | (result.fail_reason, result.output)) |
376 | 1f7d3f7d | René Nussbaumer | |
377 | 1f7d3f7d | René Nussbaumer | def Merge(self): |
378 | 1f7d3f7d | René Nussbaumer | """Does the actual merge. |
379 | 1f7d3f7d | René Nussbaumer | |
380 | 1f7d3f7d | René Nussbaumer | It runs all the steps in the right order and updates the user about steps |
381 | 1f7d3f7d | René Nussbaumer | taken. Also it keeps track of rollback_steps to undo everything. |
382 | 1f7d3f7d | René Nussbaumer | |
383 | 1f7d3f7d | René Nussbaumer | """ |
384 | 1f7d3f7d | René Nussbaumer | rbsteps = [] |
385 | 1f7d3f7d | René Nussbaumer | try: |
386 | 1f7d3f7d | René Nussbaumer | logging.info("Pre cluster verification") |
387 | 1f7d3f7d | René Nussbaumer | self._VerifyCluster() |
388 | 1f7d3f7d | René Nussbaumer | |
389 | 1f7d3f7d | René Nussbaumer | logging.info("Prepare authorized_keys") |
390 | 1f7d3f7d | René Nussbaumer | rbsteps.append("Remove our key from authorized_keys on nodes:" |
391 | 1f7d3f7d | René Nussbaumer | " %(nodes)s") |
392 | 1f7d3f7d | René Nussbaumer | self._PrepareAuthorizedKeys() |
393 | 1f7d3f7d | René Nussbaumer | |
394 | 1f7d3f7d | René Nussbaumer | rbsteps.append("Start all instances again on the merging" |
395 | 1f7d3f7d | René Nussbaumer | " clusters: %(clusters)s") |
396 | 1f7d3f7d | René Nussbaumer | logging.info("Stopping merging instances (takes a while)") |
397 | 1f7d3f7d | René Nussbaumer | self._StopMergingInstances() |
398 | 1f7d3f7d | René Nussbaumer | |
399 | 1f7d3f7d | René Nussbaumer | logging.info("Disable watcher") |
400 | 1f7d3f7d | René Nussbaumer | self._DisableWatcher() |
401 | 1f7d3f7d | René Nussbaumer | logging.info("Stop daemons on merging nodes") |
402 | 1f7d3f7d | René Nussbaumer | self._StopDaemons() |
403 | 1f7d3f7d | René Nussbaumer | logging.info("Merging config") |
404 | 1f7d3f7d | René Nussbaumer | self._FetchRemoteConfig() |
405 | d8aab233 | René Nussbaumer | |
406 | 9c03a67a | Stephen Shirley | logging.info("Stopping master daemon") |
407 | 9c03a67a | Stephen Shirley | self._KillMasterDaemon() |
408 | 9c03a67a | Stephen Shirley | |
409 | 9c03a67a | Stephen Shirley | rbsteps.append("Restore %s from another master candidate" |
410 | 9c03a67a | Stephen Shirley | " and restart master daemon" % |
411 | 9c03a67a | Stephen Shirley | constants.CLUSTER_CONF_FILE) |
412 | 9c03a67a | Stephen Shirley | self._MergeConfig() |
413 | 9c03a67a | Stephen Shirley | self._StartMasterDaemon(no_vote=True) |
414 | 9c03a67a | Stephen Shirley | |
415 | 9c03a67a | Stephen Shirley | # Point of no return, delete rbsteps |
416 | 9c03a67a | Stephen Shirley | del rbsteps[:] |
417 | 9c03a67a | Stephen Shirley | |
418 | 9c03a67a | Stephen Shirley | logging.warning("We are at the point of no return. Merge can not easily" |
419 | 9c03a67a | Stephen Shirley | " be undone after this point.") |
420 | 9c03a67a | Stephen Shirley | logging.info("Readd nodes") |
421 | 9c03a67a | Stephen Shirley | self._ReaddMergedNodesAndRedist() |
422 | 9c03a67a | Stephen Shirley | |
423 | 9c03a67a | Stephen Shirley | logging.info("Merge done, restart master daemon normally") |
424 | 9c03a67a | Stephen Shirley | self._KillMasterDaemon() |
425 | 9c03a67a | Stephen Shirley | self._StartMasterDaemon() |
426 | d8aab233 | René Nussbaumer | |
427 | 1f7d3f7d | René Nussbaumer | logging.info("Starting instances again") |
428 | 1f7d3f7d | René Nussbaumer | self._StartupAllInstances() |
429 | 1f7d3f7d | René Nussbaumer | logging.info("Post cluster verification") |
430 | 1f7d3f7d | René Nussbaumer | self._VerifyCluster() |
431 | 1f7d3f7d | René Nussbaumer | except errors.GenericError, e: |
432 | 1f7d3f7d | René Nussbaumer | logging.exception(e) |
433 | 1f7d3f7d | René Nussbaumer | |
434 | 1f7d3f7d | René Nussbaumer | if rbsteps: |
435 | 1f7d3f7d | René Nussbaumer | nodes = Flatten([data.nodes for data in self.merger_data]) |
436 | 1f7d3f7d | René Nussbaumer | info = { |
437 | 1f7d3f7d | René Nussbaumer | "clusters": self.clusters, |
438 | 1f7d3f7d | René Nussbaumer | "nodes": nodes, |
439 | 1f7d3f7d | René Nussbaumer | } |
440 | 1f7d3f7d | René Nussbaumer | logging.critical("In order to rollback do the following:") |
441 | 1f7d3f7d | René Nussbaumer | for step in rbsteps: |
442 | 3f1cf151 | René Nussbaumer | logging.critical(" * %s", step % info) |
443 | 1f7d3f7d | René Nussbaumer | else: |
444 | 1f7d3f7d | René Nussbaumer | logging.critical("Nothing to rollback.") |
445 | 1f7d3f7d | René Nussbaumer | |
446 | 1f7d3f7d | René Nussbaumer | # TODO: Keep track of steps done for a flawless resume? |
447 | 1f7d3f7d | René Nussbaumer | |
448 | 1f7d3f7d | René Nussbaumer | def Cleanup(self): |
449 | 1f7d3f7d | René Nussbaumer | """Clean up our environment. |
450 | 1f7d3f7d | René Nussbaumer | |
451 | 1f7d3f7d | René Nussbaumer | This cleans up remote private keys and configs and after that |
452 | 1f7d3f7d | René Nussbaumer | deletes the temporary directory. |
453 | 1f7d3f7d | René Nussbaumer | |
454 | 1f7d3f7d | René Nussbaumer | """ |
455 | 1f7d3f7d | René Nussbaumer | shutil.rmtree(self.work_dir) |
456 | 1f7d3f7d | René Nussbaumer | |
457 | 1f7d3f7d | René Nussbaumer | |
458 | 1f7d3f7d | René Nussbaumer | def SetupLogging(options): |
459 | 1f7d3f7d | René Nussbaumer | """Setting up logging infrastructure. |
460 | 1f7d3f7d | René Nussbaumer | |
461 | 1f7d3f7d | René Nussbaumer | @param options: Parsed command line options |
462 | 1f7d3f7d | René Nussbaumer | |
463 | 1f7d3f7d | René Nussbaumer | """ |
464 | 1f7d3f7d | René Nussbaumer | formatter = logging.Formatter("%(asctime)s: %(levelname)s %(message)s") |
465 | 1f7d3f7d | René Nussbaumer | |
466 | 1f7d3f7d | René Nussbaumer | stderr_handler = logging.StreamHandler() |
467 | 1f7d3f7d | René Nussbaumer | stderr_handler.setFormatter(formatter) |
468 | 1f7d3f7d | René Nussbaumer | if options.debug: |
469 | 1f7d3f7d | René Nussbaumer | stderr_handler.setLevel(logging.NOTSET) |
470 | 1f7d3f7d | René Nussbaumer | elif options.verbose: |
471 | 1f7d3f7d | René Nussbaumer | stderr_handler.setLevel(logging.INFO) |
472 | 1f7d3f7d | René Nussbaumer | else: |
473 | 1f7d3f7d | René Nussbaumer | stderr_handler.setLevel(logging.ERROR) |
474 | 1f7d3f7d | René Nussbaumer | |
475 | 1f7d3f7d | René Nussbaumer | root_logger = logging.getLogger("") |
476 | 1f7d3f7d | René Nussbaumer | root_logger.setLevel(logging.NOTSET) |
477 | 1f7d3f7d | René Nussbaumer | root_logger.addHandler(stderr_handler) |
478 | 1f7d3f7d | René Nussbaumer | |
479 | 1f7d3f7d | René Nussbaumer | |
480 | 1f7d3f7d | René Nussbaumer | def main(): |
481 | 1f7d3f7d | René Nussbaumer | """Main routine. |
482 | 1f7d3f7d | René Nussbaumer | |
483 | 1f7d3f7d | René Nussbaumer | """ |
484 | 1f7d3f7d | René Nussbaumer | program = os.path.basename(sys.argv[0]) |
485 | 1f7d3f7d | René Nussbaumer | |
486 | 1f7d3f7d | René Nussbaumer | parser = optparse.OptionParser(usage=("%prog [--debug|--verbose]" |
487 | 1f7d3f7d | René Nussbaumer | " [--watcher-pause-period SECONDS]" |
488 | 1f7d3f7d | René Nussbaumer | " <cluster> <cluster...>"), |
489 | 1f7d3f7d | René Nussbaumer | prog=program) |
490 | 1f7d3f7d | René Nussbaumer | parser.add_option(cli.DEBUG_OPT) |
491 | 1f7d3f7d | René Nussbaumer | parser.add_option(cli.VERBOSE_OPT) |
492 | 1f7d3f7d | René Nussbaumer | parser.add_option(PAUSE_PERIOD_OPT) |
493 | 1f7d3f7d | René Nussbaumer | |
494 | 1f7d3f7d | René Nussbaumer | (options, args) = parser.parse_args() |
495 | 1f7d3f7d | René Nussbaumer | |
496 | 1f7d3f7d | René Nussbaumer | SetupLogging(options) |
497 | 1f7d3f7d | René Nussbaumer | |
498 | 1f7d3f7d | René Nussbaumer | if not args: |
499 | 1f7d3f7d | René Nussbaumer | parser.error("No clusters specified") |
500 | 1f7d3f7d | René Nussbaumer | |
501 | 1f7d3f7d | René Nussbaumer | cluster_merger = Merger(utils.UniqueSequence(args), options.pause_period) |
502 | 1f7d3f7d | René Nussbaumer | try: |
503 | 1f7d3f7d | René Nussbaumer | try: |
504 | 1f7d3f7d | René Nussbaumer | cluster_merger.Setup() |
505 | 1f7d3f7d | René Nussbaumer | cluster_merger.Merge() |
506 | 1f7d3f7d | René Nussbaumer | except errors.GenericError, e: |
507 | 1f7d3f7d | René Nussbaumer | logging.exception(e) |
508 | 1f7d3f7d | René Nussbaumer | return constants.EXIT_FAILURE |
509 | 1f7d3f7d | René Nussbaumer | finally: |
510 | 1f7d3f7d | René Nussbaumer | cluster_merger.Cleanup() |
511 | 1f7d3f7d | René Nussbaumer | |
512 | 1f7d3f7d | René Nussbaumer | return constants.EXIT_SUCCESS |
513 | 1f7d3f7d | René Nussbaumer | |
514 | 1f7d3f7d | René Nussbaumer | |
515 | 1f7d3f7d | René Nussbaumer | if __name__ == "__main__": |
516 | 1f7d3f7d | René Nussbaumer | sys.exit(main()) |