root / tools / cluster-merge @ 0c009cc5
History | View | Annotate | Download (31.7 kB)
1 | 1f7d3f7d | René Nussbaumer | #!/usr/bin/python |
---|---|---|---|
2 | 1f7d3f7d | René Nussbaumer | # |
3 | 1f7d3f7d | René Nussbaumer | |
4 | 3d8f154f | Stephen Shirley | # Copyright (C) 2010 Google Inc. |
5 | 1f7d3f7d | René Nussbaumer | # |
6 | 1f7d3f7d | René Nussbaumer | # This program is free software; you can redistribute it and/or modify |
7 | 1f7d3f7d | René Nussbaumer | # it under the terms of the GNU General Public License as published by |
8 | 1f7d3f7d | René Nussbaumer | # the Free Software Foundation; either version 2 of the License, or |
9 | 1f7d3f7d | René Nussbaumer | # (at your option) any later version. |
10 | 1f7d3f7d | René Nussbaumer | # |
11 | 1f7d3f7d | René Nussbaumer | # This program is distributed in the hope that it will be useful, but |
12 | 1f7d3f7d | René Nussbaumer | # WITHOUT ANY WARRANTY; without even the implied warranty of |
13 | 1f7d3f7d | René Nussbaumer | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU |
14 | 1f7d3f7d | René Nussbaumer | # General Public License for more details. |
15 | 1f7d3f7d | René Nussbaumer | # |
16 | 1f7d3f7d | René Nussbaumer | # You should have received a copy of the GNU General Public License |
17 | 1f7d3f7d | René Nussbaumer | # along with this program; if not, write to the Free Software |
18 | 1f7d3f7d | René Nussbaumer | # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA |
19 | 1f7d3f7d | René Nussbaumer | # 02110-1301, USA. |
20 | 1f7d3f7d | René Nussbaumer | |
21 | 1f7d3f7d | René Nussbaumer | """Tool to merge two or more clusters together. |
22 | 1f7d3f7d | René Nussbaumer | |
23 | 1f7d3f7d | René Nussbaumer | The clusters have to run the same version of Ganeti! |
24 | 1f7d3f7d | René Nussbaumer | |
25 | 1f7d3f7d | René Nussbaumer | """ |
26 | 1f7d3f7d | René Nussbaumer | |
27 | b459a848 | Andrea Spadaccini | # pylint: disable=C0103 |
28 | 1f7d3f7d | René Nussbaumer | # C0103: Invalid name cluster-merge |
29 | 1f7d3f7d | René Nussbaumer | |
30 | 1f7d3f7d | René Nussbaumer | import logging |
31 | 1f7d3f7d | René Nussbaumer | import os |
32 | 1f7d3f7d | René Nussbaumer | import optparse |
33 | 1f7d3f7d | René Nussbaumer | import shutil |
34 | 1f7d3f7d | René Nussbaumer | import sys |
35 | 1f7d3f7d | René Nussbaumer | import tempfile |
36 | 1f7d3f7d | René Nussbaumer | |
37 | 1f7d3f7d | René Nussbaumer | from ganeti import cli |
38 | 1f7d3f7d | René Nussbaumer | from ganeti import config |
39 | 1f7d3f7d | René Nussbaumer | from ganeti import constants |
40 | 1f7d3f7d | René Nussbaumer | from ganeti import errors |
41 | 1f7d3f7d | René Nussbaumer | from ganeti import ssh |
42 | 1f7d3f7d | René Nussbaumer | from ganeti import utils |
43 | b972c223 | Andrea Spadaccini | from ganeti import netutils |
44 | 1f7d3f7d | René Nussbaumer | |
45 | 1f7d3f7d | René Nussbaumer | |
46 | 1a615be0 | Stephen Shirley | _GROUPS_MERGE = "merge" |
47 | 1a615be0 | Stephen Shirley | _GROUPS_RENAME = "rename" |
48 | 1a615be0 | Stephen Shirley | _CLUSTERMERGE_ECID = "clustermerge-ecid" |
49 | 620a9c62 | Guido Trotter | _RESTART_ALL = "all" |
50 | 620a9c62 | Guido Trotter | _RESTART_UP = "up" |
51 | 620a9c62 | Guido Trotter | _RESTART_NONE = "none" |
52 | 620a9c62 | Guido Trotter | _RESTART_CHOICES = (_RESTART_ALL, _RESTART_UP, _RESTART_NONE) |
53 | 1fcd3b81 | Guido Trotter | _PARAMS_STRICT = "strict" |
54 | 1fcd3b81 | Guido Trotter | _PARAMS_WARN = "warn" |
55 | 1fcd3b81 | Guido Trotter | _PARAMS_CHOICES = (_PARAMS_STRICT, _PARAMS_WARN) |
56 | 620a9c62 | Guido Trotter | |
57 | 1a615be0 | Stephen Shirley | |
58 | 1f7d3f7d | René Nussbaumer | PAUSE_PERIOD_OPT = cli.cli_option("-p", "--watcher-pause-period", default=1800, |
59 | 1f7d3f7d | René Nussbaumer | action="store", type="int", |
60 | 1f7d3f7d | René Nussbaumer | dest="pause_period", |
61 | 1f7d3f7d | René Nussbaumer | help=("Amount of time in seconds watcher" |
62 | 1f7d3f7d | René Nussbaumer | " should be suspended from running")) |
63 | 1a615be0 | Stephen Shirley | GROUPS_OPT = cli.cli_option("--groups", default=None, metavar="STRATEGY", |
64 | 9b945588 | Stephen Shirley | choices=(_GROUPS_MERGE, _GROUPS_RENAME), |
65 | 9b945588 | Stephen Shirley | dest="groups", |
66 | 1a615be0 | Stephen Shirley | help=("How to handle groups that have the" |
67 | 1a615be0 | Stephen Shirley | " same name (One of: %s/%s)" % |
68 | 1a615be0 | Stephen Shirley | (_GROUPS_MERGE, _GROUPS_RENAME))) |
69 | 1fcd3b81 | Guido Trotter | PARAMS_OPT = cli.cli_option("--parameter-conflicts", default=_PARAMS_STRICT, |
70 | 1fcd3b81 | Guido Trotter | metavar="STRATEGY", |
71 | 1fcd3b81 | Guido Trotter | choices=_PARAMS_CHOICES, |
72 | 1fcd3b81 | Guido Trotter | dest="params", |
73 | 1fcd3b81 | Guido Trotter | help=("How to handle params that have" |
74 | 1fcd3b81 | Guido Trotter | " different values (One of: %s/%s)" % |
75 | 1fcd3b81 | Guido Trotter | _PARAMS_CHOICES)) |
76 | 1fcd3b81 | Guido Trotter | |
77 | 620a9c62 | Guido Trotter | RESTART_OPT = cli.cli_option("--restart", default=_RESTART_ALL, |
78 | 620a9c62 | Guido Trotter | metavar="STRATEGY", |
79 | 620a9c62 | Guido Trotter | choices=_RESTART_CHOICES, |
80 | 620a9c62 | Guido Trotter | dest="restart", |
81 | 620a9c62 | Guido Trotter | help=("How to handle restarting instances" |
82 | 620a9c62 | Guido Trotter | " same name (One of: %s/%s/%s)" % |
83 | 620a9c62 | Guido Trotter | _RESTART_CHOICES)) |
84 | 1f7d3f7d | René Nussbaumer | |
85 | b7bf8b58 | Guido Trotter | SKIP_STOP_INSTANCES_OPT = \ |
86 | b7bf8b58 | Guido Trotter | cli.cli_option("--skip-stop-instances", default=True, action="store_false", |
87 | b7bf8b58 | Guido Trotter | dest="stop_instances", |
88 | b7bf8b58 | Guido Trotter | help=("Don't stop the instances on the clusters, just check " |
89 | b7bf8b58 | Guido Trotter | "that none is running")) |
90 | 1fb5f905 | Andrea Spadaccini | |
91 | 1f7d3f7d | René Nussbaumer | |
92 | 71bbe910 | Stephen Shirley | def Flatten(unflattened_list): |
93 | 1f7d3f7d | René Nussbaumer | """Flattens a list. |
94 | 1f7d3f7d | René Nussbaumer | |
95 | 71bbe910 | Stephen Shirley | @param unflattened_list: A list of unflattened list objects. |
96 | 71bbe910 | Stephen Shirley | @return: A flattened list |
97 | 1f7d3f7d | René Nussbaumer | |
98 | 1f7d3f7d | René Nussbaumer | """ |
99 | 71bbe910 | Stephen Shirley | flattened_list = [] |
100 | 1f7d3f7d | René Nussbaumer | |
101 | 71bbe910 | Stephen Shirley | for item in unflattened_list: |
102 | 1f7d3f7d | René Nussbaumer | if isinstance(item, list): |
103 | 71bbe910 | Stephen Shirley | flattened_list.extend(Flatten(item)) |
104 | 1f7d3f7d | René Nussbaumer | else: |
105 | 71bbe910 | Stephen Shirley | flattened_list.append(item) |
106 | 71bbe910 | Stephen Shirley | return flattened_list |
107 | 1f7d3f7d | René Nussbaumer | |
108 | 1f7d3f7d | René Nussbaumer | |
109 | 1f7d3f7d | René Nussbaumer | class MergerData(object): |
110 | 1f7d3f7d | René Nussbaumer | """Container class to hold data used for merger. |
111 | 1f7d3f7d | René Nussbaumer | |
112 | 1f7d3f7d | René Nussbaumer | """ |
113 | b972c223 | Andrea Spadaccini | def __init__(self, cluster, key_path, nodes, instances, master_node, |
114 | b972c223 | Andrea Spadaccini | master_ip, config_path=None): |
115 | 1f7d3f7d | René Nussbaumer | """Initialize the container. |
116 | 1f7d3f7d | René Nussbaumer | |
117 | 1f7d3f7d | René Nussbaumer | @param cluster: The name of the cluster |
118 | 1f7d3f7d | René Nussbaumer | @param key_path: Path to the ssh private key used for authentication |
119 | 8697f0fa | Guido Trotter | @param nodes: List of online nodes in the merging cluster |
120 | 1f7d3f7d | René Nussbaumer | @param instances: List of instances running on merging cluster |
121 | b972c223 | Andrea Spadaccini | @param master_node: Name of the master node |
122 | b972c223 | Andrea Spadaccini | @param master_ip: Cluster IP |
123 | 3d2e7a27 | Stephen Shirley | @param config_path: Path to the merging cluster config |
124 | 1f7d3f7d | René Nussbaumer | |
125 | 1f7d3f7d | René Nussbaumer | """ |
126 | 1f7d3f7d | René Nussbaumer | self.cluster = cluster |
127 | 1f7d3f7d | René Nussbaumer | self.key_path = key_path |
128 | 1f7d3f7d | René Nussbaumer | self.nodes = nodes |
129 | 3d2e7a27 | Stephen Shirley | self.instances = instances |
130 | b972c223 | Andrea Spadaccini | self.master_node = master_node |
131 | b972c223 | Andrea Spadaccini | self.master_ip = master_ip |
132 | 3d2e7a27 | Stephen Shirley | self.config_path = config_path |
133 | 1f7d3f7d | René Nussbaumer | |
134 | 1f7d3f7d | René Nussbaumer | |
135 | 1f7d3f7d | René Nussbaumer | class Merger(object): |
136 | 1f7d3f7d | René Nussbaumer | """Handling the merge. |
137 | 1f7d3f7d | René Nussbaumer | |
138 | 1f7d3f7d | René Nussbaumer | """ |
139 | 1fb5f905 | Andrea Spadaccini | RUNNING_STATUSES = frozenset([ |
140 | 1fb5f905 | Andrea Spadaccini | constants.INSTST_RUNNING, |
141 | 1fb5f905 | Andrea Spadaccini | constants.INSTST_ERRORUP, |
142 | 1fb5f905 | Andrea Spadaccini | ]) |
143 | e687ec01 | Michael Hanselmann | |
144 | 1fb5f905 | Andrea Spadaccini | def __init__(self, clusters, pause_period, groups, restart, params, |
145 | 1fb5f905 | Andrea Spadaccini | stop_instances): |
146 | 1f7d3f7d | René Nussbaumer | """Initialize object with sane defaults and infos required. |
147 | 1f7d3f7d | René Nussbaumer | |
148 | 1f7d3f7d | René Nussbaumer | @param clusters: The list of clusters to merge in |
149 | 1f7d3f7d | René Nussbaumer | @param pause_period: The time watcher shall be disabled for |
150 | 1a615be0 | Stephen Shirley | @param groups: How to handle group conflicts |
151 | 620a9c62 | Guido Trotter | @param restart: How to handle instance restart |
152 | 1fb5f905 | Andrea Spadaccini | @param stop_instances: Indicates whether the instances must be stopped |
153 | 1fb5f905 | Andrea Spadaccini | (True) or if the Merger must only check if no |
154 | 1fb5f905 | Andrea Spadaccini | instances are running on the mergee clusters (False) |
155 | 1f7d3f7d | René Nussbaumer | |
156 | 1f7d3f7d | René Nussbaumer | """ |
157 | 1f7d3f7d | René Nussbaumer | self.merger_data = [] |
158 | 1f7d3f7d | René Nussbaumer | self.clusters = clusters |
159 | 1f7d3f7d | René Nussbaumer | self.pause_period = pause_period |
160 | 1f7d3f7d | René Nussbaumer | self.work_dir = tempfile.mkdtemp(suffix="cluster-merger") |
161 | be8aecab | Stephen Shirley | (self.cluster_name, ) = cli.GetClient().QueryConfigValues(["cluster_name"]) |
162 | 1f7d3f7d | René Nussbaumer | self.ssh_runner = ssh.SshRunner(self.cluster_name) |
163 | 1a615be0 | Stephen Shirley | self.groups = groups |
164 | 620a9c62 | Guido Trotter | self.restart = restart |
165 | 1fcd3b81 | Guido Trotter | self.params = params |
166 | 1fb5f905 | Andrea Spadaccini | self.stop_instances = stop_instances |
167 | 620a9c62 | Guido Trotter | if self.restart == _RESTART_UP: |
168 | 620a9c62 | Guido Trotter | raise NotImplementedError |
169 | 620a9c62 | Guido Trotter | |
170 | 1f7d3f7d | René Nussbaumer | def Setup(self): |
171 | 1f7d3f7d | René Nussbaumer | """Sets up our end so we can do the merger. |
172 | 1f7d3f7d | René Nussbaumer | |
173 | 1f7d3f7d | René Nussbaumer | This method is setting us up as a preparation for the merger. |
174 | 1f7d3f7d | René Nussbaumer | It makes the initial contact and gathers information needed. |
175 | 1f7d3f7d | René Nussbaumer | |
176 | 1f7d3f7d | René Nussbaumer | @raise errors.RemoteError: for errors in communication/grabbing |
177 | 1f7d3f7d | René Nussbaumer | |
178 | 1f7d3f7d | René Nussbaumer | """ |
179 | 1f7d3f7d | René Nussbaumer | (remote_path, _, _) = ssh.GetUserFiles("root") |
180 | 1f7d3f7d | René Nussbaumer | |
181 | be8aecab | Stephen Shirley | if self.cluster_name in self.clusters: |
182 | be8aecab | Stephen Shirley | raise errors.CommandError("Cannot merge cluster %s with itself" % |
183 | be8aecab | Stephen Shirley | self.cluster_name) |
184 | be8aecab | Stephen Shirley | |
185 | 1f7d3f7d | René Nussbaumer | # Fetch remotes private key |
186 | 1f7d3f7d | René Nussbaumer | for cluster in self.clusters: |
187 | 1f7d3f7d | René Nussbaumer | result = self._RunCmd(cluster, "cat %s" % remote_path, batch=False, |
188 | 1f7d3f7d | René Nussbaumer | ask_key=False) |
189 | 1f7d3f7d | René Nussbaumer | if result.failed: |
190 | 1f7d3f7d | René Nussbaumer | raise errors.RemoteError("There was an error while grabbing ssh private" |
191 | 1f7d3f7d | René Nussbaumer | " key from %s. Fail reason: %s; output: %s" % |
192 | 1f7d3f7d | René Nussbaumer | (cluster, result.fail_reason, result.output)) |
193 | 1f7d3f7d | René Nussbaumer | |
194 | c4feafe8 | Iustin Pop | key_path = utils.PathJoin(self.work_dir, cluster) |
195 | 1f7d3f7d | René Nussbaumer | utils.WriteFile(key_path, mode=0600, data=result.stdout) |
196 | 1f7d3f7d | René Nussbaumer | |
197 | 8697f0fa | Guido Trotter | result = self._RunCmd(cluster, "gnt-node list -o name,offline" |
198 | 8697f0fa | Guido Trotter | " --no-header --separator=,", private_key=key_path) |
199 | 1f7d3f7d | René Nussbaumer | if result.failed: |
200 | 1f7d3f7d | René Nussbaumer | raise errors.RemoteError("Unable to retrieve list of nodes from %s." |
201 | 1f7d3f7d | René Nussbaumer | " Fail reason: %s; output: %s" % |
202 | 1f7d3f7d | René Nussbaumer | (cluster, result.fail_reason, result.output)) |
203 | 8697f0fa | Guido Trotter | nodes_statuses = [line.split(',') for line in result.stdout.splitlines()] |
204 | 8697f0fa | Guido Trotter | nodes = [node_status[0] for node_status in nodes_statuses |
205 | 8697f0fa | Guido Trotter | if node_status[1] == "N"] |
206 | 1f7d3f7d | René Nussbaumer | |
207 | 1f7d3f7d | René Nussbaumer | result = self._RunCmd(cluster, "gnt-instance list -o name --no-header", |
208 | 1f7d3f7d | René Nussbaumer | private_key=key_path) |
209 | 1f7d3f7d | René Nussbaumer | if result.failed: |
210 | 1f7d3f7d | René Nussbaumer | raise errors.RemoteError("Unable to retrieve list of instances from" |
211 | 1f7d3f7d | René Nussbaumer | " %s. Fail reason: %s; output: %s" % |
212 | 1f7d3f7d | René Nussbaumer | (cluster, result.fail_reason, result.output)) |
213 | 1f7d3f7d | René Nussbaumer | instances = result.stdout.splitlines() |
214 | 1f7d3f7d | René Nussbaumer | |
215 | b972c223 | Andrea Spadaccini | path = utils.PathJoin(constants.DATA_DIR, "ssconf_%s" % |
216 | b972c223 | Andrea Spadaccini | constants.SS_MASTER_NODE) |
217 | b972c223 | Andrea Spadaccini | result = self._RunCmd(cluster, "cat %s" % path, private_key=key_path) |
218 | b972c223 | Andrea Spadaccini | if result.failed: |
219 | b972c223 | Andrea Spadaccini | raise errors.RemoteError("Unable to retrieve the master node name from" |
220 | b972c223 | Andrea Spadaccini | " %s. Fail reason: %s; output: %s" % |
221 | b972c223 | Andrea Spadaccini | (cluster, result.fail_reason, result.output)) |
222 | b972c223 | Andrea Spadaccini | master_node = result.stdout.strip() |
223 | b972c223 | Andrea Spadaccini | |
224 | b972c223 | Andrea Spadaccini | path = utils.PathJoin(constants.DATA_DIR, "ssconf_%s" % |
225 | b972c223 | Andrea Spadaccini | constants.SS_MASTER_IP) |
226 | b972c223 | Andrea Spadaccini | result = self._RunCmd(cluster, "cat %s" % path, private_key=key_path) |
227 | b972c223 | Andrea Spadaccini | if result.failed: |
228 | b972c223 | Andrea Spadaccini | raise errors.RemoteError("Unable to retrieve the master IP from" |
229 | b972c223 | Andrea Spadaccini | " %s. Fail reason: %s; output: %s" % |
230 | b972c223 | Andrea Spadaccini | (cluster, result.fail_reason, result.output)) |
231 | b972c223 | Andrea Spadaccini | master_ip = result.stdout.strip() |
232 | b972c223 | Andrea Spadaccini | |
233 | b972c223 | Andrea Spadaccini | self.merger_data.append(MergerData(cluster, key_path, nodes, instances, |
234 | b972c223 | Andrea Spadaccini | master_node, master_ip)) |
235 | 1f7d3f7d | René Nussbaumer | |
236 | 8864d152 | Guido Trotter | def _PrepareAuthorizedKeys(self): |
237 | 8864d152 | Guido Trotter | """Prepare the authorized_keys on every merging node. |
238 | 1f7d3f7d | René Nussbaumer | |
239 | 1f7d3f7d | René Nussbaumer | This method add our public key to remotes authorized_key for further |
240 | 1f7d3f7d | René Nussbaumer | communication. |
241 | 1f7d3f7d | René Nussbaumer | |
242 | 1f7d3f7d | René Nussbaumer | """ |
243 | 1f7d3f7d | René Nussbaumer | (_, pub_key_file, auth_keys) = ssh.GetUserFiles("root") |
244 | 1f7d3f7d | René Nussbaumer | pub_key = utils.ReadFile(pub_key_file) |
245 | 1f7d3f7d | René Nussbaumer | |
246 | 8864d152 | Guido Trotter | for data in self.merger_data: |
247 | 8864d152 | Guido Trotter | for node in data.nodes: |
248 | 8864d152 | Guido Trotter | result = self._RunCmd(node, ("cat >> %s << '!EOF.'\n%s!EOF.\n" % |
249 | 8864d152 | Guido Trotter | (auth_keys, pub_key)), |
250 | 503cc75f | Guido Trotter | private_key=data.key_path, max_attempts=3) |
251 | 1f7d3f7d | René Nussbaumer | |
252 | 8864d152 | Guido Trotter | if result.failed: |
253 | 8864d152 | Guido Trotter | raise errors.RemoteError("Unable to add our public key to %s in %s." |
254 | 8864d152 | Guido Trotter | " Fail reason: %s; output: %s" % |
255 | 8864d152 | Guido Trotter | (node, data.cluster, result.fail_reason, |
256 | 8864d152 | Guido Trotter | result.output)) |
257 | 1f7d3f7d | René Nussbaumer | |
258 | 1f7d3f7d | René Nussbaumer | def _RunCmd(self, hostname, command, user="root", use_cluster_key=False, |
259 | 1f7d3f7d | René Nussbaumer | strict_host_check=False, private_key=None, batch=True, |
260 | 07ff0a78 | Andrea Spadaccini | ask_key=False, max_attempts=1): |
261 | 1f7d3f7d | René Nussbaumer | """Wrapping SshRunner.Run with default parameters. |
262 | 1f7d3f7d | René Nussbaumer | |
263 | 454723b5 | Iustin Pop | For explanation of parameters see L{ganeti.ssh.SshRunner.Run}. |
264 | 1f7d3f7d | René Nussbaumer | |
265 | 1f7d3f7d | René Nussbaumer | """ |
266 | 07ff0a78 | Andrea Spadaccini | for _ in range(max_attempts): |
267 | 07ff0a78 | Andrea Spadaccini | result = self.ssh_runner.Run(hostname=hostname, command=command, |
268 | 07ff0a78 | Andrea Spadaccini | user=user, use_cluster_key=use_cluster_key, |
269 | 07ff0a78 | Andrea Spadaccini | strict_host_check=strict_host_check, |
270 | 07ff0a78 | Andrea Spadaccini | private_key=private_key, batch=batch, |
271 | 07ff0a78 | Andrea Spadaccini | ask_key=ask_key) |
272 | 07ff0a78 | Andrea Spadaccini | if not result.failed: |
273 | 07ff0a78 | Andrea Spadaccini | break |
274 | 07ff0a78 | Andrea Spadaccini | |
275 | 07ff0a78 | Andrea Spadaccini | return result |
276 | 1f7d3f7d | René Nussbaumer | |
277 | 1fb5f905 | Andrea Spadaccini | def _CheckRunningInstances(self): |
278 | 1fb5f905 | Andrea Spadaccini | """Checks if on the clusters to be merged there are running instances |
279 | 1fb5f905 | Andrea Spadaccini | |
280 | 1fb5f905 | Andrea Spadaccini | @rtype: boolean |
281 | 1fb5f905 | Andrea Spadaccini | @return: True if there are running instances, False otherwise |
282 | 1fb5f905 | Andrea Spadaccini | |
283 | 1fb5f905 | Andrea Spadaccini | """ |
284 | 1fb5f905 | Andrea Spadaccini | for cluster in self.clusters: |
285 | 1fb5f905 | Andrea Spadaccini | result = self._RunCmd(cluster, "gnt-instance list -o status") |
286 | 006a51b3 | Andrea Spadaccini | if self.RUNNING_STATUSES.intersection(result.output.splitlines()): |
287 | 1fb5f905 | Andrea Spadaccini | return True |
288 | 1fb5f905 | Andrea Spadaccini | |
289 | 1fb5f905 | Andrea Spadaccini | return False |
290 | 1fb5f905 | Andrea Spadaccini | |
291 | 1f7d3f7d | René Nussbaumer | def _StopMergingInstances(self): |
292 | 1f7d3f7d | René Nussbaumer | """Stop instances on merging clusters. |
293 | 1f7d3f7d | René Nussbaumer | |
294 | 1f7d3f7d | René Nussbaumer | """ |
295 | 1f7d3f7d | René Nussbaumer | for cluster in self.clusters: |
296 | 1f7d3f7d | René Nussbaumer | result = self._RunCmd(cluster, "gnt-instance shutdown --all" |
297 | 1f7d3f7d | René Nussbaumer | " --force-multiple") |
298 | 1f7d3f7d | René Nussbaumer | |
299 | 1f7d3f7d | René Nussbaumer | if result.failed: |
300 | 1f7d3f7d | René Nussbaumer | raise errors.RemoteError("Unable to stop instances on %s." |
301 | 1f7d3f7d | René Nussbaumer | " Fail reason: %s; output: %s" % |
302 | 1f7d3f7d | René Nussbaumer | (cluster, result.fail_reason, result.output)) |
303 | 1f7d3f7d | René Nussbaumer | |
304 | 1f7d3f7d | René Nussbaumer | def _DisableWatcher(self): |
305 | 1f7d3f7d | René Nussbaumer | """Disable watch on all merging clusters, including ourself. |
306 | 1f7d3f7d | René Nussbaumer | |
307 | 1f7d3f7d | René Nussbaumer | """ |
308 | 1f7d3f7d | René Nussbaumer | for cluster in ["localhost"] + self.clusters: |
309 | 1f7d3f7d | René Nussbaumer | result = self._RunCmd(cluster, "gnt-cluster watcher pause %d" % |
310 | 1f7d3f7d | René Nussbaumer | self.pause_period) |
311 | 1f7d3f7d | René Nussbaumer | |
312 | 1f7d3f7d | René Nussbaumer | if result.failed: |
313 | 1f7d3f7d | René Nussbaumer | raise errors.RemoteError("Unable to pause watcher on %s." |
314 | 1f7d3f7d | René Nussbaumer | " Fail reason: %s; output: %s" % |
315 | 1f7d3f7d | René Nussbaumer | (cluster, result.fail_reason, result.output)) |
316 | 1f7d3f7d | René Nussbaumer | |
317 | b972c223 | Andrea Spadaccini | def _RemoveMasterIps(self): |
318 | b972c223 | Andrea Spadaccini | """Removes the master IPs from the master nodes of each cluster. |
319 | b972c223 | Andrea Spadaccini | |
320 | b972c223 | Andrea Spadaccini | """ |
321 | b972c223 | Andrea Spadaccini | for data in self.merger_data: |
322 | b972c223 | Andrea Spadaccini | master_ip_family = netutils.IPAddress.GetAddressFamily(data.master_ip) |
323 | b972c223 | Andrea Spadaccini | master_ip_len = netutils.IP4Address.iplen |
324 | b972c223 | Andrea Spadaccini | if master_ip_family == netutils.IP6Address.family: |
325 | b972c223 | Andrea Spadaccini | master_ip_len = netutils.IP6Address.iplen |
326 | b972c223 | Andrea Spadaccini | # Not using constants.IP_COMMAND_PATH because the command might run on a |
327 | b972c223 | Andrea Spadaccini | # machine in which the ip path is different, so it's better to rely on |
328 | b972c223 | Andrea Spadaccini | # $PATH. |
329 | b972c223 | Andrea Spadaccini | cmd = "ip address del %s/%s dev $(cat %s)" % ( |
330 | b972c223 | Andrea Spadaccini | data.master_ip, |
331 | b972c223 | Andrea Spadaccini | master_ip_len, |
332 | b972c223 | Andrea Spadaccini | utils.PathJoin(constants.DATA_DIR, "ssconf_%s" % |
333 | b972c223 | Andrea Spadaccini | constants.SS_MASTER_NETDEV)) |
334 | b972c223 | Andrea Spadaccini | result = self._RunCmd(data.master_node, cmd, max_attempts=3) |
335 | b972c223 | Andrea Spadaccini | if result.failed: |
336 | b972c223 | Andrea Spadaccini | raise errors.RemoteError("Unable to remove master IP on %s." |
337 | b972c223 | Andrea Spadaccini | " Fail reason: %s; output: %s" % |
338 | b972c223 | Andrea Spadaccini | (data.master_node, |
339 | b972c223 | Andrea Spadaccini | result.fail_reason, |
340 | b972c223 | Andrea Spadaccini | result.output)) |
341 | b972c223 | Andrea Spadaccini | |
342 | 1f7d3f7d | René Nussbaumer | def _StopDaemons(self): |
343 | 1f7d3f7d | René Nussbaumer | """Stop all daemons on merging nodes. |
344 | 1f7d3f7d | René Nussbaumer | |
345 | 1f7d3f7d | René Nussbaumer | """ |
346 | d8aab233 | René Nussbaumer | cmd = "%s stop-all" % constants.DAEMON_UTIL |
347 | 1f7d3f7d | René Nussbaumer | for data in self.merger_data: |
348 | 1f7d3f7d | René Nussbaumer | for node in data.nodes: |
349 | 503cc75f | Guido Trotter | result = self._RunCmd(node, cmd, max_attempts=3) |
350 | 1f7d3f7d | René Nussbaumer | |
351 | 1f7d3f7d | René Nussbaumer | if result.failed: |
352 | 1f7d3f7d | René Nussbaumer | raise errors.RemoteError("Unable to stop daemons on %s." |
353 | 1f7d3f7d | René Nussbaumer | " Fail reason: %s; output: %s." % |
354 | 1f7d3f7d | René Nussbaumer | (node, result.fail_reason, result.output)) |
355 | 1f7d3f7d | René Nussbaumer | |
356 | 1f7d3f7d | René Nussbaumer | def _FetchRemoteConfig(self): |
357 | 1f7d3f7d | René Nussbaumer | """Fetches and stores remote cluster config from the master. |
358 | 1f7d3f7d | René Nussbaumer | |
359 | 1f7d3f7d | René Nussbaumer | This step is needed before we can merge the config. |
360 | 1f7d3f7d | René Nussbaumer | |
361 | 1f7d3f7d | René Nussbaumer | """ |
362 | 1f7d3f7d | René Nussbaumer | for data in self.merger_data: |
363 | 1f7d3f7d | René Nussbaumer | result = self._RunCmd(data.cluster, "cat %s" % |
364 | 1f7d3f7d | René Nussbaumer | constants.CLUSTER_CONF_FILE) |
365 | 1f7d3f7d | René Nussbaumer | |
366 | 1f7d3f7d | René Nussbaumer | if result.failed: |
367 | 1f7d3f7d | René Nussbaumer | raise errors.RemoteError("Unable to retrieve remote config on %s." |
368 | 1f7d3f7d | René Nussbaumer | " Fail reason: %s; output %s" % |
369 | 1f7d3f7d | René Nussbaumer | (data.cluster, result.fail_reason, |
370 | 1f7d3f7d | René Nussbaumer | result.output)) |
371 | 1f7d3f7d | René Nussbaumer | |
372 | c4feafe8 | Iustin Pop | data.config_path = utils.PathJoin(self.work_dir, "%s_config.data" % |
373 | c4feafe8 | Iustin Pop | data.cluster) |
374 | 1f7d3f7d | René Nussbaumer | utils.WriteFile(data.config_path, data=result.stdout) |
375 | 1f7d3f7d | René Nussbaumer | |
376 | 1f7d3f7d | René Nussbaumer | # R0201: Method could be a function |
377 | b459a848 | Andrea Spadaccini | def _KillMasterDaemon(self): # pylint: disable=R0201 |
378 | 1f7d3f7d | René Nussbaumer | """Kills the local master daemon. |
379 | 1f7d3f7d | René Nussbaumer | |
380 | 1f7d3f7d | René Nussbaumer | @raise errors.CommandError: If unable to kill |
381 | 1f7d3f7d | René Nussbaumer | |
382 | 1f7d3f7d | René Nussbaumer | """ |
383 | 1f7d3f7d | René Nussbaumer | result = utils.RunCmd([constants.DAEMON_UTIL, "stop-master"]) |
384 | 1f7d3f7d | René Nussbaumer | if result.failed: |
385 | 1f7d3f7d | René Nussbaumer | raise errors.CommandError("Unable to stop master daemons." |
386 | 1f7d3f7d | René Nussbaumer | " Fail reason: %s; output: %s" % |
387 | 1f7d3f7d | René Nussbaumer | (result.fail_reason, result.output)) |
388 | 1f7d3f7d | René Nussbaumer | |
389 | 1f7d3f7d | René Nussbaumer | def _MergeConfig(self): |
390 | 1f7d3f7d | René Nussbaumer | """Merges all foreign config into our own config. |
391 | 1f7d3f7d | René Nussbaumer | |
392 | 1f7d3f7d | René Nussbaumer | """ |
393 | 1f7d3f7d | René Nussbaumer | my_config = config.ConfigWriter(offline=True) |
394 | 1f7d3f7d | René Nussbaumer | fake_ec_id = 0 # Needs to be uniq over the whole config merge |
395 | 1f7d3f7d | René Nussbaumer | |
396 | 1f7d3f7d | René Nussbaumer | for data in self.merger_data: |
397 | e1ab08db | Stephen Shirley | other_config = config.ConfigWriter(data.config_path, accept_foreign=True) |
398 | a6c8fd10 | Stephen Shirley | self._MergeClusterConfigs(my_config, other_config) |
399 | 8f44674f | Stephen Shirley | self._MergeNodeGroups(my_config, other_config) |
400 | 1f7d3f7d | René Nussbaumer | |
401 | 1f7d3f7d | René Nussbaumer | for node in other_config.GetNodeList(): |
402 | 1f7d3f7d | René Nussbaumer | node_info = other_config.GetNodeInfo(node) |
403 | 3e22caed | Guido Trotter | # Offline the node, it will be reonlined later at node readd |
404 | 3e22caed | Guido Trotter | node_info.master_candidate = False |
405 | 3e22caed | Guido Trotter | node_info.drained = False |
406 | 3e22caed | Guido Trotter | node_info.offline = True |
407 | 8864d152 | Guido Trotter | my_config.AddNode(node_info, _CLUSTERMERGE_ECID + str(fake_ec_id)) |
408 | 1f7d3f7d | René Nussbaumer | fake_ec_id += 1 |
409 | 1f7d3f7d | René Nussbaumer | |
410 | 1f7d3f7d | René Nussbaumer | for instance in other_config.GetInstanceList(): |
411 | 1f7d3f7d | René Nussbaumer | instance_info = other_config.GetInstanceInfo(instance) |
412 | 1f7d3f7d | René Nussbaumer | |
413 | 1f7d3f7d | René Nussbaumer | # Update the DRBD port assignments |
414 | 1f7d3f7d | René Nussbaumer | # This is a little bit hackish |
415 | 1f7d3f7d | René Nussbaumer | for dsk in instance_info.disks: |
416 | 1f7d3f7d | René Nussbaumer | if dsk.dev_type in constants.LDS_DRBD: |
417 | 1f7d3f7d | René Nussbaumer | port = my_config.AllocatePort() |
418 | 1f7d3f7d | René Nussbaumer | |
419 | 1f7d3f7d | René Nussbaumer | logical_id = list(dsk.logical_id) |
420 | 1f7d3f7d | René Nussbaumer | logical_id[2] = port |
421 | 1f7d3f7d | René Nussbaumer | dsk.logical_id = tuple(logical_id) |
422 | 1f7d3f7d | René Nussbaumer | |
423 | 1f7d3f7d | René Nussbaumer | physical_id = list(dsk.physical_id) |
424 | 1f7d3f7d | René Nussbaumer | physical_id[1] = physical_id[3] = port |
425 | 1f7d3f7d | René Nussbaumer | dsk.physical_id = tuple(physical_id) |
426 | 1f7d3f7d | René Nussbaumer | |
427 | a536aaac | Stephen Shirley | my_config.AddInstance(instance_info, |
428 | a536aaac | Stephen Shirley | _CLUSTERMERGE_ECID + str(fake_ec_id)) |
429 | 1f7d3f7d | René Nussbaumer | fake_ec_id += 1 |
430 | 1f7d3f7d | René Nussbaumer | |
431 | a6c8fd10 | Stephen Shirley | def _MergeClusterConfigs(self, my_config, other_config): |
432 | a6c8fd10 | Stephen Shirley | """Checks that all relevant cluster parameters are compatible |
433 | a6c8fd10 | Stephen Shirley | |
434 | a6c8fd10 | Stephen Shirley | """ |
435 | a6c8fd10 | Stephen Shirley | my_cluster = my_config.GetClusterInfo() |
436 | a6c8fd10 | Stephen Shirley | other_cluster = other_config.GetClusterInfo() |
437 | a6c8fd10 | Stephen Shirley | err_count = 0 |
438 | a6c8fd10 | Stephen Shirley | |
439 | a6c8fd10 | Stephen Shirley | # |
440 | a6c8fd10 | Stephen Shirley | # Generic checks |
441 | a6c8fd10 | Stephen Shirley | # |
442 | bb074298 | Guido Trotter | check_params = [ |
443 | a6c8fd10 | Stephen Shirley | "beparams", |
444 | a6c8fd10 | Stephen Shirley | "default_iallocator", |
445 | a6c8fd10 | Stephen Shirley | "drbd_usermode_helper", |
446 | a6c8fd10 | Stephen Shirley | "hidden_os", |
447 | a6c8fd10 | Stephen Shirley | "maintain_node_health", |
448 | a6c8fd10 | Stephen Shirley | "master_netdev", |
449 | a6c8fd10 | Stephen Shirley | "ndparams", |
450 | a6c8fd10 | Stephen Shirley | "nicparams", |
451 | a6c8fd10 | Stephen Shirley | "primary_ip_family", |
452 | a6c8fd10 | Stephen Shirley | "tags", |
453 | a6c8fd10 | Stephen Shirley | "uid_pool", |
454 | bb074298 | Guido Trotter | ] |
455 | 1fcd3b81 | Guido Trotter | check_params_strict = [ |
456 | 1fcd3b81 | Guido Trotter | "volume_group_name", |
457 | 1fcd3b81 | Guido Trotter | ] |
458 | bb074298 | Guido Trotter | if constants.ENABLE_FILE_STORAGE: |
459 | 1fcd3b81 | Guido Trotter | check_params_strict.append("file_storage_dir") |
460 | 1fcd3b81 | Guido Trotter | if constants.ENABLE_SHARED_FILE_STORAGE: |
461 | 1fcd3b81 | Guido Trotter | check_params_strict.append("shared_file_storage_dir") |
462 | 1fcd3b81 | Guido Trotter | check_params.extend(check_params_strict) |
463 | 1fcd3b81 | Guido Trotter | |
464 | 1fcd3b81 | Guido Trotter | if self.params == _PARAMS_STRICT: |
465 | 1fcd3b81 | Guido Trotter | params_strict = True |
466 | 1fcd3b81 | Guido Trotter | else: |
467 | 1fcd3b81 | Guido Trotter | params_strict = False |
468 | bb074298 | Guido Trotter | |
469 | a6c8fd10 | Stephen Shirley | for param_name in check_params: |
470 | a6c8fd10 | Stephen Shirley | my_param = getattr(my_cluster, param_name) |
471 | a6c8fd10 | Stephen Shirley | other_param = getattr(other_cluster, param_name) |
472 | a6c8fd10 | Stephen Shirley | if my_param != other_param: |
473 | a6c8fd10 | Stephen Shirley | logging.error("The value (%s) of the cluster parameter %s on %s" |
474 | a6c8fd10 | Stephen Shirley | " differs to this cluster's value (%s)", |
475 | a6c8fd10 | Stephen Shirley | other_param, param_name, other_cluster.cluster_name, |
476 | a6c8fd10 | Stephen Shirley | my_param) |
477 | 1fcd3b81 | Guido Trotter | if params_strict or param_name in check_params_strict: |
478 | 1fcd3b81 | Guido Trotter | err_count += 1 |
479 | a6c8fd10 | Stephen Shirley | |
480 | a6c8fd10 | Stephen Shirley | # |
481 | a6c8fd10 | Stephen Shirley | # Custom checks |
482 | a6c8fd10 | Stephen Shirley | # |
483 | a6c8fd10 | Stephen Shirley | |
484 | a6c8fd10 | Stephen Shirley | # Check default hypervisor |
485 | a6c8fd10 | Stephen Shirley | my_defhyp = my_cluster.enabled_hypervisors[0] |
486 | a6c8fd10 | Stephen Shirley | other_defhyp = other_cluster.enabled_hypervisors[0] |
487 | a6c8fd10 | Stephen Shirley | if my_defhyp != other_defhyp: |
488 | a6c8fd10 | Stephen Shirley | logging.warning("The default hypervisor (%s) differs on %s, new" |
489 | a6c8fd10 | Stephen Shirley | " instances will be created with this cluster's" |
490 | a6c8fd10 | Stephen Shirley | " default hypervisor (%s)", other_defhyp, |
491 | a6c8fd10 | Stephen Shirley | other_cluster.cluster_name, my_defhyp) |
492 | a6c8fd10 | Stephen Shirley | |
493 | a6c8fd10 | Stephen Shirley | if (set(my_cluster.enabled_hypervisors) != |
494 | a6c8fd10 | Stephen Shirley | set(other_cluster.enabled_hypervisors)): |
495 | a6c8fd10 | Stephen Shirley | logging.error("The set of enabled hypervisors (%s) on %s differs to" |
496 | a6c8fd10 | Stephen Shirley | " this cluster's set (%s)", |
497 | a6c8fd10 | Stephen Shirley | other_cluster.enabled_hypervisors, |
498 | a6c8fd10 | Stephen Shirley | other_cluster.cluster_name, my_cluster.enabled_hypervisors) |
499 | a6c8fd10 | Stephen Shirley | err_count += 1 |
500 | a6c8fd10 | Stephen Shirley | |
501 | a6c8fd10 | Stephen Shirley | # Check hypervisor params for hypervisors we care about |
502 | a6c8fd10 | Stephen Shirley | for hyp in my_cluster.enabled_hypervisors: |
503 | a6c8fd10 | Stephen Shirley | for param in my_cluster.hvparams[hyp]: |
504 | a6c8fd10 | Stephen Shirley | my_value = my_cluster.hvparams[hyp][param] |
505 | a6c8fd10 | Stephen Shirley | other_value = other_cluster.hvparams[hyp][param] |
506 | a6c8fd10 | Stephen Shirley | if my_value != other_value: |
507 | a6c8fd10 | Stephen Shirley | logging.error("The value (%s) of the %s parameter of the %s" |
508 | a6c8fd10 | Stephen Shirley | " hypervisor on %s differs to this cluster's parameter" |
509 | a6c8fd10 | Stephen Shirley | " (%s)", |
510 | a6c8fd10 | Stephen Shirley | other_value, param, hyp, other_cluster.cluster_name, |
511 | a6c8fd10 | Stephen Shirley | my_value) |
512 | 1fcd3b81 | Guido Trotter | if params_strict: |
513 | 1fcd3b81 | Guido Trotter | err_count += 1 |
514 | a6c8fd10 | Stephen Shirley | |
515 | a6c8fd10 | Stephen Shirley | # Check os hypervisor params for hypervisors we care about |
516 | a6c8fd10 | Stephen Shirley | for os_name in set(my_cluster.os_hvp.keys() + other_cluster.os_hvp.keys()): |
517 | a6c8fd10 | Stephen Shirley | for hyp in my_cluster.enabled_hypervisors: |
518 | a6c8fd10 | Stephen Shirley | my_os_hvp = self._GetOsHypervisor(my_cluster, os_name, hyp) |
519 | a6c8fd10 | Stephen Shirley | other_os_hvp = self._GetOsHypervisor(other_cluster, os_name, hyp) |
520 | a6c8fd10 | Stephen Shirley | if my_os_hvp != other_os_hvp: |
521 | a6c8fd10 | Stephen Shirley | logging.error("The OS parameters (%s) for the %s OS for the %s" |
522 | a6c8fd10 | Stephen Shirley | " hypervisor on %s differs to this cluster's parameters" |
523 | a6c8fd10 | Stephen Shirley | " (%s)", |
524 | a6c8fd10 | Stephen Shirley | other_os_hvp, os_name, hyp, other_cluster.cluster_name, |
525 | a6c8fd10 | Stephen Shirley | my_os_hvp) |
526 | 1fcd3b81 | Guido Trotter | if params_strict: |
527 | 1fcd3b81 | Guido Trotter | err_count += 1 |
528 | a6c8fd10 | Stephen Shirley | |
529 | a6c8fd10 | Stephen Shirley | # |
530 | a6c8fd10 | Stephen Shirley | # Warnings |
531 | a6c8fd10 | Stephen Shirley | # |
532 | a6c8fd10 | Stephen Shirley | if my_cluster.modify_etc_hosts != other_cluster.modify_etc_hosts: |
533 | a6c8fd10 | Stephen Shirley | logging.warning("The modify_etc_hosts value (%s) differs on %s," |
534 | a6c8fd10 | Stephen Shirley | " this cluster's value (%s) will take precedence", |
535 | a6c8fd10 | Stephen Shirley | other_cluster.modify_etc_hosts, |
536 | a6c8fd10 | Stephen Shirley | other_cluster.cluster_name, |
537 | a6c8fd10 | Stephen Shirley | my_cluster.modify_etc_hosts) |
538 | a6c8fd10 | Stephen Shirley | |
539 | a6c8fd10 | Stephen Shirley | if my_cluster.modify_ssh_setup != other_cluster.modify_ssh_setup: |
540 | a6c8fd10 | Stephen Shirley | logging.warning("The modify_ssh_setup value (%s) differs on %s," |
541 | a6c8fd10 | Stephen Shirley | " this cluster's value (%s) will take precedence", |
542 | a6c8fd10 | Stephen Shirley | other_cluster.modify_ssh_setup, |
543 | a6c8fd10 | Stephen Shirley | other_cluster.cluster_name, |
544 | a6c8fd10 | Stephen Shirley | my_cluster.modify_ssh_setup) |
545 | a6c8fd10 | Stephen Shirley | |
546 | a6c8fd10 | Stephen Shirley | # |
547 | a6c8fd10 | Stephen Shirley | # Actual merging |
548 | a6c8fd10 | Stephen Shirley | # |
549 | a6c8fd10 | Stephen Shirley | my_cluster.reserved_lvs = list(set(my_cluster.reserved_lvs + |
550 | a6c8fd10 | Stephen Shirley | other_cluster.reserved_lvs)) |
551 | a6c8fd10 | Stephen Shirley | |
552 | a6c8fd10 | Stephen Shirley | if my_cluster.prealloc_wipe_disks != other_cluster.prealloc_wipe_disks: |
553 | a6c8fd10 | Stephen Shirley | logging.warning("The prealloc_wipe_disks value (%s) on %s differs to this" |
554 | a6c8fd10 | Stephen Shirley | " cluster's value (%s). The least permissive value (%s)" |
555 | a6c8fd10 | Stephen Shirley | " will be used", other_cluster.prealloc_wipe_disks, |
556 | a6c8fd10 | Stephen Shirley | other_cluster.cluster_name, |
557 | a6c8fd10 | Stephen Shirley | my_cluster.prealloc_wipe_disks, True) |
558 | a6c8fd10 | Stephen Shirley | my_cluster.prealloc_wipe_disks = True |
559 | a6c8fd10 | Stephen Shirley | |
560 | a6c8fd10 | Stephen Shirley | for os_, osparams in other_cluster.osparams.items(): |
561 | a6c8fd10 | Stephen Shirley | if os_ not in my_cluster.osparams: |
562 | a6c8fd10 | Stephen Shirley | my_cluster.osparams[os_] = osparams |
563 | a6c8fd10 | Stephen Shirley | elif my_cluster.osparams[os_] != osparams: |
564 | a6c8fd10 | Stephen Shirley | logging.error("The OS parameters (%s) for the %s OS on %s differs to" |
565 | a6c8fd10 | Stephen Shirley | " this cluster's parameters (%s)", |
566 | a6c8fd10 | Stephen Shirley | osparams, os_, other_cluster.cluster_name, |
567 | a6c8fd10 | Stephen Shirley | my_cluster.osparams[os_]) |
568 | 1fcd3b81 | Guido Trotter | if params_strict: |
569 | 1fcd3b81 | Guido Trotter | err_count += 1 |
570 | a6c8fd10 | Stephen Shirley | |
571 | a6c8fd10 | Stephen Shirley | if err_count: |
572 | a6c8fd10 | Stephen Shirley | raise errors.ConfigurationError("Cluster config for %s has incompatible" |
573 | a6c8fd10 | Stephen Shirley | " values, please fix and re-run" % |
574 | a6c8fd10 | Stephen Shirley | other_cluster.cluster_name) |
575 | a6c8fd10 | Stephen Shirley | |
576 | a6c8fd10 | Stephen Shirley | # R0201: Method could be a function |
577 | b459a848 | Andrea Spadaccini | def _GetOsHypervisor(self, cluster, os_name, hyp): # pylint: disable=R0201 |
578 | a6c8fd10 | Stephen Shirley | if os_name in cluster.os_hvp: |
579 | a6c8fd10 | Stephen Shirley | return cluster.os_hvp[os_name].get(hyp, None) |
580 | a6c8fd10 | Stephen Shirley | else: |
581 | a6c8fd10 | Stephen Shirley | return None |
582 | a6c8fd10 | Stephen Shirley | |
583 | a6c8fd10 | Stephen Shirley | # R0201: Method could be a function |
584 | 8f44674f | Stephen Shirley | def _MergeNodeGroups(self, my_config, other_config): |
585 | 8f44674f | Stephen Shirley | """Adds foreign node groups |
586 | 8f44674f | Stephen Shirley | |
587 | 8f44674f | Stephen Shirley | ConfigWriter.AddNodeGroup takes care of making sure there are no conflicts. |
588 | 8f44674f | Stephen Shirley | """ |
589 | b459a848 | Andrea Spadaccini | # pylint: disable=R0201 |
590 | 9b945588 | Stephen Shirley | logging.info("Node group conflict strategy: %s", self.groups) |
591 | 1a615be0 | Stephen Shirley | |
592 | 1a615be0 | Stephen Shirley | my_grps = my_config.GetAllNodeGroupsInfo().values() |
593 | 1a615be0 | Stephen Shirley | other_grps = other_config.GetAllNodeGroupsInfo().values() |
594 | 1a615be0 | Stephen Shirley | |
595 | 1a615be0 | Stephen Shirley | # Check for node group naming conflicts: |
596 | 1a615be0 | Stephen Shirley | conflicts = [] |
597 | 1a615be0 | Stephen Shirley | for other_grp in other_grps: |
598 | 1a615be0 | Stephen Shirley | for my_grp in my_grps: |
599 | 1a615be0 | Stephen Shirley | if other_grp.name == my_grp.name: |
600 | 1a615be0 | Stephen Shirley | conflicts.append(other_grp) |
601 | 1a615be0 | Stephen Shirley | |
602 | 1a615be0 | Stephen Shirley | if conflicts: |
603 | 1a615be0 | Stephen Shirley | conflict_names = utils.CommaJoin([g.name for g in conflicts]) |
604 | 9b945588 | Stephen Shirley | logging.info("Node groups in both local and remote cluster: %s", |
605 | 1a615be0 | Stephen Shirley | conflict_names) |
606 | 1a615be0 | Stephen Shirley | |
607 | 1a615be0 | Stephen Shirley | # User hasn't specified how to handle conflicts |
608 | 1a615be0 | Stephen Shirley | if not self.groups: |
609 | 1a615be0 | Stephen Shirley | raise errors.CommandError("The following node group(s) are in both" |
610 | 1a615be0 | Stephen Shirley | " clusters, and no merge strategy has been" |
611 | 1a615be0 | Stephen Shirley | " supplied (see the --groups option): %s" % |
612 | 1a615be0 | Stephen Shirley | conflict_names) |
613 | 1a615be0 | Stephen Shirley | |
614 | 1a615be0 | Stephen Shirley | # User wants to rename conflicts |
615 | 3a969900 | Stephen Shirley | elif self.groups == _GROUPS_RENAME: |
616 | 1a615be0 | Stephen Shirley | for grp in conflicts: |
617 | 1a615be0 | Stephen Shirley | new_name = "%s-%s" % (grp.name, other_config.GetClusterName()) |
618 | 1a615be0 | Stephen Shirley | logging.info("Renaming remote node group from %s to %s" |
619 | 9b945588 | Stephen Shirley | " to resolve conflict", grp.name, new_name) |
620 | 1a615be0 | Stephen Shirley | grp.name = new_name |
621 | 1a615be0 | Stephen Shirley | |
622 | 3a969900 | Stephen Shirley | # User wants to merge conflicting groups |
623 | 2be7c4cc | Guido Trotter | elif self.groups == _GROUPS_MERGE: |
624 | 3a969900 | Stephen Shirley | for other_grp in conflicts: |
625 | 9b945588 | Stephen Shirley | logging.info("Merging local and remote '%s' groups", other_grp.name) |
626 | 3a969900 | Stephen Shirley | for node_name in other_grp.members[:]: |
627 | 3a969900 | Stephen Shirley | node = other_config.GetNodeInfo(node_name) |
628 | 9b945588 | Stephen Shirley | # Access to a protected member of a client class |
629 | b459a848 | Andrea Spadaccini | # pylint: disable=W0212 |
630 | 3a969900 | Stephen Shirley | other_config._UnlockedRemoveNodeFromGroup(node) |
631 | 3a969900 | Stephen Shirley | |
632 | 9b945588 | Stephen Shirley | # Access to a protected member of a client class |
633 | b459a848 | Andrea Spadaccini | # pylint: disable=W0212 |
634 | 3a969900 | Stephen Shirley | my_grp_uuid = my_config._UnlockedLookupNodeGroup(other_grp.name) |
635 | 9b945588 | Stephen Shirley | |
636 | 9b945588 | Stephen Shirley | # Access to a protected member of a client class |
637 | b459a848 | Andrea Spadaccini | # pylint: disable=W0212 |
638 | 3a969900 | Stephen Shirley | my_config._UnlockedAddNodeToGroup(node, my_grp_uuid) |
639 | 3a969900 | Stephen Shirley | node.group = my_grp_uuid |
640 | 3a969900 | Stephen Shirley | # Remove from list of groups to add |
641 | 3a969900 | Stephen Shirley | other_grps.remove(other_grp) |
642 | 3a969900 | Stephen Shirley | |
643 | 1a615be0 | Stephen Shirley | for grp in other_grps: |
644 | 8f44674f | Stephen Shirley | #TODO: handle node group conflicts |
645 | 8f44674f | Stephen Shirley | my_config.AddNodeGroup(grp, _CLUSTERMERGE_ECID) |
646 | 8f44674f | Stephen Shirley | |
647 | 8f44674f | Stephen Shirley | # R0201: Method could be a function |
648 | b459a848 | Andrea Spadaccini | def _StartMasterDaemon(self, no_vote=False): # pylint: disable=R0201 |
649 | 1f7d3f7d | René Nussbaumer | """Starts the local master daemon. |
650 | 1f7d3f7d | René Nussbaumer | |
651 | 1f7d3f7d | René Nussbaumer | @param no_vote: Should the masterd started without voting? default: False |
652 | 1f7d3f7d | René Nussbaumer | @raise errors.CommandError: If unable to start daemon. |
653 | 1f7d3f7d | René Nussbaumer | |
654 | 1f7d3f7d | René Nussbaumer | """ |
655 | 1f7d3f7d | René Nussbaumer | env = {} |
656 | 1f7d3f7d | René Nussbaumer | if no_vote: |
657 | 1f7d3f7d | René Nussbaumer | env["EXTRA_MASTERD_ARGS"] = "--no-voting --yes-do-it" |
658 | 1f7d3f7d | René Nussbaumer | |
659 | 1f7d3f7d | René Nussbaumer | result = utils.RunCmd([constants.DAEMON_UTIL, "start-master"], env=env) |
660 | 1f7d3f7d | René Nussbaumer | if result.failed: |
661 | 1f7d3f7d | René Nussbaumer | raise errors.CommandError("Couldn't start ganeti master." |
662 | 1f7d3f7d | René Nussbaumer | " Fail reason: %s; output: %s" % |
663 | 1f7d3f7d | René Nussbaumer | (result.fail_reason, result.output)) |
664 | 1f7d3f7d | René Nussbaumer | |
665 | 1f7d3f7d | René Nussbaumer | def _ReaddMergedNodesAndRedist(self): |
666 | 1f7d3f7d | René Nussbaumer | """Readds all merging nodes and make sure their config is up-to-date. |
667 | 1f7d3f7d | René Nussbaumer | |
668 | 1f7d3f7d | René Nussbaumer | @raise errors.CommandError: If anything fails. |
669 | 1f7d3f7d | René Nussbaumer | |
670 | 1f7d3f7d | René Nussbaumer | """ |
671 | 1f7d3f7d | René Nussbaumer | for data in self.merger_data: |
672 | 1f7d3f7d | René Nussbaumer | for node in data.nodes: |
673 | 1f7d3f7d | René Nussbaumer | result = utils.RunCmd(["gnt-node", "add", "--readd", |
674 | 53991408 | Stephen Shirley | "--no-ssh-key-check", "--force-join", node]) |
675 | 1f7d3f7d | René Nussbaumer | if result.failed: |
676 | d47319e2 | Guido Trotter | logging.error("%s failed to be readded. Reason: %s, output: %s", |
677 | d47319e2 | Guido Trotter | node, result.fail_reason, result.output) |
678 | 1f7d3f7d | René Nussbaumer | |
679 | 1f7d3f7d | René Nussbaumer | result = utils.RunCmd(["gnt-cluster", "redist-conf"]) |
680 | 1f7d3f7d | René Nussbaumer | if result.failed: |
681 | 1f7d3f7d | René Nussbaumer | raise errors.CommandError("Redistribution failed. Fail reason: %s;" |
682 | 1f7d3f7d | René Nussbaumer | " output: %s" % (result.fail_reason, |
683 | 1f7d3f7d | René Nussbaumer | result.output)) |
684 | 1f7d3f7d | René Nussbaumer | |
685 | 1f7d3f7d | René Nussbaumer | # R0201: Method could be a function |
686 | b459a848 | Andrea Spadaccini | def _StartupAllInstances(self): # pylint: disable=R0201 |
687 | 1f7d3f7d | René Nussbaumer | """Starts up all instances (locally). |
688 | 1f7d3f7d | René Nussbaumer | |
689 | 1f7d3f7d | René Nussbaumer | @raise errors.CommandError: If unable to start clusters |
690 | 1f7d3f7d | René Nussbaumer | |
691 | 1f7d3f7d | René Nussbaumer | """ |
692 | 1f7d3f7d | René Nussbaumer | result = utils.RunCmd(["gnt-instance", "startup", "--all", |
693 | 1f7d3f7d | René Nussbaumer | "--force-multiple"]) |
694 | 1f7d3f7d | René Nussbaumer | if result.failed: |
695 | 1f7d3f7d | René Nussbaumer | raise errors.CommandError("Unable to start all instances." |
696 | 1f7d3f7d | René Nussbaumer | " Fail reason: %s; output: %s" % |
697 | 1f7d3f7d | René Nussbaumer | (result.fail_reason, result.output)) |
698 | 1f7d3f7d | René Nussbaumer | |
699 | 1f7d3f7d | René Nussbaumer | # R0201: Method could be a function |
700 | cc2cccfc | Guido Trotter | # TODO: make this overridable, for some verify errors |
701 | b459a848 | Andrea Spadaccini | def _VerifyCluster(self): # pylint: disable=R0201 |
702 | 1f7d3f7d | René Nussbaumer | """Runs gnt-cluster verify to verify the health. |
703 | 1f7d3f7d | René Nussbaumer | |
704 | 1f7d3f7d | René Nussbaumer | @raise errors.ProgrammError: If cluster fails on verification |
705 | 1f7d3f7d | René Nussbaumer | |
706 | 1f7d3f7d | René Nussbaumer | """ |
707 | 1f7d3f7d | René Nussbaumer | result = utils.RunCmd(["gnt-cluster", "verify"]) |
708 | 1f7d3f7d | René Nussbaumer | if result.failed: |
709 | 1f7d3f7d | René Nussbaumer | raise errors.CommandError("Verification of cluster failed." |
710 | 1f7d3f7d | René Nussbaumer | " Fail reason: %s; output: %s" % |
711 | 1f7d3f7d | René Nussbaumer | (result.fail_reason, result.output)) |
712 | 1f7d3f7d | René Nussbaumer | |
713 | 1f7d3f7d | René Nussbaumer | def Merge(self): |
714 | 1f7d3f7d | René Nussbaumer | """Does the actual merge. |
715 | 1f7d3f7d | René Nussbaumer | |
716 | 1f7d3f7d | René Nussbaumer | It runs all the steps in the right order and updates the user about steps |
717 | 1f7d3f7d | René Nussbaumer | taken. Also it keeps track of rollback_steps to undo everything. |
718 | 1f7d3f7d | René Nussbaumer | |
719 | 1f7d3f7d | René Nussbaumer | """ |
720 | 1f7d3f7d | René Nussbaumer | rbsteps = [] |
721 | 1f7d3f7d | René Nussbaumer | try: |
722 | 1f7d3f7d | René Nussbaumer | logging.info("Pre cluster verification") |
723 | 1f7d3f7d | René Nussbaumer | self._VerifyCluster() |
724 | 1f7d3f7d | René Nussbaumer | |
725 | 8864d152 | Guido Trotter | logging.info("Prepare authorized_keys") |
726 | 8864d152 | Guido Trotter | rbsteps.append("Remove our key from authorized_keys on nodes:" |
727 | 8864d152 | Guido Trotter | " %(nodes)s") |
728 | 8864d152 | Guido Trotter | self._PrepareAuthorizedKeys() |
729 | 8864d152 | Guido Trotter | |
730 | 1f7d3f7d | René Nussbaumer | rbsteps.append("Start all instances again on the merging" |
731 | 1f7d3f7d | René Nussbaumer | " clusters: %(clusters)s") |
732 | 1fb5f905 | Andrea Spadaccini | if self.stop_instances: |
733 | 1fb5f905 | Andrea Spadaccini | logging.info("Stopping merging instances (takes a while)") |
734 | 1fb5f905 | Andrea Spadaccini | self._StopMergingInstances() |
735 | 1fb5f905 | Andrea Spadaccini | logging.info("Checking that no instances are running on the mergees") |
736 | 1fb5f905 | Andrea Spadaccini | instances_running = self._CheckRunningInstances() |
737 | 1fb5f905 | Andrea Spadaccini | if instances_running: |
738 | 1fb5f905 | Andrea Spadaccini | raise errors.CommandError("Some instances are still running on the" |
739 | 1fb5f905 | Andrea Spadaccini | " mergees") |
740 | 1f7d3f7d | René Nussbaumer | logging.info("Disable watcher") |
741 | 1f7d3f7d | René Nussbaumer | self._DisableWatcher() |
742 | 1f7d3f7d | René Nussbaumer | logging.info("Stop daemons on merging nodes") |
743 | 1f7d3f7d | René Nussbaumer | self._StopDaemons() |
744 | 1f7d3f7d | René Nussbaumer | logging.info("Merging config") |
745 | 1f7d3f7d | René Nussbaumer | self._FetchRemoteConfig() |
746 | b972c223 | Andrea Spadaccini | logging.info("Removing master IPs on mergee master nodes") |
747 | b972c223 | Andrea Spadaccini | self._RemoveMasterIps() |
748 | d8aab233 | René Nussbaumer | |
749 | 9c03a67a | Stephen Shirley | logging.info("Stopping master daemon") |
750 | 9c03a67a | Stephen Shirley | self._KillMasterDaemon() |
751 | 9c03a67a | Stephen Shirley | |
752 | 9c03a67a | Stephen Shirley | rbsteps.append("Restore %s from another master candidate" |
753 | 9c03a67a | Stephen Shirley | " and restart master daemon" % |
754 | 9c03a67a | Stephen Shirley | constants.CLUSTER_CONF_FILE) |
755 | 9c03a67a | Stephen Shirley | self._MergeConfig() |
756 | 9c03a67a | Stephen Shirley | self._StartMasterDaemon(no_vote=True) |
757 | 9c03a67a | Stephen Shirley | |
758 | 9c03a67a | Stephen Shirley | # Point of no return, delete rbsteps |
759 | 9c03a67a | Stephen Shirley | del rbsteps[:] |
760 | 9c03a67a | Stephen Shirley | |
761 | 9c03a67a | Stephen Shirley | logging.warning("We are at the point of no return. Merge can not easily" |
762 | 9c03a67a | Stephen Shirley | " be undone after this point.") |
763 | 9c03a67a | Stephen Shirley | logging.info("Readd nodes") |
764 | 9c03a67a | Stephen Shirley | self._ReaddMergedNodesAndRedist() |
765 | 9c03a67a | Stephen Shirley | |
766 | 9c03a67a | Stephen Shirley | logging.info("Merge done, restart master daemon normally") |
767 | 9c03a67a | Stephen Shirley | self._KillMasterDaemon() |
768 | 9c03a67a | Stephen Shirley | self._StartMasterDaemon() |
769 | d8aab233 | René Nussbaumer | |
770 | 620a9c62 | Guido Trotter | if self.restart == _RESTART_ALL: |
771 | 620a9c62 | Guido Trotter | logging.info("Starting instances again") |
772 | 620a9c62 | Guido Trotter | self._StartupAllInstances() |
773 | 620a9c62 | Guido Trotter | else: |
774 | 620a9c62 | Guido Trotter | logging.info("Not starting instances again") |
775 | 1f7d3f7d | René Nussbaumer | logging.info("Post cluster verification") |
776 | 1f7d3f7d | René Nussbaumer | self._VerifyCluster() |
777 | 1f7d3f7d | René Nussbaumer | except errors.GenericError, e: |
778 | 1f7d3f7d | René Nussbaumer | logging.exception(e) |
779 | 1f7d3f7d | René Nussbaumer | |
780 | 1f7d3f7d | René Nussbaumer | if rbsteps: |
781 | 1f7d3f7d | René Nussbaumer | nodes = Flatten([data.nodes for data in self.merger_data]) |
782 | 1f7d3f7d | René Nussbaumer | info = { |
783 | 1f7d3f7d | René Nussbaumer | "clusters": self.clusters, |
784 | 1f7d3f7d | René Nussbaumer | "nodes": nodes, |
785 | 1f7d3f7d | René Nussbaumer | } |
786 | 1f7d3f7d | René Nussbaumer | logging.critical("In order to rollback do the following:") |
787 | 1f7d3f7d | René Nussbaumer | for step in rbsteps: |
788 | 3f1cf151 | René Nussbaumer | logging.critical(" * %s", step % info) |
789 | 1f7d3f7d | René Nussbaumer | else: |
790 | 1f7d3f7d | René Nussbaumer | logging.critical("Nothing to rollback.") |
791 | 1f7d3f7d | René Nussbaumer | |
792 | 1f7d3f7d | René Nussbaumer | # TODO: Keep track of steps done for a flawless resume? |
793 | 1f7d3f7d | René Nussbaumer | |
794 | 1f7d3f7d | René Nussbaumer | def Cleanup(self): |
795 | 1f7d3f7d | René Nussbaumer | """Clean up our environment. |
796 | 1f7d3f7d | René Nussbaumer | |
797 | 1f7d3f7d | René Nussbaumer | This cleans up remote private keys and configs and after that |
798 | 1f7d3f7d | René Nussbaumer | deletes the temporary directory. |
799 | 1f7d3f7d | René Nussbaumer | |
800 | 1f7d3f7d | René Nussbaumer | """ |
801 | 1f7d3f7d | René Nussbaumer | shutil.rmtree(self.work_dir) |
802 | 1f7d3f7d | René Nussbaumer | |
803 | 1f7d3f7d | René Nussbaumer | |
804 | 1f7d3f7d | René Nussbaumer | def SetupLogging(options): |
805 | 1f7d3f7d | René Nussbaumer | """Setting up logging infrastructure. |
806 | 1f7d3f7d | René Nussbaumer | |
807 | 1f7d3f7d | René Nussbaumer | @param options: Parsed command line options |
808 | 1f7d3f7d | René Nussbaumer | |
809 | 1f7d3f7d | René Nussbaumer | """ |
810 | 1f7d3f7d | René Nussbaumer | formatter = logging.Formatter("%(asctime)s: %(levelname)s %(message)s") |
811 | 1f7d3f7d | René Nussbaumer | |
812 | 1f7d3f7d | René Nussbaumer | stderr_handler = logging.StreamHandler() |
813 | 1f7d3f7d | René Nussbaumer | stderr_handler.setFormatter(formatter) |
814 | 1f7d3f7d | René Nussbaumer | if options.debug: |
815 | 1f7d3f7d | René Nussbaumer | stderr_handler.setLevel(logging.NOTSET) |
816 | 1f7d3f7d | René Nussbaumer | elif options.verbose: |
817 | 1f7d3f7d | René Nussbaumer | stderr_handler.setLevel(logging.INFO) |
818 | 1f7d3f7d | René Nussbaumer | else: |
819 | a6c8fd10 | Stephen Shirley | stderr_handler.setLevel(logging.WARNING) |
820 | 1f7d3f7d | René Nussbaumer | |
821 | 1f7d3f7d | René Nussbaumer | root_logger = logging.getLogger("") |
822 | 1f7d3f7d | René Nussbaumer | root_logger.setLevel(logging.NOTSET) |
823 | 1f7d3f7d | René Nussbaumer | root_logger.addHandler(stderr_handler) |
824 | 1f7d3f7d | René Nussbaumer | |
825 | 1f7d3f7d | René Nussbaumer | |
826 | 1f7d3f7d | René Nussbaumer | def main(): |
827 | 1f7d3f7d | René Nussbaumer | """Main routine. |
828 | 1f7d3f7d | René Nussbaumer | |
829 | 1f7d3f7d | René Nussbaumer | """ |
830 | 1f7d3f7d | René Nussbaumer | program = os.path.basename(sys.argv[0]) |
831 | 1f7d3f7d | René Nussbaumer | |
832 | d2b17e37 | Guido Trotter | parser = optparse.OptionParser(usage="%%prog [options...] <cluster...>", |
833 | d2b17e37 | Guido Trotter | prog=program) |
834 | 1f7d3f7d | René Nussbaumer | parser.add_option(cli.DEBUG_OPT) |
835 | 1f7d3f7d | René Nussbaumer | parser.add_option(cli.VERBOSE_OPT) |
836 | 1f7d3f7d | René Nussbaumer | parser.add_option(PAUSE_PERIOD_OPT) |
837 | 1a615be0 | Stephen Shirley | parser.add_option(GROUPS_OPT) |
838 | 620a9c62 | Guido Trotter | parser.add_option(RESTART_OPT) |
839 | 1fcd3b81 | Guido Trotter | parser.add_option(PARAMS_OPT) |
840 | 1fb5f905 | Andrea Spadaccini | parser.add_option(SKIP_STOP_INSTANCES_OPT) |
841 | 1f7d3f7d | René Nussbaumer | |
842 | 1f7d3f7d | René Nussbaumer | (options, args) = parser.parse_args() |
843 | 1f7d3f7d | René Nussbaumer | |
844 | 1f7d3f7d | René Nussbaumer | SetupLogging(options) |
845 | 1f7d3f7d | René Nussbaumer | |
846 | 1f7d3f7d | René Nussbaumer | if not args: |
847 | 1f7d3f7d | René Nussbaumer | parser.error("No clusters specified") |
848 | 1f7d3f7d | René Nussbaumer | |
849 | 1a615be0 | Stephen Shirley | cluster_merger = Merger(utils.UniqueSequence(args), options.pause_period, |
850 | 1fb5f905 | Andrea Spadaccini | options.groups, options.restart, options.params, |
851 | 1fb5f905 | Andrea Spadaccini | options.stop_instances) |
852 | 1f7d3f7d | René Nussbaumer | try: |
853 | 1f7d3f7d | René Nussbaumer | try: |
854 | 1f7d3f7d | René Nussbaumer | cluster_merger.Setup() |
855 | 1f7d3f7d | René Nussbaumer | cluster_merger.Merge() |
856 | 1f7d3f7d | René Nussbaumer | except errors.GenericError, e: |
857 | 1f7d3f7d | René Nussbaumer | logging.exception(e) |
858 | 1f7d3f7d | René Nussbaumer | return constants.EXIT_FAILURE |
859 | 1f7d3f7d | René Nussbaumer | finally: |
860 | 1f7d3f7d | René Nussbaumer | cluster_merger.Cleanup() |
861 | 1f7d3f7d | René Nussbaumer | |
862 | 1f7d3f7d | René Nussbaumer | return constants.EXIT_SUCCESS |
863 | 1f7d3f7d | René Nussbaumer | |
864 | 1f7d3f7d | René Nussbaumer | |
865 | 1f7d3f7d | René Nussbaumer | if __name__ == "__main__": |
866 | 1f7d3f7d | René Nussbaumer | sys.exit(main()) |