root / lib / watcher / nodemaint.py @ 1e0d3321
History | View | Annotate | Download (4.8 kB)
1 | 9ca87fb3 | Michael Hanselmann | #
|
---|---|---|---|
2 | 9ca87fb3 | Michael Hanselmann | #
|
3 | 9ca87fb3 | Michael Hanselmann | |
4 | 9ca87fb3 | Michael Hanselmann | # Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011 Google Inc.
|
5 | 9ca87fb3 | Michael Hanselmann | #
|
6 | 9ca87fb3 | Michael Hanselmann | # This program is free software; you can redistribute it and/or modify
|
7 | 9ca87fb3 | Michael Hanselmann | # it under the terms of the GNU General Public License as published by
|
8 | 9ca87fb3 | Michael Hanselmann | # the Free Software Foundation; either version 2 of the License, or
|
9 | 9ca87fb3 | Michael Hanselmann | # (at your option) any later version.
|
10 | 9ca87fb3 | Michael Hanselmann | #
|
11 | 9ca87fb3 | Michael Hanselmann | # This program is distributed in the hope that it will be useful, but
|
12 | 9ca87fb3 | Michael Hanselmann | # WITHOUT ANY WARRANTY; without even the implied warranty of
|
13 | 9ca87fb3 | Michael Hanselmann | # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
|
14 | 9ca87fb3 | Michael Hanselmann | # General Public License for more details.
|
15 | 9ca87fb3 | Michael Hanselmann | #
|
16 | 9ca87fb3 | Michael Hanselmann | # You should have received a copy of the GNU General Public License
|
17 | 9ca87fb3 | Michael Hanselmann | # along with this program; if not, write to the Free Software
|
18 | 9ca87fb3 | Michael Hanselmann | # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
|
19 | 9ca87fb3 | Michael Hanselmann | # 02110-1301, USA.
|
20 | 9ca87fb3 | Michael Hanselmann | |
21 | 9ca87fb3 | Michael Hanselmann | |
22 | 9ca87fb3 | Michael Hanselmann | """Module doing node maintenance for Ganeti watcher.
|
23 | 9ca87fb3 | Michael Hanselmann |
|
24 | 9ca87fb3 | Michael Hanselmann | """
|
25 | 9ca87fb3 | Michael Hanselmann | |
26 | 9ca87fb3 | Michael Hanselmann | import logging |
27 | 9ca87fb3 | Michael Hanselmann | |
28 | 9ca87fb3 | Michael Hanselmann | from ganeti import bdev |
29 | 9ca87fb3 | Michael Hanselmann | from ganeti import constants |
30 | 9ca87fb3 | Michael Hanselmann | from ganeti import errors |
31 | 9ca87fb3 | Michael Hanselmann | from ganeti import hypervisor |
32 | 9ca87fb3 | Michael Hanselmann | from ganeti import netutils |
33 | 9ca87fb3 | Michael Hanselmann | from ganeti import ssconf |
34 | 9ca87fb3 | Michael Hanselmann | from ganeti import utils |
35 | 9ca87fb3 | Michael Hanselmann | from ganeti import confd |
36 | 9ca87fb3 | Michael Hanselmann | |
37 | b459a848 | Andrea Spadaccini | import ganeti.confd.client # pylint: disable=W0611 |
38 | 9ca87fb3 | Michael Hanselmann | |
39 | 9ca87fb3 | Michael Hanselmann | |
40 | 9ca87fb3 | Michael Hanselmann | class NodeMaintenance(object): |
41 | 9ca87fb3 | Michael Hanselmann | """Talks to confd daemons and possible shutdown instances/drbd devices.
|
42 | 9ca87fb3 | Michael Hanselmann |
|
43 | 9ca87fb3 | Michael Hanselmann | """
|
44 | 9ca87fb3 | Michael Hanselmann | def __init__(self): |
45 | 9ca87fb3 | Michael Hanselmann | self.store_cb = confd.client.StoreResultCallback()
|
46 | 9ca87fb3 | Michael Hanselmann | self.filter_cb = confd.client.ConfdFilterCallback(self.store_cb) |
47 | 9ca87fb3 | Michael Hanselmann | self.confd_client = confd.client.GetConfdClient(self.filter_cb) |
48 | 9ca87fb3 | Michael Hanselmann | |
49 | 9ca87fb3 | Michael Hanselmann | @staticmethod
|
50 | 9ca87fb3 | Michael Hanselmann | def ShouldRun(): |
51 | 9ca87fb3 | Michael Hanselmann | """Checks whether node maintenance should run.
|
52 | 9ca87fb3 | Michael Hanselmann |
|
53 | 9ca87fb3 | Michael Hanselmann | """
|
54 | 9ca87fb3 | Michael Hanselmann | try:
|
55 | 9ca87fb3 | Michael Hanselmann | return ssconf.SimpleStore().GetMaintainNodeHealth()
|
56 | 9ca87fb3 | Michael Hanselmann | except errors.ConfigurationError, err:
|
57 | 9ca87fb3 | Michael Hanselmann | logging.error("Configuration error, not activating node maintenance: %s",
|
58 | 9ca87fb3 | Michael Hanselmann | err) |
59 | 9ca87fb3 | Michael Hanselmann | return False |
60 | 9ca87fb3 | Michael Hanselmann | |
61 | 9ca87fb3 | Michael Hanselmann | @staticmethod
|
62 | 9ca87fb3 | Michael Hanselmann | def GetRunningInstances(): |
63 | 9ca87fb3 | Michael Hanselmann | """Compute list of hypervisor/running instances.
|
64 | 9ca87fb3 | Michael Hanselmann |
|
65 | 9ca87fb3 | Michael Hanselmann | """
|
66 | 9ca87fb3 | Michael Hanselmann | hyp_list = ssconf.SimpleStore().GetHypervisorList() |
67 | 9ca87fb3 | Michael Hanselmann | results = [] |
68 | 9ca87fb3 | Michael Hanselmann | for hv_name in hyp_list: |
69 | 9ca87fb3 | Michael Hanselmann | try:
|
70 | 9ca87fb3 | Michael Hanselmann | hv = hypervisor.GetHypervisor(hv_name) |
71 | 9ca87fb3 | Michael Hanselmann | ilist = hv.ListInstances() |
72 | 9ca87fb3 | Michael Hanselmann | results.extend([(iname, hv_name) for iname in ilist]) |
73 | b459a848 | Andrea Spadaccini | except: # pylint: disable=W0702 |
74 | 9ca87fb3 | Michael Hanselmann | logging.error("Error while listing instances for hypervisor %s",
|
75 | 9ca87fb3 | Michael Hanselmann | hv_name, exc_info=True)
|
76 | 9ca87fb3 | Michael Hanselmann | return results
|
77 | 9ca87fb3 | Michael Hanselmann | |
78 | 9ca87fb3 | Michael Hanselmann | @staticmethod
|
79 | 9ca87fb3 | Michael Hanselmann | def GetUsedDRBDs(): |
80 | 9ca87fb3 | Michael Hanselmann | """Get list of used DRBD minors.
|
81 | 9ca87fb3 | Michael Hanselmann |
|
82 | 9ca87fb3 | Michael Hanselmann | """
|
83 | 9ca87fb3 | Michael Hanselmann | return bdev.DRBD8.GetUsedDevs().keys()
|
84 | 9ca87fb3 | Michael Hanselmann | |
85 | 9ca87fb3 | Michael Hanselmann | @classmethod
|
86 | 9ca87fb3 | Michael Hanselmann | def DoMaintenance(cls, role): |
87 | 9ca87fb3 | Michael Hanselmann | """Maintain the instance list.
|
88 | 9ca87fb3 | Michael Hanselmann |
|
89 | 9ca87fb3 | Michael Hanselmann | """
|
90 | 9ca87fb3 | Michael Hanselmann | if role == constants.CONFD_NODE_ROLE_OFFLINE:
|
91 | 9ca87fb3 | Michael Hanselmann | inst_running = cls.GetRunningInstances() |
92 | 9ca87fb3 | Michael Hanselmann | cls.ShutdownInstances(inst_running) |
93 | 9ca87fb3 | Michael Hanselmann | drbd_running = cls.GetUsedDRBDs() |
94 | 9ca87fb3 | Michael Hanselmann | cls.ShutdownDRBD(drbd_running) |
95 | 9ca87fb3 | Michael Hanselmann | else:
|
96 | 9ca87fb3 | Michael Hanselmann | logging.debug("Not doing anything for role %s", role)
|
97 | 9ca87fb3 | Michael Hanselmann | |
98 | 9ca87fb3 | Michael Hanselmann | @staticmethod
|
99 | 9ca87fb3 | Michael Hanselmann | def ShutdownInstances(inst_running): |
100 | 9ca87fb3 | Michael Hanselmann | """Shutdown running instances.
|
101 | 9ca87fb3 | Michael Hanselmann |
|
102 | 9ca87fb3 | Michael Hanselmann | """
|
103 | 9ca87fb3 | Michael Hanselmann | names_running = set([i[0] for i in inst_running]) |
104 | 9ca87fb3 | Michael Hanselmann | if names_running:
|
105 | 9ca87fb3 | Michael Hanselmann | logging.info("Following instances should not be running,"
|
106 | 9ca87fb3 | Michael Hanselmann | " shutting them down: %s", utils.CommaJoin(names_running))
|
107 | 9ca87fb3 | Michael Hanselmann | # this dictionary will collapse duplicate instance names (only
|
108 | 9ca87fb3 | Michael Hanselmann | # xen pvm/vhm) into a single key, which is fine
|
109 | 9ca87fb3 | Michael Hanselmann | i2h = dict(inst_running)
|
110 | 9ca87fb3 | Michael Hanselmann | for name in names_running: |
111 | 9ca87fb3 | Michael Hanselmann | hv_name = i2h[name] |
112 | 9ca87fb3 | Michael Hanselmann | hv = hypervisor.GetHypervisor(hv_name) |
113 | 9ca87fb3 | Michael Hanselmann | hv.StopInstance(None, force=True, name=name) |
114 | 9ca87fb3 | Michael Hanselmann | |
115 | 9ca87fb3 | Michael Hanselmann | @staticmethod
|
116 | 9ca87fb3 | Michael Hanselmann | def ShutdownDRBD(drbd_running): |
117 | 9ca87fb3 | Michael Hanselmann | """Shutdown active DRBD devices.
|
118 | 9ca87fb3 | Michael Hanselmann |
|
119 | 9ca87fb3 | Michael Hanselmann | """
|
120 | 9ca87fb3 | Michael Hanselmann | if drbd_running:
|
121 | 9ca87fb3 | Michael Hanselmann | logging.info("Following DRBD minors should not be active,"
|
122 | 9ca87fb3 | Michael Hanselmann | " shutting them down: %s", utils.CommaJoin(drbd_running))
|
123 | 9ca87fb3 | Michael Hanselmann | for minor in drbd_running: |
124 | b459a848 | Andrea Spadaccini | # pylint: disable=W0212
|
125 | 9ca87fb3 | Michael Hanselmann | # using the private method as is, pending enhancements to the DRBD
|
126 | 9ca87fb3 | Michael Hanselmann | # interface
|
127 | 9ca87fb3 | Michael Hanselmann | bdev.DRBD8._ShutdownAll(minor) |
128 | 9ca87fb3 | Michael Hanselmann | |
129 | 9ca87fb3 | Michael Hanselmann | def Exec(self): |
130 | 9ca87fb3 | Michael Hanselmann | """Check node status versus cluster desired state.
|
131 | 9ca87fb3 | Michael Hanselmann |
|
132 | 9ca87fb3 | Michael Hanselmann | """
|
133 | aa224134 | Iustin Pop | if not constants.ENABLE_CONFD: |
134 | aa224134 | Iustin Pop | logging.warning("Confd use not enabled, cannot do maintenance")
|
135 | aa224134 | Iustin Pop | return
|
136 | aa224134 | Iustin Pop | |
137 | 9ca87fb3 | Michael Hanselmann | my_name = netutils.Hostname.GetSysName() |
138 | e687ec01 | Michael Hanselmann | req = \ |
139 | e687ec01 | Michael Hanselmann | confd.client.ConfdClientRequest(type=constants.CONFD_REQ_NODE_ROLE_BYNAME, |
140 | e687ec01 | Michael Hanselmann | query=my_name) |
141 | 9ca87fb3 | Michael Hanselmann | self.confd_client.SendRequest(req, async=False, coverage=-1) |
142 | 9ca87fb3 | Michael Hanselmann | timed_out, _, _ = self.confd_client.WaitForReply(req.rsalt)
|
143 | 9ca87fb3 | Michael Hanselmann | if not timed_out: |
144 | 9ca87fb3 | Michael Hanselmann | # should have a valid response
|
145 | 9ca87fb3 | Michael Hanselmann | status, result = self.store_cb.GetResponse(req.rsalt)
|
146 | 9ca87fb3 | Michael Hanselmann | assert status, "Missing result but received replies" |
147 | 9ca87fb3 | Michael Hanselmann | if not self.filter_cb.consistent[req.rsalt]: |
148 | 9ca87fb3 | Michael Hanselmann | logging.warning("Inconsistent replies, not doing anything")
|
149 | 9ca87fb3 | Michael Hanselmann | return
|
150 | 9ca87fb3 | Michael Hanselmann | self.DoMaintenance(result.server_reply.answer)
|
151 | 9ca87fb3 | Michael Hanselmann | else:
|
152 | 9ca87fb3 | Michael Hanselmann | logging.warning("Confd query timed out, cannot do maintenance actions") |