Revision 0e9a423f snf-cyclades-app/synnefo/logic/management/commands/reconcile.py
| b/snf-cyclades-app/synnefo/logic/management/commands/reconcile.py | ||
|---|---|---|
| 36 | 36 |
""" |
| 37 | 37 |
import sys |
| 38 | 38 |
import datetime |
| 39 |
import subprocess |
|
| 39 | 40 |
|
| 40 | 41 |
from optparse import make_option |
| 41 | 42 |
|
| ... | ... | |
| 65 | 66 |
make_option('--detect-build-errors', action='store_true',
|
| 66 | 67 |
dest='detect_build_errors', default=False, |
| 67 | 68 |
help='Detect instances with build error'), |
| 69 |
make_option('--detect-unsynced-nics', action='store_true',
|
|
| 70 |
dest='detect_unsynced_nics', default=False, |
|
| 71 |
help='Detect unsynced nics between DB and Ganeti'), |
|
| 68 | 72 |
make_option('--detect-all', action='store_true',
|
| 69 | 73 |
dest='detect_all', |
| 70 | 74 |
default=False, help='Enable all --detect-* arguments'), |
| ... | ... | |
| 78 | 82 |
make_option('--fix-build-errors', action='store_true',
|
| 79 | 83 |
dest='fix_build_errors', default=False, |
| 80 | 84 |
help='Fix (remove) instances with build errors'), |
| 85 |
make_option('--fix-unsynced-nics', action='store_true',
|
|
| 86 |
dest='fix_unsynced_nics', default=False, |
|
| 87 |
help='Fix unsynced nics between DB and Ganeti'), |
|
| 81 | 88 |
make_option('--fix-all', action='store_true', dest='fix_all',
|
| 82 | 89 |
default=False, help='Enable all --fix-* arguments')) |
| 83 | 90 |
|
| ... | ... | |
| 109 | 116 |
D = reconciliation.get_servers_from_db() |
| 110 | 117 |
G = reconciliation.get_instances_from_ganeti() |
| 111 | 118 |
|
| 119 |
DBNics = reconciliation.get_nics_from_db() |
|
| 120 |
GNics = reconciliation.get_nics_from_ganeti() |
|
| 112 | 121 |
# |
| 113 | 122 |
# Detect problems |
| 114 | 123 |
# |
| ... | ... | |
| 152 | 161 |
elif verbosity == 2: |
| 153 | 162 |
print >> sys.stderr, "Found no instances with build errors." |
| 154 | 163 |
|
| 164 |
if options['detect_unsynced_nics']: |
|
| 165 |
def pretty_print_nics(nics): |
|
| 166 |
if not nics: |
|
| 167 |
print ''.ljust(18) + 'None' |
|
| 168 |
for index, info in nics.items(): |
|
| 169 |
print ''.ljust(18) + 'nic/' + str(index) + ': MAC: %s, IP: %s, Network: %s' % \ |
|
| 170 |
(info['mac'], info['ipv4'], info['network']) |
|
| 171 |
|
|
| 172 |
unsynced_nics = reconciliation.unsynced_nics(DBNics, GNics) |
|
| 173 |
if len(unsynced_nics) > 0: |
|
| 174 |
print >> sys.stderr, "The nics of servers with the folloing ID's "\ |
|
| 175 |
"are unsynced:" |
|
| 176 |
for id, nics in unsynced_nics.items(): |
|
| 177 |
print ''.ljust(2) + '%6d:' % id |
|
| 178 |
print ''.ljust(8) + '%8s:' % 'DB' |
|
| 179 |
pretty_print_nics(nics[0]) |
|
| 180 |
print ''.ljust(8) + '%8s:' % 'Ganeti' |
|
| 181 |
pretty_print_nics(nics[1]) |
|
| 182 |
elif verbosity == 2: |
|
| 183 |
print >> sys.stderr, "All instance nics are synced." |
|
| 184 |
|
|
| 155 | 185 |
# |
| 156 | 186 |
# Then fix them |
| 157 | 187 |
# |
| ... | ... | |
| 184 | 214 |
opcode = "OP_INSTANCE_REBOOT" if ganeti_up \ |
| 185 | 215 |
else "OP_INSTANCE_SHUTDOWN" |
| 186 | 216 |
event_time = datetime.datetime.now() |
| 187 |
backend.process_op_status(vm=vm, etime=event_time ,jobid=-0,
|
|
| 217 |
backend.process_op_status(vm=vm, etime=event_time, jobid=-0,
|
|
| 188 | 218 |
opcode=opcode, status='success', |
| 189 | 219 |
logmsg='Reconciliation: simulated Ganeti event') |
| 190 | 220 |
print >> sys.stderr, " ...done" |
| ... | ... | |
| 195 | 225 |
for id in build_errors: |
| 196 | 226 |
vm = VirtualMachine.objects.get(pk=id) |
| 197 | 227 |
event_time = datetime.datetime.now() |
| 198 |
backend.process_op_status(vm=vm, etime=event_time ,jobid=-0,
|
|
| 228 |
backend.process_op_status(vm=vm, etime=event_time, jobid=-0,
|
|
| 199 | 229 |
opcode="OP_INSTANCE_CREATE", status='error', |
| 200 | 230 |
logmsg='Reconciliation: simulated Ganeti event') |
| 201 | 231 |
print >> sys.stderr, " ...done" |
| 202 | 232 |
|
| 233 |
if options['fix_unsynced_nics'] and len(unsynced_nics) > 0: |
|
| 234 |
print >> sys.stderr, "Setting the nics of %d out-of-sync VMs:" % \ |
|
| 235 |
len(unsynced_nics) |
|
| 236 |
for id, nics in unsynced_nics.items(): |
|
| 237 |
vm = VirtualMachine.objects.get(pk=id) |
|
| 238 |
nics = nics[1] # Ganeti nics |
|
| 239 |
if nics == {}: # No nics
|
|
| 240 |
vm.nics.all.delete() |
|
| 241 |
continue |
|
| 242 |
for index, nic in nics.items(): |
|
| 243 |
# Produce ipv6 |
|
| 244 |
ipv6 = mac2eui64(nic['mac'], settings.PUBLIC_IPV6_PREFIX) |
|
| 245 |
nic['ipv6'] = ipv6 |
|
| 246 |
# Rename ipv4 to ip |
|
| 247 |
nic['ip'] = nic['ipv4'] |
|
| 248 |
# Dict to sorted list |
|
| 249 |
final_nics = [] |
|
| 250 |
nics_keys = nics.keys() |
|
| 251 |
nics_keys.sort() |
|
| 252 |
for i in nics_keys: |
|
| 253 |
if nics[i]['network']: |
|
| 254 |
final_nics.append(nics[i]) |
|
| 255 |
else: |
|
| 256 |
print 'Network of nic %d of vm %s is None. ' \ |
|
| 257 |
'Can not reconcile' % (i, vm.backend_vm_id) |
|
| 258 |
event_time = datetime.datetime.now() |
|
| 259 |
backend.process_net_status(vm=vm, etime=event_time, nics=final_nics) |
|
| 260 |
print >> sys.stderr, " ...done" |
|
| 261 |
|
|
| 262 |
|
|
| 263 |
def mac2eui64(mac, prefixstr): |
|
| 264 |
process = subprocess.Popen(["mac2eui64", mac, prefixstr], |
|
| 265 |
stdout=subprocess.PIPE) |
|
| 266 |
return process.stdout.read().rstrip() |
|
Also available in: Unified diff