Statistics
| Branch: | Tag: | Revision:

root / snf-cyclades-app / synnefo / logic / reconciliation.py @ d7841399

History | View | Annotate | Download (10.1 kB)

1 9fea53cc Vangelis Koukis
#!/usr/bin/env python
2 9fea53cc Vangelis Koukis
# -*- coding: utf-8 -*-
3 9fea53cc Vangelis Koukis
#
4 9fea53cc Vangelis Koukis
# Copyright 2011 GRNET S.A. All rights reserved.
5 9fea53cc Vangelis Koukis
#
6 9fea53cc Vangelis Koukis
# Redistribution and use in source and binary forms, with or
7 9fea53cc Vangelis Koukis
# without modification, are permitted provided that the following
8 9fea53cc Vangelis Koukis
# conditions are met:
9 9fea53cc Vangelis Koukis
#
10 9fea53cc Vangelis Koukis
#   1. Redistributions of source code must retain the above
11 9fea53cc Vangelis Koukis
#      copyright notice, this list of conditions and the following
12 9fea53cc Vangelis Koukis
#      disclaimer.
13 9fea53cc Vangelis Koukis
#
14 9fea53cc Vangelis Koukis
#   2. Redistributions in binary form must reproduce the above
15 9fea53cc Vangelis Koukis
#      copyright notice, this list of conditions and the following
16 9fea53cc Vangelis Koukis
#      disclaimer in the documentation and/or other materials
17 9fea53cc Vangelis Koukis
#      provided with the distribution.
18 9fea53cc Vangelis Koukis
#
19 9fea53cc Vangelis Koukis
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
20 9fea53cc Vangelis Koukis
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
21 9fea53cc Vangelis Koukis
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22 9fea53cc Vangelis Koukis
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
23 9fea53cc Vangelis Koukis
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24 9fea53cc Vangelis Koukis
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25 9fea53cc Vangelis Koukis
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
26 9fea53cc Vangelis Koukis
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
27 9fea53cc Vangelis Koukis
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
28 9fea53cc Vangelis Koukis
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
29 9fea53cc Vangelis Koukis
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30 9fea53cc Vangelis Koukis
# POSSIBILITY OF SUCH DAMAGE.
31 9fea53cc Vangelis Koukis
#
32 9fea53cc Vangelis Koukis
# The views and conclusions contained in the software and
33 9fea53cc Vangelis Koukis
# documentation are those of the authors and should not be
34 9fea53cc Vangelis Koukis
# interpreted as representing official policies, either expressed
35 9fea53cc Vangelis Koukis
# or implied, of GRNET S.A.
36 9fea53cc Vangelis Koukis
#
37 9fea53cc Vangelis Koukis
"""Business logic for reconciliation
38 9fea53cc Vangelis Koukis

39 9fea53cc Vangelis Koukis
Reconcile the contents of the DB with the actual state of the
40 9fea53cc Vangelis Koukis
Ganeti backend.
41 9fea53cc Vangelis Koukis

42 9fea53cc Vangelis Koukis
Let D be the set of VMs in the DB, G the set of VMs in Ganeti.
43 9fea53cc Vangelis Koukis
RULES:
44 9fea53cc Vangelis Koukis
    R1. Stale servers in DB:
45 9fea53cc Vangelis Koukis
            For any v in D but not in G:
46 9fea53cc Vangelis Koukis
            Set deleted=True.
47 9fea53cc Vangelis Koukis
    R2. Orphan instances in Ganet:
48 9fea53cc Vangelis Koukis
            For any v in G with deleted=True in D:
49 9fea53cc Vangelis Koukis
            Issue OP_INSTANCE_DESTROY.
50 9fea53cc Vangelis Koukis
    R3. Unsynced operstate:
51 9fea53cc Vangelis Koukis
            For any v whose operating state differs between G and V:
52 9fea53cc Vangelis Koukis
            Set the operating state in D based on the state in G.
53 9fea53cc Vangelis Koukis
In the code, D, G are Python dicts mapping instance ids to operating state.
54 9fea53cc Vangelis Koukis
For D, the operating state is chosen from VirtualMachine.OPER_STATES.
55 9fea53cc Vangelis Koukis
For G, the operating state is True if the machine is up, False otherwise.
56 9fea53cc Vangelis Koukis

57 9fea53cc Vangelis Koukis
"""
58 9fea53cc Vangelis Koukis
59 9e98ba3c Giorgos Verigakis
import logging
60 9fea53cc Vangelis Koukis
import sys
61 0e9a423f Christos Stavrakakis
import itertools
62 9fea53cc Vangelis Koukis
63 9fea53cc Vangelis Koukis
from django.core.management import setup_environ
64 9fea53cc Vangelis Koukis
try:
65 9fea53cc Vangelis Koukis
    from synnefo import settings
66 9fea53cc Vangelis Koukis
except ImportError:
67 9fea53cc Vangelis Koukis
    raise Exception("Cannot import settings, make sure PYTHONPATH contains "
68 9fea53cc Vangelis Koukis
                    "the parent directory of the Synnefo Django project.")
69 9fea53cc Vangelis Koukis
setup_environ(settings)
70 9fea53cc Vangelis Koukis
71 4161cb41 Christos Stavrakakis
72 4161cb41 Christos Stavrakakis
from datetime import datetime, timedelta
73 4161cb41 Christos Stavrakakis
74 3524241a Christos Stavrakakis
from synnefo.db.models import (VirtualMachine, pooled_rapi_client)
75 3524241a Christos Stavrakakis
from synnefo.logic.rapi import GanetiApiError
76 f5b4f2a3 Christos Stavrakakis
from synnefo.logic.backend import get_ganeti_instances
77 d30f29aa Christos Stavrakakis
from synnefo.logic import utils
78 9fea53cc Vangelis Koukis
79 9fea53cc Vangelis Koukis
80 9e98ba3c Giorgos Verigakis
log = logging.getLogger()
81 9e98ba3c Giorgos Verigakis
82 9e98ba3c Giorgos Verigakis
83 9fea53cc Vangelis Koukis
def stale_servers_in_db(D, G):
84 9fea53cc Vangelis Koukis
    idD = set(D.keys())
85 9fea53cc Vangelis Koukis
    idG = set(G.keys())
86 9fea53cc Vangelis Koukis
87 4161cb41 Christos Stavrakakis
    stale = set()
88 4161cb41 Christos Stavrakakis
    for i in idD - idG:
89 4161cb41 Christos Stavrakakis
        if D[i] == 'BUILD':
90 4161cb41 Christos Stavrakakis
            vm = VirtualMachine.objects.get(id=i)
91 4161cb41 Christos Stavrakakis
            # Check time to avoid many rapi calls
92 4161cb41 Christos Stavrakakis
            if datetime.now() > vm.backendtime + timedelta(seconds=5):
93 4216cd83 Christos Stavrakakis
                with pooled_rapi_client(vm) as c:
94 4216cd83 Christos Stavrakakis
                    try:
95 3524241a Christos Stavrakakis
                        job_status = c.GetJobStatus(vm.backendjobid)['status']
96 3524241a Christos Stavrakakis
                        if job_status in ('queued', 'waiting', 'running'):
97 3524241a Christos Stavrakakis
                            # Server is still building in Ganeti
98 3524241a Christos Stavrakakis
                            continue
99 3524241a Christos Stavrakakis
                        else:
100 3524241a Christos Stavrakakis
                            c.GetInstance(utils.id_to_instance_name(i))
101 3524241a Christos Stavrakakis
                            # Server has just been created in Ganeti
102 3524241a Christos Stavrakakis
                            continue
103 4216cd83 Christos Stavrakakis
                    except GanetiApiError:
104 4216cd83 Christos Stavrakakis
                        stale.add(i)
105 4161cb41 Christos Stavrakakis
        else:
106 4161cb41 Christos Stavrakakis
            stale.add(i)
107 4161cb41 Christos Stavrakakis
108 4161cb41 Christos Stavrakakis
    return stale
109 9fea53cc Vangelis Koukis
110 9fea53cc Vangelis Koukis
111 9fea53cc Vangelis Koukis
def orphan_instances_in_ganeti(D, G):
112 9fea53cc Vangelis Koukis
    idD = set(D.keys())
113 9fea53cc Vangelis Koukis
    idG = set(G.keys())
114 9fea53cc Vangelis Koukis
115 9fea53cc Vangelis Koukis
    return idG - idD
116 9fea53cc Vangelis Koukis
117 9fea53cc Vangelis Koukis
118 9fea53cc Vangelis Koukis
def unsynced_operstate(D, G):
119 9fea53cc Vangelis Koukis
    unsynced = set()
120 9fea53cc Vangelis Koukis
    idD = set(D.keys())
121 9fea53cc Vangelis Koukis
    idG = set(G.keys())
122 9fea53cc Vangelis Koukis
123 9fea53cc Vangelis Koukis
    for i in idD & idG:
124 9fea53cc Vangelis Koukis
        if (G[i] and D[i] != 'STARTED' or
125 9fea53cc Vangelis Koukis
            not G[i] and D[i] not in ('BUILD', 'ERROR', 'STOPPED')):
126 9fea53cc Vangelis Koukis
            unsynced.add((i, D[i], G[i]))
127 4161cb41 Christos Stavrakakis
        if not G[i] and D[i] == 'BUILD':
128 4161cb41 Christos Stavrakakis
            vm = VirtualMachine.objects.get(id=i)
129 4161cb41 Christos Stavrakakis
            # Check time to avoid many rapi calls
130 4161cb41 Christos Stavrakakis
            if datetime.now() > vm.backendtime + timedelta(seconds=5):
131 4216cd83 Christos Stavrakakis
                with pooled_rapi_client(vm) as c:
132 4216cd83 Christos Stavrakakis
                    try:
133 3524241a Christos Stavrakakis
                        job_info = c.GetJobStatus(job_id=vm.backendjobid)
134 3524241a Christos Stavrakakis
                        if job_info['status'] == 'success':
135 3524241a Christos Stavrakakis
                            unsynced.add((i, D[i], G[i]))
136 4216cd83 Christos Stavrakakis
                    except GanetiApiError:
137 4216cd83 Christos Stavrakakis
                        pass
138 9fea53cc Vangelis Koukis
139 9fea53cc Vangelis Koukis
    return unsynced
140 9fea53cc Vangelis Koukis
141 9fea53cc Vangelis Koukis
142 4161cb41 Christos Stavrakakis
def instances_with_build_errors(D, G):
143 4161cb41 Christos Stavrakakis
    failed = set()
144 4161cb41 Christos Stavrakakis
    idD = set(D.keys())
145 4161cb41 Christos Stavrakakis
    idG = set(G.keys())
146 4161cb41 Christos Stavrakakis
147 4161cb41 Christos Stavrakakis
    for i in idD & idG:
148 4161cb41 Christos Stavrakakis
        if not G[i] and D[i] == 'BUILD':
149 4161cb41 Christos Stavrakakis
            vm = VirtualMachine.objects.get(id=i)
150 4161cb41 Christos Stavrakakis
            # Check time to avoid many rapi calls
151 4161cb41 Christos Stavrakakis
            if datetime.now() > vm.backendtime + timedelta(seconds=5):
152 4216cd83 Christos Stavrakakis
                with pooled_rapi_client(vm) as c:
153 4216cd83 Christos Stavrakakis
                    try:
154 3524241a Christos Stavrakakis
                        job_info = c.GetJobStatus(job_id=vm.backendjobid)
155 3524241a Christos Stavrakakis
                        if job_info['status'] == 'error':
156 3524241a Christos Stavrakakis
                            failed.add(i)
157 4216cd83 Christos Stavrakakis
                    except GanetiApiError:
158 4216cd83 Christos Stavrakakis
                        failed.add(i)
159 4161cb41 Christos Stavrakakis
160 4161cb41 Christos Stavrakakis
    return failed
161 4161cb41 Christos Stavrakakis
162 4161cb41 Christos Stavrakakis
163 9fea53cc Vangelis Koukis
def get_servers_from_db():
164 c807398f Kostas Papadimitriou
    vms = VirtualMachine.objects.filter(deleted=False, backend__offline=False)
165 9fea53cc Vangelis Koukis
    return dict(map(lambda x: (x.id, x.operstate), vms))
166 9fea53cc Vangelis Koukis
167 9fea53cc Vangelis Koukis
168 9fea53cc Vangelis Koukis
def get_instances_from_ganeti():
169 f5b4f2a3 Christos Stavrakakis
    ganeti_instances = get_ganeti_instances(bulk=True)
170 9fea53cc Vangelis Koukis
    snf_instances = {}
171 9fea53cc Vangelis Koukis
172 9fea53cc Vangelis Koukis
    prefix = settings.BACKEND_PREFIX_ID
173 9fea53cc Vangelis Koukis
    for i in ganeti_instances:
174 9fea53cc Vangelis Koukis
        if i['name'].startswith(prefix):
175 9fea53cc Vangelis Koukis
            try:
176 d30f29aa Christos Stavrakakis
                id = utils.id_from_instance_name(i['name'])
177 9fea53cc Vangelis Koukis
            except Exception:
178 9e98ba3c Giorgos Verigakis
                log.error("Ignoring instance with malformed name %s",
179 9fea53cc Vangelis Koukis
                              i['name'])
180 9fea53cc Vangelis Koukis
                continue
181 9fea53cc Vangelis Koukis
182 9fea53cc Vangelis Koukis
            if id in snf_instances:
183 9e98ba3c Giorgos Verigakis
                log.error("Ignoring instance with duplicate Synnefo id %s",
184 9fea53cc Vangelis Koukis
                    i['name'])
185 9fea53cc Vangelis Koukis
                continue
186 9fea53cc Vangelis Koukis
187 9fea53cc Vangelis Koukis
            snf_instances[id] = i['oper_state']
188 9fea53cc Vangelis Koukis
189 9fea53cc Vangelis Koukis
    return snf_instances
190 9fea53cc Vangelis Koukis
191 0e9a423f Christos Stavrakakis
#
192 0e9a423f Christos Stavrakakis
# Nics
193 0e9a423f Christos Stavrakakis
#
194 0e9a423f Christos Stavrakakis
def get_nics_from_ganeti():
195 0e9a423f Christos Stavrakakis
    """Get network interfaces for each ganeti instance.
196 0e9a423f Christos Stavrakakis

197 0e9a423f Christos Stavrakakis
    """
198 0e9a423f Christos Stavrakakis
    instances = get_ganeti_instances(bulk=True)
199 0e9a423f Christos Stavrakakis
    prefix = settings.BACKEND_PREFIX_ID
200 0e9a423f Christos Stavrakakis
201 0e9a423f Christos Stavrakakis
    snf_instances_nics = {}
202 0e9a423f Christos Stavrakakis
    for i in instances:
203 0e9a423f Christos Stavrakakis
        if i['name'].startswith(prefix):
204 0e9a423f Christos Stavrakakis
            try:
205 d30f29aa Christos Stavrakakis
                id = utils.id_from_instance_name(i['name'])
206 0e9a423f Christos Stavrakakis
            except Exception:
207 0e9a423f Christos Stavrakakis
                log.error("Ignoring instance with malformed name %s",
208 0e9a423f Christos Stavrakakis
                              i['name'])
209 0e9a423f Christos Stavrakakis
                continue
210 0e9a423f Christos Stavrakakis
            if id in snf_instances_nics:
211 0e9a423f Christos Stavrakakis
                log.error("Ignoring instance with duplicate Synnefo id %s",
212 0e9a423f Christos Stavrakakis
                    i['name'])
213 0e9a423f Christos Stavrakakis
                continue
214 0e9a423f Christos Stavrakakis
215 0e9a423f Christos Stavrakakis
            ips = zip(itertools.repeat('ipv4'), i['nic.ips'])
216 0e9a423f Christos Stavrakakis
            macs = zip(itertools.repeat('mac'), i['nic.macs'])
217 0e9a423f Christos Stavrakakis
            networks = zip(itertools.repeat('network'), i['nic.networks'])
218 0e9a423f Christos Stavrakakis
            # modes = zip(itertools.repeat('mode'), i['nic.modes'])
219 0e9a423f Christos Stavrakakis
            # links = zip(itertools.repeat('link'), i['nic.links'])
220 0e9a423f Christos Stavrakakis
            # nics = zip(ips,macs,modes,networks,links)
221 0e9a423f Christos Stavrakakis
            nics = zip(ips, macs, networks)
222 0e9a423f Christos Stavrakakis
            nics = map(lambda x:dict(x), nics)
223 0e9a423f Christos Stavrakakis
            nics = dict(enumerate(nics))
224 0e9a423f Christos Stavrakakis
            snf_instances_nics[id] = nics
225 0e9a423f Christos Stavrakakis
226 0e9a423f Christos Stavrakakis
    return snf_instances_nics
227 0e9a423f Christos Stavrakakis
228 0e9a423f Christos Stavrakakis
229 0e9a423f Christos Stavrakakis
def get_nics_from_db():
230 0e9a423f Christos Stavrakakis
    """Get network interfaces for each vm in DB.
231 0e9a423f Christos Stavrakakis

232 0e9a423f Christos Stavrakakis
    """
233 0e9a423f Christos Stavrakakis
    instances = VirtualMachine.objects.filter(deleted=False)
234 0e9a423f Christos Stavrakakis
    instances_nics = {}
235 0e9a423f Christos Stavrakakis
    for instance in instances:
236 0e9a423f Christos Stavrakakis
        nics = {}
237 0e9a423f Christos Stavrakakis
        for n in instance.nics.all():
238 0e9a423f Christos Stavrakakis
            ipv4 = n.ipv4
239 0e9a423f Christos Stavrakakis
            nic = {'mac':      n.mac,
240 0e9a423f Christos Stavrakakis
                   'network':  n.network.backend_id,
241 0e9a423f Christos Stavrakakis
                   'ipv4':     ipv4 if ipv4 != '' else None
242 0e9a423f Christos Stavrakakis
                   }
243 0e9a423f Christos Stavrakakis
            nics[n.index] = nic
244 0e9a423f Christos Stavrakakis
        instances_nics[instance.id] = nics
245 0e9a423f Christos Stavrakakis
    return instances_nics
246 0e9a423f Christos Stavrakakis
247 0e9a423f Christos Stavrakakis
248 0e9a423f Christos Stavrakakis
def unsynced_nics(DBNics, GNics):
249 0e9a423f Christos Stavrakakis
    """Find unsynced network interfaces between DB and Ganeti.
250 0e9a423f Christos Stavrakakis

251 0e9a423f Christos Stavrakakis
    @ rtype: dict; {instance_id: ganeti_nics}
252 0e9a423f Christos Stavrakakis
    @ return Dictionary containing the instances ids that have unsynced network
253 0e9a423f Christos Stavrakakis
    interfaces between DB and Ganeti and the network interfaces in Ganeti.
254 0e9a423f Christos Stavrakakis

255 0e9a423f Christos Stavrakakis
    """
256 0e9a423f Christos Stavrakakis
    idD = set(DBNics.keys())
257 0e9a423f Christos Stavrakakis
    idG = set(GNics.keys())
258 0e9a423f Christos Stavrakakis
259 0e9a423f Christos Stavrakakis
    unsynced = {}
260 0e9a423f Christos Stavrakakis
    for i in idD & idG:
261 0e9a423f Christos Stavrakakis
        nicsD = DBNics[i]
262 0e9a423f Christos Stavrakakis
        nicsG = GNics[i]
263 0e9a423f Christos Stavrakakis
        if len(nicsD) != len(nicsG):
264 0e9a423f Christos Stavrakakis
            unsynced[i] = (nicsD, nicsG)
265 0e9a423f Christos Stavrakakis
            continue
266 0e9a423f Christos Stavrakakis
        for index in nicsG.keys():
267 0e9a423f Christos Stavrakakis
            nicD = nicsD[index]
268 0e9a423f Christos Stavrakakis
            nicG = nicsG[index]
269 0e9a423f Christos Stavrakakis
            if nicD['ipv4'] != nicG['ipv4'] or \
270 0e9a423f Christos Stavrakakis
               nicD['mac'] != nicG['mac'] or \
271 0e9a423f Christos Stavrakakis
               nicD['network'] != nicG['network']:
272 3524241a Christos Stavrakakis
                unsynced[i] = (nicsD, nicsG)
273 3524241a Christos Stavrakakis
                break
274 0e9a423f Christos Stavrakakis
275 0e9a423f Christos Stavrakakis
    return unsynced
276 0e9a423f Christos Stavrakakis
277 0e9a423f Christos Stavrakakis
#
278 0e9a423f Christos Stavrakakis
# Networks
279 0e9a423f Christos Stavrakakis
#
280 3524241a Christos Stavrakakis
281 3524241a Christos Stavrakakis
282 0e9a423f Christos Stavrakakis
def get_networks_from_ganeti(backend):
283 44e2c577 Christos Stavrakakis
    prefix = settings.BACKEND_PREFIX_ID + 'net-'
284 0e9a423f Christos Stavrakakis
285 0e9a423f Christos Stavrakakis
    networks = {}
286 3524241a Christos Stavrakakis
    with pooled_rapi_client(backend) as c:
287 3524241a Christos Stavrakakis
        for net in c.GetNetworks(bulk=True):
288 3524241a Christos Stavrakakis
            if net['name'].startswith(prefix):
289 3524241a Christos Stavrakakis
                id = utils.id_from_network_name(net['name'])
290 3524241a Christos Stavrakakis
                networks[id] = net
291 0e9a423f Christos Stavrakakis
292 0e9a423f Christos Stavrakakis
    return networks
293 0e9a423f Christos Stavrakakis
294 0e9a423f Christos Stavrakakis
295 0e9a423f Christos Stavrakakis
def hanging_networks(backend, GNets):
296 0e9a423f Christos Stavrakakis
    """Get networks that are not connected to all Nodegroups.
297 0e9a423f Christos Stavrakakis

298 0e9a423f Christos Stavrakakis
    """
299 0e9a423f Christos Stavrakakis
    def get_network_groups(group_list):
300 0e9a423f Christos Stavrakakis
        groups = set()
301 0e9a423f Christos Stavrakakis
        for g in group_list:
302 0e9a423f Christos Stavrakakis
            g_name = g.split('(')[0]
303 0e9a423f Christos Stavrakakis
            groups.add(g_name)
304 0e9a423f Christos Stavrakakis
        return groups
305 0e9a423f Christos Stavrakakis
306 3524241a Christos Stavrakakis
    with pooled_rapi_client(backend) as c:
307 3524241a Christos Stavrakakis
        groups = set(c.GetGroups())
308 0e9a423f Christos Stavrakakis
309 0e9a423f Christos Stavrakakis
    hanging = {}
310 0e9a423f Christos Stavrakakis
    for id, info in GNets.items():
311 0e9a423f Christos Stavrakakis
        group_list = get_network_groups(info['group_list'])
312 0e9a423f Christos Stavrakakis
        if group_list != groups:
313 0e9a423f Christos Stavrakakis
            hanging[id] = groups - group_list
314 0e9a423f Christos Stavrakakis
    return hanging
315 0e9a423f Christos Stavrakakis
316 9fea53cc Vangelis Koukis
317 a65ee5fc Vangelis Koukis
# Only for testing this module individually
318 9fea53cc Vangelis Koukis
def main():
319 9fea53cc Vangelis Koukis
    print get_instances_from_ganeti()
320 9fea53cc Vangelis Koukis
321 9fea53cc Vangelis Koukis
322 9fea53cc Vangelis Koukis
if __name__ == "__main__":
323 9fea53cc Vangelis Koukis
    sys.exit(main())