Synnefo

import bitarray

                               pooled_rapi_client, Network,

                               BackendNetwork)

from synnefo.db.pools import IPPool

class NetworkReconciler(object):

    def __init__(self, logger, fix=False, conflicting_ips=False):

        self.log = logger

        self.conflicting_ips = conflicting_ips

        self.fix = fix

    @transaction.commit_on_success

    def reconcile_networks(self):

        # Get models from DB

        backends = Backend.objects.exclude(offline=True)

        networks = Network.objects.filter(deleted=False)

        self.event_time = datetime.now()

        # Get info from all ganeti backends

        ganeti_networks = {}

        ganeti_hanging_networks = {}

        for b in backends:

            g_nets = get_networks_from_ganeti(b)

            ganeti_networks[b] = g_nets

            g_hanging_nets = hanging_networks(b, g_nets)

            ganeti_hanging_networks[b] = g_hanging_nets

        # Perform reconciliation for each network

        for network in networks:

            ip_available_maps = []

            ip_reserved_maps = []

            for bend in backends:

                bnet = get_backend_network(network, bend)

                gnet = ganeti_networks[bend].get(network.id)

                if not bnet:

                    if network.floating_ip_pool:

                        # Network is a floating IP pool and does not exist in

                        # backend. We need to create it

                        bnet = self.reconcile_parted_network(network, bend)

                    elif not gnet:

                        # Network does not exist either in Ganeti nor in BD.

                        continue

                    else:

                        # Network exists in Ganeti and not in DB.

                        if network.action != "DESTROY" and not network.public:

                            bnet = self.reconcile_parted_network(network, bend)

                        else:

                            continue

                if not gnet:

                    # Network does not exist in Ganeti. If the network action

                    # is DESTROY, we have to mark as deleted in DB, else we

                    # have to create it in Ganeti.

                    if network.action == "DESTROY":

                        if bnet.operstate != "DELETED":

                            self.reconcile_stale_network(bnet)

                    else:

                        self.reconcile_missing_network(network, bend)

                    # Skip rest reconciliation!

                    continue

                try:

                    hanging_groups = ganeti_hanging_networks[bend][network.id]

                except KeyError:

                    # Network is connected to all nodegroups

                    hanging_groups = []

                if hanging_groups:

                    # CASE-3: Ganeti networks not connected to all nodegroups

                    self.reconcile_hanging_groups(network, bend,

                                                  hanging_groups)

                    continue

                if bnet.operstate != 'ACTIVE':

                    # CASE-4: Unsynced network state. At this point the network

                    # exists and is connected to all nodes so is must be

                    # active!

                    self.reconcile_unsynced_network(network, bend, bnet)

                # Get ganeti IP Pools

                available_map, reserved_map = get_network_pool(gnet)

                ip_available_maps.append(available_map)

                ip_reserved_maps.append(reserved_map)

            if ip_available_maps or ip_reserved_maps:

                # CASE-5: Unsynced IP Pools

                self.reconcile_ip_pools(network, ip_available_maps,

                                        ip_reserved_maps)

            if self.conflicting_ips:

                self.detect_conflicting_ips()

        # CASE-6: Orphan networks

        self.reconcile_orphan_networks(networks, ganeti_networks)

    def reconcile_parted_network(self, network, backend):

        self.log.info("D: Missing DB entry for network %s in backend %s",

                      network, backend)

        if self.fix:

            network.create_backend_network(backend)

            self.log.info("F: Created DB entry")

            bnet = get_backend_network(network, backend)

            return bnet

    def reconcile_stale_network(self, backend_network):

        self.log.info("D: Stale DB entry for network %s in backend %s",

                      backend_network.network, backend_network.backend)

        if self.fix:

            backend_mod.process_network_status(

                backend_network, self.event_time, 0,

                "OP_NETWORK_REMOVE",

                "success",

                "Reconciliation simulated event")

            self.log.info("F: Reconciled event: OP_NETWORK_REMOVE")

    def reconcile_missing_network(self, network, backend):

        self.log.info("D: Missing Ganeti network %s in backend %s",

                      network, backend)

        if self.fix:

            backend_mod.create_network(network, backend)

            self.log.info("F: Issued OP_NETWORK_CONNECT")

    def reconcile_hanging_groups(self, network, backend, hanging_groups):

        self.log.info('D: Network %s in backend %s is not connected to '

                      'the following groups:', network, backend)

        self.log.info('-  ' + '\n-  '.join(hanging_groups))

        if self.fix:

            for group in hanging_groups:

                self.log.info('F: Connecting network %s to nodegroup %s',

                              network, group)

                backend_mod.connect_network(network, backend, depends=[],

                                            group=group)

    def reconcile_unsynced_network(self, network, backend, backend_network):

        self.log.info("D: Unsynced network %s in backend %s", network, backend)

        if self.fix:

            self.log.info("F: Issuing OP_NETWORK_CONNECT")

            backend_mod.process_network_status(

                backend_network, self.event_time, 0,

                "OP_NETWORK_CONNECT",

                "success",

                "Reconciliation simulated eventd")

    def reconcile_ip_pools(self, network, available_maps, reserved_maps):

        available_map = reduce(lambda x, y: x & y, available_maps)

        reserved_map = reduce(lambda x, y: x & y, reserved_maps)

        pool = network.get_pool()

        # Temporary release unused floating IPs

        temp_pool = network.get_pool()

        used_ips = network.nics.values_list("ipv4", flat=True)

        unused_static_ips = network.floating_ips.exclude(ipv4__in=used_ips)

        map(lambda ip: temp_pool.put(ip.ipv4), unused_static_ips)

        if temp_pool.available != available_map:

            self.log.info("D: Unsynced available map of network %s:\n"

                          "\tDB: %r\n\tGB: %r", network,

                          temp_pool.available.to01(),

                          available_map.to01())

            if self.fix:

                pool.available = available_map

                # Release unsued floating IPs, as they are not included in the

                # available map

                map(lambda ip: pool.reserve(ip.ipv4), unused_static_ips)

                pool.save()

        if pool.reserved != reserved_map:

            self.log.info("D: Unsynced reserved map of network %s:\n"

                          "\tDB: %r\n\tGB: %r", network, pool.reserved.to01(),

                          reserved_map.to01())

            if self.fix:

                pool.reserved = reserved_map

                pool.save()

    def detect_conflicting_ips(self, network):

        """Detect NIC's that have the same IP in the same network."""

        machine_ips = network.nics.all().values_list('ipv4', 'machine')

        ips = map(lambda x: x[0], machine_ips)

        distinct_ips = set(ips)

        if len(distinct_ips) < len(ips):

            for i in distinct_ips:

                ips.remove(i)

            for i in ips:

                machines = [utils.id_to_instance_name(x[1])

                            for x in machine_ips if x[0] == i]

                self.log.info('D: Conflicting IP:%s Machines: %s',

                              i, ', '.join(machines))

    def reconcile_orphan_networks(self, db_networks, ganeti_networks):

        # Detect Orphan Networks in Ganeti

        db_network_ids = set([net.id for net in db_networks])

        for back_end, ganeti_networks in ganeti_networks.items():

            ganeti_network_ids = set(ganeti_networks.keys())

            orphans = ganeti_network_ids - db_network_ids

            if len(orphans) > 0:

                self.log.info('D: Orphan Networks in backend %s:',

                              back_end.clustername)

                self.log.info('-  ' + '\n-  '.join([str(o) for o in orphans]))

                if self.fix:

                    for net_id in orphans:

                        self.log.info('Disconnecting and deleting network %d',

                                      net_id)

                        try:

                            network = Network.objects.get(id=net_id)

                            backend_mod.delete_network(network,

                                                       backend=back_end)

                        except Network.DoesNotExist:

                            self.log.info("Not entry for network %s in DB !!",

                                          net_id)

def get_backend_network(network, backend):

    try:

        return BackendNetwork.objects.get(network=network, backend=backend)

    except BackendNetwork.DoesNotExist:

        return None

def get_network_pool(gnet):

    """Return available and reserved IP maps.

    Extract the available and reserved IP map from the info return from Ganeti

    for a network.

    """

    converter = IPPool(Foo(gnet['network']))

    a_map = bitarray_from_map(gnet['map'])

    a_map.invert()

    reserved = gnet['external_reservations']

    r_map = a_map.copy()

    r_map.setall(True)

    if reserved:

        for address in reserved.split(','):

            index = converter.value_to_index(address.strip())

            a_map[index] = True

            r_map[index] = False

    return a_map, r_map

def bitarray_from_map(bitmap):

    return bitarray.bitarray(bitmap.replace("X", "1").replace(".", "0"))

class Foo():

    def __init__(self, subnet):

        self.available_map = ''

        self.reserved_map = ''

        self.size = 0

        self.network = Foo.Foo1(subnet)

    class Foo1():

        def __init__(self, subnet):

            self.subnet = subnet

            self.gateway = None