Synnefo

                               BackendNetwork, BridgePoolTable,

                               MacPrefixPoolTable)

from synnefo.db import pools

    def __init__(self, logger, fix=False):

        self.backends = Backend.objects.exclude(offline=True)

        self.networks = Network.objects.filter(deleted=False)

        self.ganeti_networks = {}

        self.ganeti_hanging_networks = {}

        for b in self.backends:

            self.ganeti_networks[b] = g_nets

            self.ganeti_hanging_networks[b] = g_hanging_nets

        self._reconcile_orphan_networks()

        for network in self.networks:

            self._reconcile_network(network)

    @transaction.commit_on_success

    def _reconcile_network(self, network):

        """Reconcile a network with corresponging Ganeti networks.

        Reconcile a Network and the associated BackendNetworks with the

        corresponding Ganeti networks in all Ganeti backends.

        """

        network_ip_pool = network.get_pool()  # X-Lock on IP Pool

        for bend in self.backends:

            bnet = get_backend_network(network, bend)

            gnet = self.ganeti_networks[bend].get(network.id)

            if not bnet:

                if network.floating_ip_pool:

                    # Network is a floating IP pool and does not exist in

                    # backend. We need to create it

                    bnet = self.reconcile_parted_network(network, bend)

                elif not gnet:

                    # Network does not exist either in Ganeti nor in BD.

                    continue

                else:

                    # Network exists in Ganeti and not in DB.

                    if network.action != "DESTROY" and not network.public:

                        bnet = self.reconcile_parted_network(network, bend)

                    else:

                        continue

            if not gnet:

                # Network does not exist in Ganeti. If the network action

                # is DESTROY, we have to mark as deleted in DB, else we

                # have to create it in Ganeti.

                if network.action == "DESTROY":

                    if bnet.operstate != "DELETED":

                        self.reconcile_stale_network(bnet)

                else:

                    self.reconcile_missing_network(network, bend)

                # Skip rest reconciliation!

                continue

            try:

                hanging_groups = self.ganeti_hanging_networks[bend][network.id]

            except KeyError:

                # Network is connected to all nodegroups

                hanging_groups = []

            if hanging_groups:

                # CASE-3: Ganeti networks not connected to all nodegroups

                self.reconcile_hanging_groups(network, bend,

                                              hanging_groups)

                continue

            if bnet.operstate != 'ACTIVE':

                # CASE-4: Unsynced network state. At this point the network

                # exists and is connected to all nodes so is must be

                # active!

                self.reconcile_unsynced_network(network, bend, bnet)

            # Check that externally reserved IPs of the network in Ganeti are

            # also externally reserved to the IP pool

            externally_reserved = gnet['external_reservations']

            for ip in externally_reserved.split(","):

                ip = ip.strip()

                if not network_ip_pool.is_reserved(ip):

                    msg = ("D: IP '%s' is reserved for network '%s' in"

                           " backend '%s' but not in DB.")

                    self.log.info(msg, ip, network, bend)

                    if self.fix:

                        network_ip_pool.reserve(ip, external=True)

                        network_ip_pool.save()

                        self.log.info("F: Reserved IP '%s'", ip)

    def _reconcile_orphan_networks(self):

        db_networks = self.networks

        ganeti_networks = self.ganeti_networks

class PoolReconciler(object):

    def __init__(self, logger, fix=False):

        self.log = logger

        self.fix = fix

    def reconcile(self):

        self.reconcile_bridges()

        self.reconcile_mac_prefixes()

        for network in Network.objects.filter(deleted=False):

            self.reconcile_ip_pool(network)

    @transaction.commit_on_success

    def reconcile_bridges(self):

        networks = Network.objects.filter(deleted=False,

                                          flavor="PHYSICAL_VLAN")

        check_unique_values(objects=networks, field='link', logger=self.log)

        try:

            pool = BridgePoolTable.get_pool()

        except pools.EmptyPool:

            self.log.info("There is no available pool for bridges.")

            return

        used_bridges = set(networks.values_list('link', flat=True))

        check_pool_consistent(pool=pool, pool_class=pools.BridgePool,

                              used_values=used_bridges, fix=self.fix,

                              logger=self.log)

    @transaction.commit_on_success

    def reconcile_mac_prefixes(self):

        networks = Network.objects.filter(deleted=False, flavor="MAC_FILTERED")

        check_unique_values(objects=networks, field='mac_prefix',

                            logger=self.log)

        try:

            pool = MacPrefixPoolTable.get_pool()

        except pools.EmptyPool:

            self.log.info("There is no available pool for MAC prefixes.")

            return

        used_mac_prefixes = set(networks.values_list('mac_prefix', flat=True))

        check_pool_consistent(pool=pool, pool_class=pools.MacPrefixPool,

                              used_values=used_mac_prefixes, fix=self.fix,

                              logger=self.log)

    @transaction.commit_on_success

    def reconcile_ip_pool(self, network):

        # Check that all NICs have unique IPv4 address

        nics = network.nics.filter(ipv4__isnull=False)

        check_unique_values(objects=nics, field='ipv4', logger=self.log)

        # Check that all Floating IPs have unique IPv4 address

        floating_ips = network.floating_ips.filter(deleted=False)

        check_unique_values(objects=floating_ips, field='ipv4',

                            logger=self.log)

        # First get(lock) the IP pool of the network to prevent new NICs

        # from being created.

        network_ip_pool = network.get_pool()

        used_ips = set(list(nics.values_list("ipv4", flat=True)) +

                       list(floating_ips.values_list("ipv4", flat=True)))

        check_pool_consistent(pool=network_ip_pool,

                              pool_class=pools.IPPool,

                              used_values=used_ips,

                              fix=self.fix, logger=self.log)

def check_unique_values(objects, field, logger):

    used_values = list(objects.values_list(field, flat=True))

    if len(used_values) != len(set(used_values)):

        duplicate_values = [v for v in used_values if used_values.count(v) > 1]

        for value in duplicate_values:

            filter_args = {field: value}

            using_objects = objects.filter(**filter_args)

            msg = "Value '%s' is used as %s for more than one objects: %s"

            logger.error(msg, value, field, ",".join(map(str, using_objects)))

        return False

    logger.debug("Values for field '%s' are unique.", field)

    return True

def check_pool_consistent(pool, pool_class, used_values, fix, logger):

    dummy_pool = create_empty_pool(pool, pool_class)

    [dummy_pool.reserve(value) for value in used_values]

    if dummy_pool.available != pool.available:

        msg = "'%s' is not consistent!\nPool: %s\nUsed: %s"

        pool_diff = dummy_pool.available ^ pool.available

        for index in pool_diff.itersearch(bitarray.bitarray("1")):

            value = pool.index_to_value(int(index))

            msg = "%s is incosistent! Value '%s' is %s but should be %s."

            value1 = pool.is_available(value) and "available" or "unavailable"

            value2 = dummy_pool.is_available(value) and "available"\

                or "unavailable"

            logger.error(msg, pool, value, value1, value2)

        if fix:

            pool.available = dummy_pool.available

            pool.save()

            logger.info("Fixed available map of pool '%s'", pool)

def create_empty_pool(pool, pool_class):

    pool_row = pool.pool_table

    pool_row.available_map = ""

    pool_row.reserved_map = ""

    return pool_class(pool_row)