Synnefo

# Copyright 2011-2012 GRNET S.A. All rights reserved.

import datetime

from django.core.management.base import BaseCommand, CommandError

from synnefo.db.models import (Backend, VirtualMachine, Network,

                               pooled_rapi_client)

from synnefo.logic import reconciliation, utils

from synnefo.logic import backend as backend_mod

from synnefo.util.mac2eui64 import mac2eui64

    output_transaction = True  # The management command runs inside

                               # an SQL transaction

        make_option('--detect-stale', action='store_true', dest='detect_stale',

                    default=False, help='Detect stale VM entries in DB'),

        make_option('--detect-orphans', action='store_true',

                    dest='detect_orphans',

                    default=False, help='Detect orphan instances in Ganeti'),

        make_option('--detect-unsynced', action='store_true',

                    dest='detect_unsynced',

                    default=False, help='Detect unsynced operstate between ' +

                                        'DB and Ganeti'),

        make_option('--detect-build-errors', action='store_true',

                    dest='detect_build_errors', default=False,

                    help='Detect instances with build error'),

        make_option('--detect-unsynced-nics', action='store_true',

                    dest='detect_unsynced_nics', default=False,

                    help='Detect unsynced nics between DB and Ganeti'),

        make_option('--detect-unsynced-flavors', action='store_true',

                    dest='detect_unsynced_flavors', default=False,

                    help='Detect unsynced flavors between DB and Ganeti'),

        make_option('--detect-all', action='store_true',

                    dest='detect_all',

                    default=False, help='Enable all --detect-* arguments'),

        make_option('--fix-build-errors', action='store_true',

                    dest='fix_build_errors', default=False,

                    help='Fix (remove) instances with build errors'),

        make_option('--backend-id', default=None, dest='backend-id',

                    help='Reconcilie VMs only for this backend'),

        keys_detect = [k for k in options.keys() if k.startswith('detect_')]

        if not reduce(lambda x, y: x or y,

                      map(lambda x: options[x], keys_detect)):

            options['detect_all'] = True

        if options['detect_all']:

            for kd in keys_detect:

                options[kd] = True

        for kf in keys_fix:

            kd = kf.replace('fix_', 'detect_', 1)

            if (options[kf] and not options[kd]):

                raise CommandError("Cannot use --%s without corresponding "

                                   "--%s argument" % (kf, kd))

        verbosity = int(options['verbosity'])

        self._process_args(options)

            backends = Backend.objects.filter(offline=False)

        with_nics = options["detect_unsynced_nics"]

        DBVMs = reconciliation.get_servers_from_db(backends, with_nics)

        GanetiVMs = reconciliation.get_instances_from_ganeti(backends)

        #

        # Detect problems

        #

        if options['detect_stale']:

            stale = reconciliation.stale_servers_in_db(DBVMs, GanetiVMs)

            if len(stale) > 0:

                print >> sys.stderr, "Found the following stale server IDs: "

                print "    " + "\n    ".join(

                    [str(x) for x in stale])

            elif verbosity == 2:

                print >> sys.stderr, "Found no stale server IDs in DB."

        if options['detect_orphans']:

            orphans = reconciliation.orphan_instances_in_ganeti(DBVMs,

                                                                GanetiVMs)

            if len(orphans) > 0:

                print >> sys.stderr, "Found orphan Ganeti instances with IDs: "

                print "    " + "\n    ".join(

                    [str(x) for x in orphans])

            elif verbosity == 2:

                print >> sys.stderr, "Found no orphan Ganeti instances."

        if options['detect_unsynced']:

            unsynced = reconciliation.unsynced_operstate(DBVMs, GanetiVMs)

            if len(unsynced) > 0:

                print >> sys.stderr, "The operstate of the following server" \

                                     " IDs is out-of-sync:"

                print "    " + "\n    ".join(

                    ["%d is %s in DB, %s in Ganeti" %

                     (x[0], x[1], ('UP' if x[2] else 'DOWN'))

                     for x in unsynced])

            elif verbosity == 2:

                print >> sys.stderr, "The operstate of all servers is in sync."

        if options['detect_build_errors']:

            build_errors = reconciliation.\

                instances_with_build_errors(DBVMs, GanetiVMs)

            if len(build_errors) > 0:

                msg = "The os for the following server IDs was not build"\

                      " successfully:"

                print >> sys.stderr, msg

                print "    " + "\n    ".join(

                    ["%d" % x for x in build_errors])

            elif verbosity == 2:

                print >> sys.stderr, "Found no instances with build errors."

        if options['detect_unsynced_nics']:

            def pretty_print_nics(nics):

                if not nics:

                    print ''.ljust(18) + 'None'

                for index, info in nics.items():

                    print ''.ljust(18) + 'nic/' + str(index) +\

                          ': MAC: %s, IP: %s, Network: %s' % \

                          (info['mac'], info['ipv4'], info['network'])

            unsynced_nics = reconciliation.unsynced_nics(DBVMs, GanetiVMs)

            if len(unsynced_nics) > 0:

                msg = "The NICs of the servers with the following IDs are"\

                      " unsynced:"

                print >> sys.stderr, msg

                for id, nics in unsynced_nics.items():

                    print ''.ljust(2) + '%6d:' % id

                    print ''.ljust(8) + '%8s:' % 'DB'

                    pretty_print_nics(nics[0])

                    print ''.ljust(8) + '%8s:' % 'Ganeti'

                    pretty_print_nics(nics[1])

            elif verbosity == 2:

                print >> sys.stderr, "All instance nics are synced."

        if options["detect_unsynced_flavors"]:

            unsynced_flavors = reconciliation.unsynced_flavors(DBVMs,

                                                               GanetiVMs)

            if len(unsynced_flavors) > 0:

                print >> sys.stderr, "The flavor of the following server" \

                                     " IDs is out-of-sync:"

                print "    " + "\n    ".join(

                    ["%d is %s in DB, %s in Ganeti" %

                     (x[0], x[1], x[2])

                     for x in unsynced_flavors])

            elif verbosity == 2:

                print >> sys.stderr, "All instance flavors are synced."

        #

        # Then fix them

        #

        if options['fix_stale'] and len(stale) > 0:

            print >> sys.stderr, \

                "Simulating successful Ganeti removal for %d " \

                "servers in the DB:" % len(stale)

            for vm in VirtualMachine.objects.filter(pk__in=stale):

                event_time = datetime.datetime.now()

                backend_mod.process_op_status(

                    vm=vm,

                    etime=event_time,

                    jobid=-0,

                    opcode='OP_INSTANCE_REMOVE', status='success',

                    logmsg='Reconciliation: simulated Ganeti event')

            print >> sys.stderr, "    ...done"

        if options['fix_orphans'] and len(orphans) > 0:

            print >> sys.stderr, \

                "Issuing OP_INSTANCE_REMOVE for %d Ganeti instances:" % \

                len(orphans)

            for id in orphans:

                try:

                    vm = VirtualMachine.objects.get(pk=id)

                    with pooled_rapi_client(vm) as client:

                        client.DeleteInstance(utils.id_to_instance_name(id))

                except VirtualMachine.DoesNotExist:

                    print >> sys.stderr, "No entry for VM %d in DB !!" % id

            print >> sys.stderr, "    ...done"

        if options['fix_unsynced'] and len(unsynced) > 0:

            print >> sys.stderr, "Setting the state of %d out-of-sync VMs:" % \

                len(unsynced)

            for id, db_state, ganeti_up in unsynced:

                vm = VirtualMachine.objects.get(pk=id)

                opcode = "OP_INSTANCE_REBOOT" if ganeti_up \

                         else "OP_INSTANCE_SHUTDOWN"

                event_time = datetime.datetime.now()

                backend_mod.process_op_status(

                    vm=vm, etime=event_time, jobid=-0,

                    opcode=opcode, status='success',

                    logmsg='Reconciliation: simulated Ganeti event')

            print >> sys.stderr, "    ...done"

        if options['fix_build_errors'] and len(build_errors) > 0:

            print >> sys.stderr, "Setting the state of %d build-errors VMs:" %\

                                 len(build_errors)

            for id in build_errors:

                vm = VirtualMachine.objects.get(pk=id)

                event_time = datetime.datetime.now()

                backend_mod.process_op_status(

                    vm=vm, etime=event_time, jobid=-0,

                    opcode="OP_INSTANCE_CREATE", status='error',

                    logmsg='Reconciliation: simulated Ganeti event')

            print >> sys.stderr, "    ...done"

        if options['fix_unsynced_nics'] and len(unsynced_nics) > 0:

            print >> sys.stderr, "Setting the nics of %d out-of-sync VMs:" % \

                                 len(unsynced_nics)

            for id, nics in unsynced_nics.items():

                vm = VirtualMachine.objects.get(pk=id)

                nics = nics[1]  # Ganeti nics

                if nics == {}:  # No nics

                    vm.nics.all.delete()

                    continue

                for index, nic in nics.items():

                    net_id = utils.id_from_network_name(nic['network'])

                    subnet6 = Network.objects.get(id=net_id).subnet6

                    # Produce ipv6

                    ipv6 = subnet6 and mac2eui64(nic['mac'], subnet6) or None

                    nic['ipv6'] = ipv6

                    # Rename ipv4 to ip

                    nic['ip'] = nic['ipv4']

                # Dict to sorted list

                final_nics = []

                nics_keys = nics.keys()

                nics_keys.sort()

                for i in nics_keys:

                    if nics[i]['network']:

                        final_nics.append(nics[i])

                    else:

                        print 'Network of nic %d of vm %s is None. ' \

                              'Can not reconcile' % (i, vm.backend_vm_id)

                event_time = datetime.datetime.now()

                backend_mod.process_net_status(vm=vm, etime=event_time,

                                               nics=final_nics)

            print >> sys.stderr, "    ...done"

        if options["fix_unsynced_flavors"] and len(unsynced_flavors) > 0:

            print >> sys.stderr, "Setting the flavor of %d unsynced VMs:" % \

                len(unsynced_flavors)

            for id, db_flavor, gnt_flavor in unsynced_flavors:

                vm = VirtualMachine.objects.get(pk=id)

                old_state = vm.operstate

                opcode = "OP_INSTANCE_SET_PARAMS"

                beparams = {"vcpus": gnt_flavor.cpu,

                            "minmem": gnt_flavor.ram,

                            "maxmem": gnt_flavor.ram}

                event_time = datetime.datetime.now()

                backend_mod.process_op_status(

                    vm=vm, etime=event_time, jobid=-0,

                    opcode=opcode, status='success',

                    beparams=beparams,

                    logmsg='Reconciliation: simulated Ganeti event')

                # process_op_status with beparams will set the vmstate to

                # shutdown. Fix this be returning it to old state

                vm = VirtualMachine.objects.get(pk=id)

                vm.operstate = old_state

                vm.save()

            print >> sys.stderr, "    ...done"

#!/usr/bin/env python

# Copyright 2011 GRNET S.A. All rights reserved.

import logging

import sys

import itertools

from collections import namedtuple

from synnefo.db.models import (VirtualMachine, NetworkInterface, Flavor,

from synnefo.logic.backend import get_instances

from synnefo.logic import utils

log = logging.getLogger()

def needs_reconciliation(vm):

    now = datetime.now()

    return (now > vm.updated + timedelta(seconds=CHECK_INTERVAL)) or\

           (now > vm.backendtime + timedelta(seconds=2*CHECK_INTERVAL))

VMState = namedtuple("VMState", ["state", "cpu", "ram", "nics"])

def stale_servers_in_db(D, G):

    idD = set(D.keys())

    idG = set(G.keys())

    stale = set()

    for i in idD - idG:

        if D[i] == 'BUILD':

            vm = VirtualMachine.objects.get(id=i)

            if needs_reconciliation(vm):

                with pooled_rapi_client(vm) as c:

                    try:

                        job_status = c.GetJobStatus(vm.backendjobid)['status']

                        if job_status in ('queued', 'waiting', 'running'):

                            # Server is still building in Ganeti

                            continue

                        else:

                            c.GetInstance(utils.id_to_instance_name(i))

                            # Server has just been created in Ganeti

                            continue

                    except GanetiApiError:

                        stale.add(i)

            stale.add(i)

    return stale

def orphan_instances_in_ganeti(D, G):

    idD = set(D.keys())

    idG = set(G.keys())

    return idG - idD

def unsynced_operstate(D, G):

    unsynced = set()

    idD = set(D.keys())

    idG = set(G.keys())

    for i in idD & idG:

        dbstate = D[i].state

        gntstate = G[i].state

        vm_unsynced = (gntstate and dbstate != "STARTED") or\

            (not gntstate and dbstate not in ('BUILD', 'ERROR', 'STOPPED'))

        if vm_unsynced:

            unsynced.add((i, dbstate, gntstate))

        if not gntstate and dbstate == 'BUILD':

            vm = VirtualMachine.objects.get(id=i)

            if needs_reconciliation(vm):

                with pooled_rapi_client(vm) as c:

                    try:

                        job_info = c.GetJobStatus(job_id=vm.backendjobid)

                        if job_info['status'] == 'success':

                            unsynced.add((i, dbstate, gntstate))

                    except GanetiApiError:

                        pass

    return unsynced

def unsynced_flavors(D, G):

    unsynced = set()

    idD = set(D.keys())

    idG = set(G.keys())

    for i in idD & idG:

        if D[i].ram != G[i].ram or D[i].cpu != G[i].cpu:

            db_flavor = VirtualMachine.objects.get(id=i).flavor

                                    ram=G[i].ram, cpu=G[i].cpu,

                                    disk=db_flavor.disk,

                                    disk_template=db_flavor.disk_template)

                gnt_flavor = None

            unsynced.add((i, db_flavor, gnt_flavor))

    return unsynced

def instances_with_build_errors(D, G):

    failed = set()

    idD = set(D.keys())

    idG = set(G.keys())

    for i in idD & idG:

        if not G[i] and D[i] == 'BUILD':

            vm = VirtualMachine.objects.get(id=i)

            if not vm.backendjobid:  # VM has not been enqueued in the backend

                if datetime.now() > vm.created + timedelta(seconds=120):

                    # If a job has not been enqueued after 2 minutues, then

                    # it must be a stale entry..

                    failed.add(i)

            elif needs_reconciliation(vm):

                # Check time to avoid many rapi calls

                with pooled_rapi_client(vm) as c:

                    try:

                        job_info = c.GetJobStatus(job_id=vm.backendjobid)

                        if job_info['status'] == 'error':

                            failed.add(i)

                    except GanetiApiError:

                        failed.add(i)

    return failed

def get_servers_from_db(backends, with_nics=True):

    vms = VirtualMachine.objects.filter(deleted=False, backend__in=backends)

    vm_info = vms.values_list("id", "operstate", "flavor__cpu", "flavor__ram")

    if with_nics:

        nics = NetworkInterface.objects.filter(machine__in=vms)\

                               .order_by("machine")\

                               .values_list("machine", "index", "mac", "ipv4",

                                            "network")

        vm_nics = {}

        for machine, machine_nics in itertools.groupby(nics,

                                                       lambda nic: nic[0]):

            vm_nics[machine] = {}

            for machine, index, mac, ipv4, network in machine_nics:

                nic = {'mac':      mac,

                       'network':  utils.id_to_network_name(network),

                       'ipv4':     ipv4 if ipv4 != '' else None

                       }

                vm_nics[machine][index] = nic

    servers = dict([(vm_id, VMState(state=state,

                                    cpu=cpu,

                                    ram=ram,

                                    nics=vm_nics.get(vm_id, [])))

                    for vm_id, state, cpu, ram in vm_info])

    return servers

def get_instances_from_ganeti(backends):

    instances = []

    for backend in backends:

        instances.append(get_instances(backend))

    ganeti_instances = reduce(list.__add__, instances, [])

    snf_instances = {}

    prefix = settings.BACKEND_PREFIX_ID

    for i in ganeti_instances:

        if i['name'].startswith(prefix):

            try:

                id = utils.id_from_instance_name(i['name'])

            except Exception:

                log.error("Ignoring instance with malformed name %s",

                          i['name'])

                continue

            if id in snf_instances:

                log.error("Ignoring instance with duplicate Synnefo id %s",

                          i['name'])

                continue

            nics = get_nics_from_instance(i)

            beparams = i["beparams"]

            vcpus = beparams["vcpus"]

            ram = beparams["maxmem"]

            snf_instances[id] = VMState(state=i["oper_state"],

                                        cpu=vcpus,

                                        ram=ram,

                                        nics=nics)

    return snf_instances

# Nics

#

def get_nics_from_ganeti(backends):

    """Get network interfaces for each ganeti instance.

    """

    instances = []

    for backend in backends:

        instances.append(get_instances(backend))

    instances = reduce(list.__add__, instances, [])

    prefix = settings.BACKEND_PREFIX_ID

    snf_instances_nics = {}

    for i in instances:

        if i['name'].startswith(prefix):

            try:

                id = utils.id_from_instance_name(i['name'])

            except Exception:

                log.error("Ignoring instance with malformed name %s",

                          i['name'])

                continue

            if id in snf_instances_nics:

                log.error("Ignoring instance with duplicate Synnefo id %s",

                          i['name'])

                continue

            snf_instances_nics[id] = get_nics_from_instance(i)

    return snf_instances_nics

def get_nics_from_instance(i):

    ips = zip(itertools.repeat('ipv4'), i['nic.ips'])

    macs = zip(itertools.repeat('mac'), i['nic.macs'])

    networks = zip(itertools.repeat('network'), i['nic.networks'])

    # modes = zip(itertools.repeat('mode'), i['nic.modes'])

    # links = zip(itertools.repeat('link'), i['nic.links'])

    # nics = zip(ips,macs,modes,networks,links)

    nics = zip(ips, macs, networks)

    nics = map(lambda x: dict(x), nics)

    nics = dict(enumerate(nics))

    return nics

def unsynced_nics(DBVMs, GanetiVMs):

    """Find unsynced network interfaces between DB and Ganeti.

    @ rtype: dict; {instance_id: ganeti_nics}

    @ return Dictionary containing the instances ids that have unsynced network

    interfaces between DB and Ganeti and the network interfaces in Ganeti.

    """

    idD = set(DBVMs.keys())

    idG = set(GanetiVMs.keys())

    unsynced = {}

    for i in idD & idG:

        nicsD = DBVMs[i].nics

        nicsG = GanetiVMs[i].nics

        if len(nicsD) != len(nicsG):

            unsynced[i] = (nicsD, nicsG)

            continue

        for index in nicsG.keys():

            nicD = nicsD[index]

            nicG = nicsG[index]

            diff = (nicD['ipv4'] != nicG['ipv4'] or

                    nicD['mac'] != nicG['mac'] or

                    nicD['network'] != nicG['network'])

            if diff:

                    unsynced[i] = (nicsD, nicsG)

                    break

    return unsynced

#

# Only for testing this module individually

def main():

    print get_instances_from_ganeti()

if __name__ == "__main__":

    sys.exit(main())