Synnefo

# Copyright 2011-2012 GRNET S.A. All rights reserved.

#

# Redistribution and use in source and binary forms, with or

# without modification, are permitted provided that the following

# conditions are met:

#

#   1. Redistributions of source code must retain the above

#      copyright notice, this list of conditions and the following

#      disclaimer.

#

#   2. Redistributions in binary form must reproduce the above

#      copyright notice, this list of conditions and the following

#      disclaimer in the documentation and/or other materials

#      provided with the distribution.

#

# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS

# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED

# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR

# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR

# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,

# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT

# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF

# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED

# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT

# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN

# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE

# POSSIBILITY OF SUCH DAMAGE.

#

# The views and conclusions contained in the software and

# documentation are those of the authors and should not be

# interpreted as representing official policies, either expressed

# or implied, of GRNET S.A.

from base64 import b64decode

from django import dispatch

from django.conf import settings

from django.conf.urls.defaults import patterns

from django.db import transaction

from django.http import HttpResponse

from django.template.loader import render_to_string

from django.utils import simplejson as json

from synnefo.api import faults, util

from synnefo.api.actions import server_actions

from synnefo.api.common import method_not_allowed

from synnefo.db.models import (VirtualMachine, VirtualMachineMetadata,

                               NetworkInterface)

from synnefo.logic.backend import create_instance, delete_instance

from synnefo.logic.utils import get_rsapi_state

from synnefo.logic.rapi import GanetiApiError

from synnefo.logic.backend_allocator import BackendAllocator

from synnefo import quotas

# server creation signal

server_created = dispatch.Signal(providing_args=["created_vm_params"])

from logging import getLogger

log = getLogger('synnefo.api')

urlpatterns = patterns(

    'synnefo.api.servers',

    (r'^(?:/|.json|.xml)?$', 'demux'),

    (r'^/detail(?:.json|.xml)?$', 'list_servers', {'detail': True}),

    (r'^/(\d+)(?:.json|.xml)?$', 'server_demux'),

    (r'^/(\d+)/action(?:.json|.xml)?$', 'server_action'),

    (r'^/(\d+)/ips(?:.json|.xml)?$', 'list_addresses'),

    (r'^/(\d+)/ips/(.+?)(?:.json|.xml)?$', 'list_addresses_by_network'),

    (r'^/(\d+)/meta(?:.json|.xml)?$', 'metadata_demux'),

    (r'^/(\d+)/meta/(.+?)(?:.json|.xml)?$', 'metadata_item_demux'),

    (r'^/(\d+)/stats(?:.json|.xml)?$', 'server_stats'),

    (r'^/(\d+)/diagnostics(?:.json)?$', 'get_server_diagnostics'),

)

def demux(request):

    if request.method == 'GET':

        return list_servers(request)

    elif request.method == 'POST':

        return create_server(request)

    else:

        return method_not_allowed(request)

def server_demux(request, server_id):

    if request.method == 'GET':

        return get_server_details(request, server_id)

    elif request.method == 'PUT':

        return update_server_name(request, server_id)

    elif request.method == 'DELETE':

        return delete_server(request, server_id)

    else:

        return method_not_allowed(request)

def metadata_demux(request, server_id):

    if request.method == 'GET':

        return list_metadata(request, server_id)

    elif request.method == 'POST':

        return update_metadata(request, server_id)

    else:

        return method_not_allowed(request)

def metadata_item_demux(request, server_id, key):

    if request.method == 'GET':

        return get_metadata_item(request, server_id, key)

    elif request.method == 'PUT':

        return create_metadata_item(request, server_id, key)

    elif request.method == 'DELETE':

        return delete_metadata_item(request, server_id, key)

    else:

        return method_not_allowed(request)

def nic_to_dict(nic):

    d = {'id': util.construct_nic_id(nic),

         'network_id': str(nic.network.id),

         'mac_address': nic.mac,

         'ipv4': nic.ipv4 if nic.ipv4 else None,

         'ipv6': nic.ipv6 if nic.ipv6 else None}

    if nic.firewall_profile:

        d['firewallProfile'] = nic.firewall_profile

    return d

def vm_to_dict(vm, detail=False):

    d = dict(id=vm.id, name=vm.name)

    if detail:

        d['status'] = get_rsapi_state(vm)

        d['progress'] = 100 if get_rsapi_state(vm) == 'ACTIVE' \

            else vm.buildpercentage

        d['hostId'] = vm.hostid

        d['updated'] = util.isoformat(vm.updated)

        d['created'] = util.isoformat(vm.created)

        d['flavorRef'] = vm.flavor.id

        d['imageRef'] = vm.imageid

        d['suspended'] = vm.suspended

        metadata = dict((m.meta_key, m.meta_value) for m in vm.metadata.all())

        if metadata:

            d['metadata'] = {'values': metadata}

        vm_nics = vm.nics.filter(state="ACTIVE").order_by("index")

        attachments = map(nic_to_dict, vm_nics)

        if attachments:

            d['attachments'] = {'values': attachments}

        # include the latest vm diagnostic, if set

        diagnostic = vm.get_last_diagnostic()

        if diagnostic:

            d['diagnostics'] = diagnostics_to_dict([diagnostic])

    return d

def diagnostics_to_dict(diagnostics):

    """

    Extract api data from diagnostics QuerySet.

    """

    entries = list()

    for diagnostic in diagnostics:

        # format source date if set

        formatted_source_date = None

        if diagnostic.source_date:

            formatted_source_date = util.isoformat(diagnostic.source_date)

        entry = {

            'source': diagnostic.source,

            'created': util.isoformat(diagnostic.created),

            'message': diagnostic.message,

            'details': diagnostic.details,

            'level': diagnostic.level,

        }

        if formatted_source_date:

            entry['source_date'] = formatted_source_date

        entries.append(entry)

    return entries

def render_server(request, server, status=200):

    if request.serialization == 'xml':

        data = render_to_string('server.xml', {

            'server': server,

            'is_root': True})

    else:

        data = json.dumps({'server': server})

    return HttpResponse(data, status=status)

def render_diagnostics(request, diagnostics_dict, status=200):

    """

    Render diagnostics dictionary to json response.

    """

    return HttpResponse(json.dumps(diagnostics_dict), status=status)

@util.api_method('GET')

def get_server_diagnostics(request, server_id):

    """

    Virtual machine diagnostics api view.

    """

    log.debug('server_diagnostics %s', server_id)

    vm = util.get_vm(server_id, request.user_uniq)

    diagnostics = diagnostics_to_dict(vm.diagnostics.all())

    return render_diagnostics(request, diagnostics)

@util.api_method('GET')

def list_servers(request, detail=False):

    # Normal Response Codes: 200, 203

    # Error Response Codes: computeFault (400, 500),

    #                       serviceUnavailable (503),

    #                       unauthorized (401),

    #                       badRequest (400),

    #                       overLimit (413)

    log.debug('list_servers detail=%s', detail)

    user_vms = VirtualMachine.objects.filter(userid=request.user_uniq)

    since = util.isoparse(request.GET.get('changes-since'))

    if since:

        user_vms = user_vms.filter(updated__gte=since)

        if not user_vms:

            return HttpResponse(status=304)

    else:

        user_vms = user_vms.filter(deleted=False)

    servers = [vm_to_dict(server, detail)

               for server in user_vms.order_by('id')]

    if request.serialization == 'xml':

        data = render_to_string('list_servers.xml', {

            'servers': servers,

            'detail': detail})

    else:

        data = json.dumps({'servers': {'values': servers}})

    return HttpResponse(data, status=200)

@util.api_method('POST')

# Use manual transactions. Backend and IP pool allocations need exclusive

# access (SELECT..FOR UPDATE). Running create_server with commit_on_success

# would result in backends and public networks to be locked until the job is

# sent to the Ganeti backend.

@quotas.uses_commission

@transaction.commit_manually

def create_server(serials, request):

    # Normal Response Code: 202

    # Error Response Codes: computeFault (400, 500),

    #                       serviceUnavailable (503),

    #                       unauthorized (401),

    #                       badMediaType(415),

    #                       itemNotFound (404),

    #                       badRequest (400),

    #                       serverCapacityUnavailable (503),

    #                       overLimit (413)

    try:

        req = util.get_request_dict(request)

        log.info('create_server %s', req)

        user_id = request.user_uniq

        try:

            server = req['server']

            name = server['name']

            metadata = server.get('metadata', {})

            assert isinstance(metadata, dict)

            image_id = server['imageRef']

            flavor_id = server['flavorRef']

            personality = server.get('personality', [])

            assert isinstance(personality, list)

        except (KeyError, AssertionError):

            raise faults.BadRequest("Malformed request")

        # Verify that personalities are well-formed

        util.verify_personality(personality)

        # Get image information

        image = util.get_image_dict(image_id, user_id)

        # Get flavor (ensure it is active)

        flavor = util.get_flavor(flavor_id, include_deleted=False)

        # Allocate VM to backend

        backend_allocator = BackendAllocator()

        backend = backend_allocator.allocate(request.user_uniq, flavor)

        if backend is None:

            log.error("No available backends for VM with flavor %s", flavor)

            raise faults.ServiceUnavailable("No available backends")

    except:

        transaction.rollback()

        raise

    else:

        transaction.commit()

    # Allocate IP from public network

    try:

        (network, address) = util.get_public_ip(backend)

        nic = {'ip': address, 'network': network.backend_id}

    except:

        transaction.rollback()

        raise

    else:

        transaction.commit()

    # Fix flavor for archipelago

    password = util.random_password()

    disk_template, provider = util.get_flavor_provider(flavor)

    if provider:

        flavor.disk_template = disk_template

        flavor.disk_provider = provider

        flavor.disk_origin = None

        if provider == 'vlmc':

            flavor.disk_origin = image['checksum']

            image['backend_id'] = 'null'

    else:

        flavor.disk_provider = None

    try:

        # Issue commission

        serial = quotas.issue_vm_commission(user_id, flavor)

        serials.append(serial)

        # Make the commission accepted, since in the end of this

        # transaction the VM will have been created in the DB.

        serial.accepted = True

        serial.save()

        # We must save the VM instance now, so that it gets a valid

        # vm.backend_vm_id.

        vm = VirtualMachine.objects.create(

            name=name,

            backend=backend,

            userid=user_id,

            imageid=image_id,

            flavor=flavor,

            action="CREATE",

            serial=serial)

        # Create VM's public NIC. Do not wait notification form ganeti hooks to

        # create this NIC, because if the hooks never run (e.g. building error)

        # the VM's public IP address will never be released!

        NetworkInterface.objects.create(machine=vm, network=network, index=0,

                                        ipv4=address, state="Buidling")

        log.info("Created entry in DB for VM '%s'", vm)

        # dispatch server created signal

        server_created.send(sender=vm, created_vm_params={

            'img_id': image['backend_id'],

            'img_passwd': password,

            'img_format': str(image['format']),

            'img_personality': json.dumps(personality),

            'img_properties': json.dumps(image['metadata']),

        })

        # Also we must create the VM metadata in the same transaction.

        for key, val in metadata.items():

            VirtualMachineMetadata.objects.create(

                meta_key=key,

                meta_value=val,

                vm=vm)

    except:

        transaction.rollback()

        raise

    else:

        transaction.commit()

    try:

        jobID = create_instance(vm, nic, flavor, image)

        # At this point the job is enqueued in the Ganeti backend

        vm.backendjobid = jobID

        vm.save()

        transaction.commit()

        log.info("User %s created VM %s, NIC %s, Backend %s, JobID %s",

                 user_id, vm, nic, backend, str(jobID))

    except GanetiApiError as e:

        log.exception("Can not communicate to backend %s: %s. Deleting VM %s",

                      backend, e, vm)

        vm.delete()

        transaction.commit()

        raise

    except:

        transaction.rollback()

        raise

    server = vm_to_dict(vm, detail=True)

    server['status'] = 'BUILD'

    server['adminPass'] = password

    response = render_server(request, server, status=202)

    return response

@util.api_method('GET')

def get_server_details(request, server_id):

    # Normal Response Codes: 200, 203

    # Error Response Codes: computeFault (400, 500),

    #                       serviceUnavailable (503),

    #                       unauthorized (401),

    #                       badRequest (400),

    #                       itemNotFound (404),

    #                       overLimit (413)

    log.debug('get_server_details %s', server_id)

    vm = util.get_vm(server_id, request.user_uniq)

    server = vm_to_dict(vm, detail=True)

    return render_server(request, server)

@util.api_method('PUT')

def update_server_name(request, server_id):

    # Normal Response Code: 204

    # Error Response Codes: computeFault (400, 500),

    #                       serviceUnavailable (503),

    #                       unauthorized (401),

    #                       badRequest (400),

    #                       badMediaType(415),

    #                       itemNotFound (404),

    #                       buildInProgress (409),

    #                       overLimit (413)

    req = util.get_request_dict(request)

    log.info('update_server_name %s %s', server_id, req)

    try:

        name = req['server']['name']

    except (TypeError, KeyError):

        raise faults.BadRequest("Malformed request")

    vm = util.get_vm(server_id, request.user_uniq, for_update=True,

                     non_suspended=True)

    vm.name = name

    vm.save()

    return HttpResponse(status=204)

@util.api_method('DELETE')

@transaction.commit_on_success

def delete_server(request, server_id):

    # Normal Response Codes: 204

    # Error Response Codes: computeFault (400, 500),

    #                       serviceUnavailable (503),

    #                       unauthorized (401),

    #                       itemNotFound (404),

    #                       unauthorized (401),

    #                       buildInProgress (409),

    #                       overLimit (413)

    log.info('delete_server %s', server_id)

    vm = util.get_vm(server_id, request.user_uniq, for_update=True,

                     non_suspended=True)

    start_action(vm, 'DESTROY')

    delete_instance(vm)

    return HttpResponse(status=204)

# additional server actions

ARBITRARY_ACTIONS = ['console', 'firewallProfile']

@util.api_method('POST')

def server_action(request, server_id):

    req = util.get_request_dict(request)

    log.debug('server_action %s %s', server_id, req)

    if len(req) != 1:

        raise faults.BadRequest("Malformed request")

    # Do not allow any action on deleted or suspended VMs

    vm = util.get_vm(server_id, request.user_uniq, for_update=True,

                     non_deleted=True, non_suspended=True)

    try:

        key = req.keys()[0]

        if key not in ARBITRARY_ACTIONS:

            start_action(vm, key_to_action(key))

        val = req[key]

        assert isinstance(val, dict)

        return server_actions[key](request, vm, val)

    except KeyError:

        raise faults.BadRequest("Unknown action")

    except AssertionError:

        raise faults.BadRequest("Invalid argument")

def key_to_action(key):

    """Map HTTP request key to a VM Action"""

    if key == "shutdown":

        return "STOP"

    if key == "delete":

        return "DESTROY"

    if key in ARBITRARY_ACTIONS:

        return None

    else:

        return key.upper()

def start_action(vm, action):

    log.debug("Applying action %s to VM %s", action, vm)

    if not action:

        return

    if not action in [x[0] for x in VirtualMachine.ACTIONS]:

        raise faults.ServiceUnavailable("Action %s not supported" % action)

    # No actions to deleted VMs

    if vm.deleted:

        raise VirtualMachine.DeletedError

    # No actions to machines being built. They may be destroyed, however.

    if vm.operstate == 'BUILD' and action != 'DESTROY':

        raise VirtualMachine.BuildingError

    vm.action = action

    vm.backendjobid = None

    vm.backendopcode = None

    vm.backendjobstatus = None

    vm.backendlogmsg = None

    vm.save()

@util.api_method('GET')

def list_addresses(request, server_id):

    # Normal Response Codes: 200, 203

    # Error Response Codes: computeFault (400, 500),

    #                       serviceUnavailable (503),

    #                       unauthorized (401),

    #                       badRequest (400),

    #                       overLimit (413)

    log.debug('list_addresses %s', server_id)

    vm = util.get_vm(server_id, request.user_uniq)

    addresses = [nic_to_dict(nic) for nic in vm.nics.all()]

    if request.serialization == 'xml':

        data = render_to_string('list_addresses.xml', {'addresses': addresses})

    else:

        data = json.dumps({'addresses': {'values': addresses}})

    return HttpResponse(data, status=200)

@util.api_method('GET')

def list_addresses_by_network(request, server_id, network_id):

    # Normal Response Codes: 200, 203

    # Error Response Codes: computeFault (400, 500),

    #                       serviceUnavailable (503),

    #                       unauthorized (401),

    #                       badRequest (400),

    #                       itemNotFound (404),

    #                       overLimit (413)

    log.debug('list_addresses_by_network %s %s', server_id, network_id)

    machine = util.get_vm(server_id, request.user_uniq)

    network = util.get_network(network_id, request.user_uniq)

    nic = util.get_nic(machine, network)

    address = nic_to_dict(nic)

    if request.serialization == 'xml':

        data = render_to_string('address.xml', {'address': address})

    else:

        data = json.dumps({'network': address})

    return HttpResponse(data, status=200)

@util.api_method('GET')

def list_metadata(request, server_id):

    # Normal Response Codes: 200, 203

    # Error Response Codes: computeFault (400, 500),

    #                       serviceUnavailable (503),

    #                       unauthorized (401),

    #                       badRequest (400),

    #                       overLimit (413)

    log.debug('list_server_metadata %s', server_id)

    vm = util.get_vm(server_id, request.user_uniq)

    metadata = dict((m.meta_key, m.meta_value) for m in vm.metadata.all())

    return util.render_metadata(request, metadata, use_values=True, status=200)

@util.api_method('POST')

def update_metadata(request, server_id):

    # Normal Response Code: 201

    # Error Response Codes: computeFault (400, 500),

    #                       serviceUnavailable (503),

    #                       unauthorized (401),

    #                       badRequest (400),

    #                       buildInProgress (409),

    #                       badMediaType(415),

    #                       overLimit (413)

    req = util.get_request_dict(request)

    log.info('update_server_metadata %s %s', server_id, req)

    vm = util.get_vm(server_id, request.user_uniq, non_suspended=True)

    try:

        metadata = req['metadata']

        assert isinstance(metadata, dict)

    except (KeyError, AssertionError):

        raise faults.BadRequest("Malformed request")

    for key, val in metadata.items():

        meta, created = vm.metadata.get_or_create(meta_key=key)

        meta.meta_value = val

        meta.save()

    vm.save()

    vm_meta = dict((m.meta_key, m.meta_value) for m in vm.metadata.all())

    return util.render_metadata(request, vm_meta, status=201)

@util.api_method('GET')

def get_metadata_item(request, server_id, key):

    # Normal Response Codes: 200, 203

    # Error Response Codes: computeFault (400, 500),

    #                       serviceUnavailable (503),

    #                       unauthorized (401),

    #                       itemNotFound (404),

    #                       badRequest (400),

    #                       overLimit (413)

    log.debug('get_server_metadata_item %s %s', server_id, key)

    vm = util.get_vm(server_id, request.user_uniq)

    meta = util.get_vm_meta(vm, key)

    d = {meta.meta_key: meta.meta_value}

    return util.render_meta(request, d, status=200)

@util.api_method('PUT')

@transaction.commit_on_success

def create_metadata_item(request, server_id, key):

    # Normal Response Code: 201

    # Error Response Codes: computeFault (400, 500),

    #                       serviceUnavailable (503),

    #                       unauthorized (401),

    #                       itemNotFound (404),

    #                       badRequest (400),

    #                       buildInProgress (409),

    #                       badMediaType(415),

    #                       overLimit (413)

    req = util.get_request_dict(request)

    log.info('create_server_metadata_item %s %s %s', server_id, key, req)

    vm = util.get_vm(server_id, request.user_uniq, non_suspended=True)

    try:

        metadict = req['meta']

        assert isinstance(metadict, dict)

        assert len(metadict) == 1

        assert key in metadict

    except (KeyError, AssertionError):

        raise faults.BadRequest("Malformed request")

    meta, created = VirtualMachineMetadata.objects.get_or_create(

        meta_key=key,

        vm=vm)

    meta.meta_value = metadict[key]

    meta.save()

    vm.save()

    d = {meta.meta_key: meta.meta_value}

    return util.render_meta(request, d, status=201)

@util.api_method('DELETE')

@transaction.commit_on_success

def delete_metadata_item(request, server_id, key):

    # Normal Response Code: 204

    # Error Response Codes: computeFault (400, 500),

    #                       serviceUnavailable (503),

    #                       unauthorized (401),

    #                       itemNotFound (404),

    #                       badRequest (400),

    #                       buildInProgress (409),

    #                       badMediaType(415),

    #                       overLimit (413),

    log.info('delete_server_metadata_item %s %s', server_id, key)

    vm = util.get_vm(server_id, request.user_uniq, non_suspended=True)

    meta = util.get_vm_meta(vm, key)

    meta.delete()

    vm.save()

    return HttpResponse(status=204)

@util.api_method('GET')

def server_stats(request, server_id):

    # Normal Response Codes: 200

    # Error Response Codes: computeFault (400, 500),

    #                       serviceUnavailable (503),

    #                       unauthorized (401),

    #                       badRequest (400),

    #                       itemNotFound (404),

    #                       overLimit (413)

    log.debug('server_stats %s', server_id)

    vm = util.get_vm(server_id, request.user_uniq)

    #secret = util.encrypt(vm.backend_vm_id)

    secret = vm.backend_vm_id      # XXX disable backend id encryption

    stats = {

        'serverRef': vm.id,

        'refresh': settings.STATS_REFRESH_PERIOD,

        'cpuBar': settings.CPU_BAR_GRAPH_URL % secret,

        'cpuTimeSeries': settings.CPU_TIMESERIES_GRAPH_URL % secret,

        'netBar': settings.NET_BAR_GRAPH_URL % secret,

        'netTimeSeries': settings.NET_TIMESERIES_GRAPH_URL % secret}

    if request.serialization == 'xml':

        data = render_to_string('server_stats.xml', stats)

    else:

        data = json.dumps({'stats': stats})

    return HttpResponse(data, status=200)