commit 5d80553360b232a31cf89fa560f73849cd3ea42c
Author: Christos Stavrakakis <cstavr@grnet.gr>
Date:   Thu Jan 23 23:10:59 2014 +0200

    cyclades: Create servers with volumes
    
    Extend Cyclades POST /servers API endpoint to support specifying volumes
    when creating a server. The call is extended with the
    'block_device_mapping_v2' field that contains a list of dictionaries,
    each of which contains the following information about the devices that
    the server must have:
    
    * source_type: One of'image', 'snapshot', 'volume' or 'blank'
    * uuid: The UUID of the object that is specified by source_type (if
            source_type is not blank)
    * size: The size of the volume
    * delete_on_termination: Whether the volumes will be preserved or
                             automaticaly deleted when the server is deleted
    
    If no volume is specified, then a volume is automatically created with
    the size of the flavor and with source the image that is specified in
    the 'imageRef' field of the API call.
    
    The follow restrictions apply:
    * Source_type can be other than 'blank', only if the disk template is
      ext_ or if the volume is created as that root volume of a server, so
      that it will filled with data by snf-image.
    * The root volume cannot be blank.
    * In order to use a volume, it must be in AVAILABLE status. This will
      never happend until detachable volumes. However the user can "clone"
      a user volume either by creating a new volume from cinder and then
      using this new volume or by taking a snapshot and then use the
      snapshot as a source.
    * The size field is required if the source is image, snapshot or blank.
      Also, the size of the volume must be equal or bigger from the source
      size (if any)
    
    Finally move some code from 'servers' module to 'server_attachments' and
    'commands' to avoid cyclic imports.

diff --git a/snf-cyclades-app/synnefo/logic/servers.py b/snf-cyclades-app/synnefo/logic/servers.py
index 9ec38a2..6e16d6f 100644
--- a/snf-cyclades-app/synnefo/logic/servers.py
+++ b/snf-cyclades-app/synnefo/logic/servers.py
@@ -30,22 +30,22 @@
 import logging
 
 from socket import getfqdn
-from functools import wraps
 from django import dispatch
 from django.db import transaction
 from django.utils import simplejson as json
 
 from snf_django.lib.api import faults
 from django.conf import settings
-from synnefo import quotas
 from synnefo.api import util
 from synnefo.logic import backend, ips, utils
 from synnefo.logic.backend_allocator import BackendAllocator
 from synnefo.db.models import (NetworkInterface, VirtualMachine,
-                               VirtualMachineMetadata, IPAddressLog, Network,
-                               Volume)
+                               VirtualMachineMetadata, IPAddressLog, Network)
 from vncauthproxy.client import request_forwarding as request_vnc_forwarding
 from synnefo.logic import rapi
+from synnefo.volume.volumes import _create_volume
+from synnefo.volume.util import get_volume
+from synnefo.logic import commands
 
 log = logging.getLogger(__name__)
 
@@ -53,129 +53,20 @@ log = logging.getLogger(__name__)
 server_created = dispatch.Signal(providing_args=["created_vm_params"])
 
 
-def validate_server_action(vm, action):
-    if vm.deleted:
-        raise faults.BadRequest("Server '%s' has been deleted." % vm.id)
-
-    # Destroyin a server should always be permitted
-    if action == "DESTROY":
-        return
-
-    # Check that there is no pending action
-    pending_action = vm.task
-    if pending_action:
-        if pending_action == "BUILD":
-            raise faults.BuildInProgress("Server '%s' is being build." % vm.id)
-        raise faults.BadRequest("Cannot perform '%s' action while there is a"
-                                " pending '%s'." % (action, pending_action))
-
-    # Check if action can be performed to VM's operstate
-    operstate = vm.operstate
-    if operstate == "ERROR":
-        raise faults.BadRequest("Cannot perform '%s' action while server is"
-                                " in 'ERROR' state." % action)
-    elif operstate == "BUILD" and action != "BUILD":
-        raise faults.BuildInProgress("Server '%s' is being build." % vm.id)
-    elif (action == "START" and operstate != "STOPPED") or\
-         (action == "STOP" and operstate != "STARTED") or\
-         (action == "RESIZE" and operstate != "STOPPED") or\
-         (action in ["CONNECT", "DISCONNECT"]
-          and operstate != "STOPPED"
-          and not settings.GANETI_USE_HOTPLUG) or \
-         (action in ["ATTACH_VOLUME", "DETACH_VOLUME"]
-          and operstate != "STOPPED"
-          and not settings.GANETI_USE_HOTPLUG):
-        raise faults.BadRequest("Cannot perform '%s' action while server is"
-                                " in '%s' state." % (action, operstate))
-    return
-
-
-def server_command(action, action_fields=None):
-    """Handle execution of a server action.
-
-    Helper function to validate and execute a server action, handle quota
-    commission and update the 'task' of the VM in the DB.
-
-    1) Check if action can be performed. If it can, then there must be no
-       pending task (with the exception of DESTROY).
-    2) Handle previous commission if unresolved:
-       * If it is not pending and it to accept, then accept
-       * If it is not pending and to reject or is pending then reject it. Since
-       the action can be performed only if there is no pending task, then there
-       can be no pending commission. The exception is DESTROY, but in this case
-       the commission can safely be rejected, and the dispatcher will generate
-       the correct ones!
-    3) Issue new commission and associate it with the VM. Also clear the task.
-    4) Send job to ganeti
-    5) Update task and commit
-    """
-    def decorator(func):
-        @wraps(func)
-        @transaction.commit_on_success
-        def wrapper(vm, *args, **kwargs):
-            user_id = vm.userid
-            validate_server_action(vm, action)
-            vm.action = action
-
-            commission_name = "client: api, resource: %s" % vm
-            quotas.handle_resource_commission(vm, action=action,
-                                              action_fields=action_fields,
-                                              commission_name=commission_name)
-            vm.save()
-
-            # XXX: Special case for server creation!
-            if action == "BUILD":
-                # Perform a commit, because the VirtualMachine must be saved to
-                # DB before the OP_INSTANCE_CREATE job in enqueued in Ganeti.
-                # Otherwise, messages will arrive from snf-dispatcher about
-                # this instance, before the VM is stored in DB.
-                transaction.commit()
-                # After committing the locks are released. Refetch the instance
-                # to guarantee x-lock.
-                vm = VirtualMachine.objects.select_for_update().get(id=vm.id)
-
-            # Send the job to Ganeti and get the associated jobID
-            try:
-                job_id = func(vm, *args, **kwargs)
-            except Exception as e:
-                if vm.serial is not None:
-                    # Since the job never reached Ganeti, reject the commission
-                    log.debug("Rejecting commission: '%s', could not perform"
-                              " action '%s': %s" % (vm.serial,  action, e))
-                    transaction.rollback()
-                    quotas.reject_resource_serial(vm)
-                    transaction.commit()
-                raise
-
-            if action == "BUILD" and vm.serial is not None:
-                # XXX: Special case for server creation: we must accept the
-                # commission because the VM has been stored in DB. Also, if
-                # communication with Ganeti fails, the job will never reach
-                # Ganeti, and the commission will never be resolved.
-                quotas.accept_resource_serial(vm)
-
-            log.info("user: %s, vm: %s, action: %s, job_id: %s, serial: %s",
-                     user_id, vm.id, action, job_id, vm.serial)
-
-            # store the new task in the VM
-            if job_id is not None:
-                vm.task = action
-                vm.task_job_id = job_id
-            vm.save()
-
-            return vm
-        return wrapper
-    return decorator
-
-
 @transaction.commit_on_success
-def create(userid, name, password, flavor, image, metadata={},
-           personality=[], networks=None, use_backend=None):
+def create(userid, name, password, flavor, image_id, metadata={},
+           personality=[], networks=None, volumes=None,
+           use_backend=None):
+
+    # Get image information
+    # TODO: Image is not mandatory if disks are specified
+    image = util.get_image_dict(image_id, userid)
 
     # Check that image fits into the disk
-    if image["size"] > (flavor.disk << 30):
-        msg = "Flavor's disk size '%s' is smaller than the image's size '%s'"
-        raise faults.BadRequest(msg % (flavor.disk << 30, image["size"]))
+    if int(image["size"]) > (flavor.disk << 30):
+        msg = ("Flavor's disk size '%s' is smaller than the image's"
+               "size '%s'" % (flavor.disk << 30, image["size"]))
+        raise faults.BadRequest(msg)
 
     if use_backend is None:
         # Allocate server to a Ganeti backend
@@ -187,18 +78,6 @@ def create(userid, name, password, flavor, image, metadata={},
     # Create the ports for the server
     ports = create_instance_ports(userid, networks)
 
-    # Fix flavor for archipelago
-    disk_template, provider = util.get_flavor_provider(flavor)
-    if provider:
-        flavor.disk_template = disk_template
-        flavor.disk_provider = provider
-        flavor.disk_origin = None
-        if provider in settings.GANETI_CLONE_PROVIDERS:
-            flavor.disk_origin = image['checksum']
-            image['backend_id'] = 'null'
-    else:
-        flavor.disk_provider = None
-
     # We must save the VM instance now, so that it gets a valid
     # vm.backend_vm_id.
     vm = VirtualMachine.objects.create(name=name,
@@ -215,7 +94,35 @@ def create(userid, name, password, flavor, image, metadata={},
         port.index = index
         port.save()
 
-    volumes = create_instance_volumes(vm, flavor, image)
+    # If no volumes are specified, we automatically create a volume with the
+    # size of the flavor and filled with the specified image.
+    if not volumes:
+        volumes = [{"source_type": "image",
+                    "source_uuid": image["id"],
+                    "size": flavor.disk,
+                    "delete_on_termination": True}]
+
+    assert(len(volumes) > 0), "Cannot create server without volumes"
+
+    if volumes[0]["source_type"] == "blank":
+        raise faults.BadRequest("Root volume cannot be blank")
+
+    server_volumes = []
+    for index, vol_info in enumerate(volumes):
+        if vol_info["source_type"] == "volume":
+            uuid = vol_info["source_uuid"]
+            v = get_volume(userid, uuid, for_update=True,
+                           exception=faults.BadRequest)
+            if v.status != "AVAILABLE":
+                raise faults.BadRequest("Cannot use volume while it is in %s"
+                                        " status" % v.status)
+            v.delete_on_termination = vol_info["delete_on_termination"]
+            v.index = index
+            v.save()
+        else:
+            v = _create_volume(server=vm, user_id=userid,
+                               index=index, **vol_info)
+        server_volumes.append(v)
 
     for key, val in metadata.items():
         VirtualMachineMetadata.objects.create(
@@ -224,7 +131,7 @@ def create(userid, name, password, flavor, image, metadata={},
             vm=vm)
 
     # Create the server in Ganeti.
-    vm = create_server(vm, ports, volumes, flavor, image, personality,
+    vm = create_server(vm, ports, server_volumes, flavor, image, personality,
                        password)
 
     return vm
@@ -250,13 +157,21 @@ def allocate_new_server(userid, flavor):
     return use_backend
 
 
-@server_command("BUILD")
+@commands.server_command("BUILD")
 def create_server(vm, nics, volumes, flavor, image, personality, password):
     # dispatch server created signal needed to trigger the 'vmapi', which
     # enriches the vm object with the 'config_url' attribute which must be
     # passed to the Ganeti job.
+
+    # If the root volume has a provider, then inform snf-image to not fill
+    # the volume with data
+    image_id = image["backend_id"]
+    root_volume = volumes[0]
+    if root_volume.origin is not None:
+        image_id = "null"
+
     server_created.send(sender=vm, created_vm_params={
-        'img_id': image['backend_id'],
+        'img_id': image_id,
         'img_passwd': password,
         'img_format': str(image['format']),
         'img_personality': json.dumps(personality),
@@ -283,22 +198,7 @@ def create_server(vm, nics, volumes, flavor, image, personality, password):
     return jobID
 
 
-def create_instance_volumes(vm, flavor, image):
-    name = "Root volume of server: %s" % vm.id
-    volume = Volume.objects.create(userid=vm.userid,
-                                   machine=vm,
-                                   name=name,
-                                   size=flavor.disk,
-                                   source=Volume.SOURCE_IMAGE_PREFIX+image["id"],
-                                   origin=image["checksum"],
-                                   status="CREATING")
-
-    volume.save()
-
-    return [volume]
-
-
-@server_command("DESTROY")
+@commands.server_command("DESTROY")
 def destroy(vm, shutdown_timeout=None):
     # XXX: Workaround for race where OP_INSTANCE_REMOVE starts executing on
     # Ganeti before OP_INSTANCE_CREATE. This will be fixed when
@@ -312,19 +212,19 @@ def destroy(vm, shutdown_timeout=None):
     return backend.delete_instance(vm, shutdown_timeout=shutdown_timeout)
 
 
-@server_command("START")
+@commands.server_command("START")
 def start(vm):
     log.info("Starting VM %s", vm)
     return backend.startup_instance(vm)
 
 
-@server_command("STOP")
+@commands.server_command("STOP")
 def stop(vm, shutdown_timeout=None):
     log.info("Stopping VM %s", vm)
     return backend.shutdown_instance(vm, shutdown_timeout=shutdown_timeout)
 
 
-@server_command("REBOOT")
+@commands.server_command("REBOOT")
 def reboot(vm, reboot_type, shutdown_timeout=None):
     if reboot_type not in ("SOFT", "HARD"):
         raise faults.BadRequest("Malformed request. Invalid reboot"
@@ -338,7 +238,7 @@ def reboot(vm, reboot_type, shutdown_timeout=None):
 def resize(vm, flavor):
     action_fields = {"beparams": {"vcpus": flavor.cpu,
                                   "maxmem": flavor.ram}}
-    comm = server_command("RESIZE", action_fields=action_fields)
+    comm = commands.server_command("RESIZE", action_fields=action_fields)
     return comm(_resize)(vm, flavor)
 
 
@@ -358,7 +258,7 @@ def _resize(vm, flavor):
     return backend.resize_instance(vm, vcpus=flavor.cpu, memory=flavor.ram)
 
 
-@server_command("SET_FIREWALL_PROFILE")
+@commands.server_command("SET_FIREWALL_PROFILE")
 def set_firewall_profile(vm, profile, nic):
     log.info("Setting VM %s, NIC %s, firewall %s", vm, nic, profile)
 
@@ -368,7 +268,7 @@ def set_firewall_profile(vm, profile, nic):
     return None
 
 
-@server_command("CONNECT")
+@commands.server_command("CONNECT")
 def connect(vm, network, port=None):
     if port is None:
         port = _create_port(vm.userid, network)
@@ -379,7 +279,7 @@ def connect(vm, network, port=None):
     return backend.connect_to_network(vm, port)
 
 
-@server_command("DISCONNECT")
+@commands.server_command("DISCONNECT")
 def disconnect(vm, nic):
     log.info("Removing NIC %s from VM %s", nic, vm)
     return backend.disconnect_from_network(vm, nic)
@@ -760,76 +660,3 @@ def _port_for_request(user_id, network_dict):
     else:
         raise faults.BadRequest("Network 'uuid' or 'port' attribute"
                                 " is required.")
-
-
-@server_command("ATTACH_VOLUME")
-def attach_volume(vm, volume):
-    """Attach a volume to a server.
-
-    The volume must be in 'AVAILABLE' status in order to be attached. Also,
-    number of the volumes that are attached to the server must remain less
-    than 'GANETI_MAX_DISKS_PER_INSTANCE' setting. This function will send
-    the corresponding job to Ganeti backend and update the status of the
-    volume to 'ATTACHING'.
-
-    """
-    # Check volume state
-    if volume.status not in ["AVAILABLE", "CREATING"]:
-        raise faults.BadRequest("Cannot attach volume while volume is in"
-                                " '%s' status." % volume.status)
-
-    # Check that disk templates are the same
-    if volume.disk_template != vm.flavor.disk_template:
-        msg = ("Volume and server must have the same disk template. Volume has"
-               " disk template '%s' while server has '%s'"
-               % (volume.disk_template, vm.flavor.disk_template))
-        raise faults.BadRequest(msg)
-
-    # Check maximum disk per instance hard limit
-    if vm.volumes.count() == settings.GANETI_MAX_DISKS_PER_INSTANCE:
-        raise faults.BadRequest("Maximum volumes per server limit reached")
-
-    jobid = backend.attach_volume(vm, volume)
-
-    log.info("Attached volume '%s' to server '%s'. JobID: '%s'", volume.id,
-             volume.machine_id, jobid)
-
-    volume.backendjobid = jobid
-    volume.machine = vm
-    volume.status = "ATTACHING"
-    volume.save()
-    return jobid
-
-
-@server_command("DETACH_VOLUME")
-def detach_volume(vm, volume):
-    """Detach a volume to a server.
-
-    The volume must be in 'IN_USE' status in order to be detached. Also,
-    the root volume of the instance (index=0) can not be detached. This
-    function will send the corresponding job to Ganeti backend and update the
-    status of the volume to 'DETACHING'.
-
-    """
-
-    _check_attachment(vm, volume)
-    if volume.status != "IN_USE":
-        #TODO: Maybe allow other statuses as well ?
-        raise faults.BadRequest("Cannot detach volume while volume is in"
-                                " '%s' status." % volume.status)
-    if volume.index == 0:
-        raise faults.BadRequest("Cannot detach the root volume of a server")
-    jobid = backend.detach_volume(vm, volume)
-    log.info("Detached volume '%s' from server '%s'. JobID: '%s'", volume.id,
-             volume.machine_id, jobid)
-    volume.backendjobid = jobid
-    volume.status = "DETACHING"
-    volume.save()
-    return jobid
-
-
-def _check_attachment(vm, volume):
-    """Check that volume is attached to vm."""
-    if volume.machine_id != vm.id:
-        raise faults.BadRequest("Volume '%s' is not attached to server '%s'"
-                                % volume.id, vm.id)
