X-Git-Url: https://code.grnet.gr/git/ganeti-local/blobdiff_plain/e8448672828cd801991e2d513e704e72901c5068..4a78c361a6de3bcbf98f02abfe41ae3b11de2b00:/lib/objects.py diff --git a/lib/objects.py b/lib/objects.py index 3ed05e9..5ba4a1b 100644 --- a/lib/objects.py +++ b/lib/objects.py @@ -1,7 +1,7 @@ # # -# Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011 Google Inc. +# Copyright (C) 2006, 2007, 2008, 2009, 2010, 2011, 2012 Google Inc. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by @@ -26,13 +26,15 @@ pass to and from external parties. """ -# pylint: disable-msg=E0203,W0201 +# pylint: disable=E0203,W0201,R0902 # E0203: Access to member %r before its definition, since we use # objects.py which doesn't explicitely initialise its members # W0201: Attribute '%s' defined outside __init__ +# R0902: Allow instances of these objects to have more than 20 attributes + import ConfigParser import re import copy @@ -41,6 +43,8 @@ from cStringIO import StringIO from ganeti import errors from ganeti import constants +from ganeti import netutils +from ganeti import utils from socket import AF_INET @@ -76,6 +80,39 @@ def FillDict(defaults_dict, custom_dict, skip_keys=None): return ret_dict +def FillIPolicy(default_ipolicy, custom_ipolicy, skip_keys=None): + """Fills an instance policy with defaults. + + """ + assert frozenset(default_ipolicy.keys()) == constants.IPOLICY_ALL_KEYS + ret_dict = {} + for key in constants.IPOLICY_ISPECS: + ret_dict[key] = FillDict(default_ipolicy[key], + custom_ipolicy.get(key, {}), + skip_keys=skip_keys) + # list items + for key in [constants.IPOLICY_DTS]: + ret_dict[key] = list(custom_ipolicy.get(key, default_ipolicy[key])) + # other items which we know we can directly copy (immutables) + for key in constants.IPOLICY_PARAMETERS: + ret_dict[key] = custom_ipolicy.get(key, default_ipolicy[key]) + + return ret_dict + + +def FillDiskParams(default_dparams, custom_dparams, skip_keys=None): + """Fills the disk parameter defaults. + + @see: L{FillDict} for parameters and return value + + """ + assert frozenset(default_dparams.keys()) == constants.DISK_TEMPLATES + + return dict((dt, FillDict(default_dparams[dt], custom_dparams.get(dt, {}), + skip_keys=skip_keys)) + for dt in constants.DISK_TEMPLATES) + + def UpgradeGroupedParams(target, defaults): """Update all groups for the target parameter. @@ -93,6 +130,63 @@ def UpgradeGroupedParams(target, defaults): return target +def UpgradeBeParams(target): + """Update the be parameters dict to the new format. + + @type target: dict + @param target: "be" parameters dict + + """ + if constants.BE_MEMORY in target: + memory = target[constants.BE_MEMORY] + target[constants.BE_MAXMEM] = memory + target[constants.BE_MINMEM] = memory + del target[constants.BE_MEMORY] + + +def UpgradeDiskParams(diskparams): + """Upgrade the disk parameters. + + @type diskparams: dict + @param diskparams: disk parameters to upgrade + @rtype: dict + @return: the upgraded disk parameters dict + + """ + if not diskparams: + result = {} + else: + result = FillDiskParams(constants.DISK_DT_DEFAULTS, diskparams) + + return result + + +def UpgradeNDParams(ndparams): + """Upgrade ndparams structure. + + @type ndparams: dict + @param ndparams: disk parameters to upgrade + @rtype: dict + @return: the upgraded node parameters dict + + """ + if ndparams is None: + ndparams = {} + + return FillDict(constants.NDC_DEFAULTS, ndparams) + + +def MakeEmptyIPolicy(): + """Create empty IPolicy dictionary. + + """ + return dict([ + (constants.ISPECS_MIN, {}), + (constants.ISPECS_MAX, {}), + (constants.ISPECS_STD, {}), + ]) + + class ConfigObject(object): """A generic config object. @@ -134,6 +228,9 @@ class ConfigObject(object): slots.extend(getattr(parent, "__slots__", [])) return slots + #: Public getter for the defined slots + GetAllSlots = _all_slots + def ToDict(self): """Convert to a dict holding only standard python types. @@ -170,7 +267,7 @@ class ConfigObject(object): raise errors.ConfigurationError("Invalid object passed to FromDict:" " expected dict, got %s" % type(val)) val_str = dict([(str(k), v) for k, v in val.iteritems()]) - obj = cls(**val_str) # pylint: disable-msg=W0142 + obj = cls(**val_str) # pylint: disable=W0142 return obj @staticmethod @@ -315,6 +412,25 @@ class TaggableObject(ConfigObject): return obj +class MasterNetworkParameters(ConfigObject): + """Network configuration parameters for the master + + @ivar name: master name + @ivar ip: master IP + @ivar netmask: master netmask + @ivar netdev: master network device + @ivar ip_family: master IP family + + """ + __slots__ = [ + "name", + "ip", + "netmask", + "netdev", + "ip_family" + ] + + class ConfigData(ConfigObject): """Top-level config object.""" __slots__ = [ @@ -415,7 +531,7 @@ class NIC(ConfigObject): class Disk(ConfigObject): """Config object representing a block device.""" __slots__ = ["dev_type", "logical_id", "physical_id", - "children", "iv_name", "size", "mode"] + "children", "iv_name", "size", "mode", "params"] def CreateOnSecondary(self): """Test if this device needs to be created on a secondary node.""" @@ -444,6 +560,8 @@ class Disk(ConfigObject): return "/dev/%s/%s" % (self.logical_id[0], self.logical_id[1]) elif self.dev_type == constants.LD_BLOCKDEV: return self.logical_id[1] + elif self.dev_type == constants.LD_RBD: + return "/dev/%s/%s" % (self.logical_id[0], self.logical_id[1]) return None def ChildrenNeeded(self): @@ -487,7 +605,7 @@ class Disk(ConfigObject): """ if self.dev_type in [constants.LD_LV, constants.LD_FILE, - constants.LD_BLOCKDEV]: + constants.LD_BLOCKDEV, constants.LD_RBD]: result = [node] elif self.dev_type in constants.LDS_DRBD: result = [self.logical_id[0], self.logical_id[1]] @@ -562,7 +680,8 @@ class Disk(ConfigObject): actual algorithms from bdev. """ - if self.dev_type in (constants.LD_LV, constants.LD_FILE): + if self.dev_type in (constants.LD_LV, constants.LD_FILE, + constants.LD_RBD): self.size += amount elif self.dev_type == constants.LD_DRBD8: if self.children: @@ -572,6 +691,21 @@ class Disk(ConfigObject): raise errors.ProgrammerError("Disk.RecordGrow called for unsupported" " disk type %s" % self.dev_type) + def Update(self, size=None, mode=None): + """Apply changes to size and mode. + + """ + if self.dev_type == constants.LD_DRBD8: + if self.children: + self.children[0].Update(size=size, mode=mode) + else: + assert not self.children + + if size is not None: + self.size = size + if mode is not None: + self.mode = mode + def UnsetSize(self): """Sets recursively the size to zero for the disk and its children. @@ -663,7 +797,7 @@ class Disk(ConfigObject): """ if self.dev_type == constants.LD_LV: - val = " parameters + @rtype: list(dict) + @return: a list of dicts, one for each node of the disk hierarchy. Each dict + contains the LD parameters of the node. The tree is flattened in-order. + + """ + if disk_template not in constants.DISK_TEMPLATES: + raise errors.ProgrammerError("Unknown disk template %s" % disk_template) + + assert disk_template in disk_params + + result = list() + dt_params = disk_params[disk_template] + if disk_template == constants.DT_DRBD8: + drbd_params = { + constants.LDP_RESYNC_RATE: dt_params[constants.DRBD_RESYNC_RATE], + constants.LDP_BARRIERS: dt_params[constants.DRBD_DISK_BARRIERS], + constants.LDP_NO_META_FLUSH: dt_params[constants.DRBD_META_BARRIERS], + constants.LDP_DEFAULT_METAVG: dt_params[constants.DRBD_DEFAULT_METAVG], + constants.LDP_DISK_CUSTOM: dt_params[constants.DRBD_DISK_CUSTOM], + constants.LDP_NET_CUSTOM: dt_params[constants.DRBD_NET_CUSTOM], + constants.LDP_DYNAMIC_RESYNC: dt_params[constants.DRBD_DYNAMIC_RESYNC], + constants.LDP_PLAN_AHEAD: dt_params[constants.DRBD_PLAN_AHEAD], + constants.LDP_FILL_TARGET: dt_params[constants.DRBD_FILL_TARGET], + constants.LDP_DELAY_TARGET: dt_params[constants.DRBD_DELAY_TARGET], + constants.LDP_MAX_RATE: dt_params[constants.DRBD_MAX_RATE], + constants.LDP_MIN_RATE: dt_params[constants.DRBD_MIN_RATE], + } + + drbd_params = \ + FillDict(constants.DISK_LD_DEFAULTS[constants.LD_DRBD8], + drbd_params) + + result.append(drbd_params) + + # data LV + data_params = { + constants.LDP_STRIPES: dt_params[constants.DRBD_DATA_STRIPES], + } + data_params = \ + FillDict(constants.DISK_LD_DEFAULTS[constants.LD_LV], + data_params) + result.append(data_params) + + # metadata LV + meta_params = { + constants.LDP_STRIPES: dt_params[constants.DRBD_META_STRIPES], + } + meta_params = \ + FillDict(constants.DISK_LD_DEFAULTS[constants.LD_LV], + meta_params) + result.append(meta_params) + + elif (disk_template == constants.DT_FILE or + disk_template == constants.DT_SHARED_FILE): + result.append(constants.DISK_LD_DEFAULTS[constants.LD_FILE]) + + elif disk_template == constants.DT_PLAIN: + params = { + constants.LDP_STRIPES: dt_params[constants.LV_STRIPES], + } + params = \ + FillDict(constants.DISK_LD_DEFAULTS[constants.LD_LV], + params) + result.append(params) + + elif disk_template == constants.DT_BLOCK: + result.append(constants.DISK_LD_DEFAULTS[constants.LD_BLOCKDEV]) + + elif disk_template == constants.DT_RBD: + params = { + constants.LDP_POOL: dt_params[constants.RBD_POOL] + } + params = \ + FillDict(constants.DISK_LD_DEFAULTS[constants.LD_RBD], + params) + result.append(params) + + return result + + +class InstancePolicy(ConfigObject): + """Config object representing instance policy limits dictionary. + + + Note that this object is not actually used in the config, it's just + used as a placeholder for a few functions. + + """ + @classmethod + def CheckParameterSyntax(cls, ipolicy): + """ Check the instance policy for validity. + + """ + for param in constants.ISPECS_PARAMETERS: + InstancePolicy.CheckISpecSyntax(ipolicy, param) + if constants.IPOLICY_DTS in ipolicy: + InstancePolicy.CheckDiskTemplates(ipolicy[constants.IPOLICY_DTS]) + for key in constants.IPOLICY_PARAMETERS: + if key in ipolicy: + InstancePolicy.CheckParameter(key, ipolicy[key]) + wrong_keys = frozenset(ipolicy.keys()) - constants.IPOLICY_ALL_KEYS + if wrong_keys: + raise errors.ConfigurationError("Invalid keys in ipolicy: %s" % + utils.CommaJoin(wrong_keys)) + + @classmethod + def CheckISpecSyntax(cls, ipolicy, name): + """Check the instance policy for validity on a given key. + + We check if the instance policy makes sense for a given key, that is + if ipolicy[min][name] <= ipolicy[std][name] <= ipolicy[max][name]. + + @type ipolicy: dict + @param ipolicy: dictionary with min, max, std specs + @type name: string + @param name: what are the limits for + @raise errors.ConfigureError: when specs for given name are not valid + + """ + min_v = ipolicy[constants.ISPECS_MIN].get(name, 0) + std_v = ipolicy[constants.ISPECS_STD].get(name, min_v) + max_v = ipolicy[constants.ISPECS_MAX].get(name, std_v) + err = ("Invalid specification of min/max/std values for %s: %s/%s/%s" % + (name, + ipolicy[constants.ISPECS_MIN].get(name, "-"), + ipolicy[constants.ISPECS_MAX].get(name, "-"), + ipolicy[constants.ISPECS_STD].get(name, "-"))) + if min_v > std_v or std_v > max_v: + raise errors.ConfigurationError(err) + + @classmethod + def CheckDiskTemplates(cls, disk_templates): + """Checks the disk templates for validity. + + """ + wrong = frozenset(disk_templates).difference(constants.DISK_TEMPLATES) + if wrong: + raise errors.ConfigurationError("Invalid disk template(s) %s" % + utils.CommaJoin(wrong)) + + @classmethod + def CheckParameter(cls, key, value): + """Checks a parameter. + + Currently we expect all parameters to be float values. + + """ + try: + float(value) + except (TypeError, ValueError), err: + raise errors.ConfigurationError("Invalid value for key" " '%s':" + " '%s', error: %s" % (key, value, err)) + class Instance(TaggableObject): """Config object representing an instance.""" @@ -722,7 +1024,7 @@ class Instance(TaggableObject): "hvparams", "beparams", "osparams", - "admin_up", + "admin_state", "nics", "disks", "disk_template", @@ -790,7 +1092,9 @@ class Instance(TaggableObject): node = self.primary_node if lvmap is None: - lvmap = { node : [] } + lvmap = { + node: [], + } ret = lvmap else: if not node in lvmap: @@ -802,7 +1106,7 @@ class Instance(TaggableObject): for dev in devs: if dev.dev_type == constants.LD_LV: - lvmap[node].append(dev.logical_id[0]+"/"+dev.logical_id[1]) + lvmap[node].append(dev.logical_id[0] + "/" + dev.logical_id[1]) elif dev.dev_type in constants.LDS_DRBD: if dev.children: @@ -860,6 +1164,13 @@ class Instance(TaggableObject): """Custom function for instances. """ + if "admin_state" not in val: + if val.get("admin_up", False): + val["admin_state"] = constants.ADMINST_UP + else: + val["admin_state"] = constants.ADMINST_DOWN + if "admin_up" in val: + del val["admin_up"] obj = super(Instance, cls).FromDict(val) obj.nics = cls._ContainerFromDicts(obj.nics, list, NIC) obj.disks = cls._ContainerFromDicts(obj.disks, list, Disk) @@ -881,6 +1192,7 @@ class Instance(TaggableObject): pass if self.osparams is None: self.osparams = {} + UpgradeBeParams(self.beparams) class OS(ConfigObject): @@ -943,8 +1255,49 @@ class OS(ConfigObject): return cls.SplitNameVariant(name)[1] +class NodeHvState(ConfigObject): + """Hypvervisor state on a node. + + @ivar mem_total: Total amount of memory + @ivar mem_node: Memory used by, or reserved for, the node itself (not always + available) + @ivar mem_hv: Memory used by hypervisor or lost due to instance allocation + rounding + @ivar mem_inst: Memory used by instances living on node + @ivar cpu_total: Total node CPU core count + @ivar cpu_node: Number of CPU cores reserved for the node itself + + """ + __slots__ = [ + "mem_total", + "mem_node", + "mem_hv", + "mem_inst", + "cpu_total", + "cpu_node", + ] + _TIMESTAMPS + + +class NodeDiskState(ConfigObject): + """Disk state on a node. + + """ + __slots__ = [ + "total", + "reserved", + "overhead", + ] + _TIMESTAMPS + + class Node(TaggableObject): - """Config object representing a node.""" + """Config object representing a node. + + @ivar hv_state: Hypervisor state (e.g. number of CPUs) + @ivar hv_state_static: Hypervisor state overriden by user + @ivar disk_state: Disk state (e.g. free space) + @ivar disk_state_static: Disk state overriden by user + + """ __slots__ = [ "name", "primary_ip", @@ -958,13 +1311,17 @@ class Node(TaggableObject): "vm_capable", "ndparams", "powered", + "hv_state", + "hv_state_static", + "disk_state", + "disk_state_static", ] + _TIMESTAMPS + _UUID def UpgradeConfig(self): """Fill defaults for missing configuration values. """ - # pylint: disable-msg=E0203 + # pylint: disable=E0203 # because these are "defined" via slots, not manually if self.master_capable is None: self.master_capable = True @@ -978,6 +1335,41 @@ class Node(TaggableObject): if self.powered is None: self.powered = True + def ToDict(self): + """Custom function for serializing. + + """ + data = super(Node, self).ToDict() + + hv_state = data.get("hv_state", None) + if hv_state is not None: + data["hv_state"] = self._ContainerToDicts(hv_state) + + disk_state = data.get("disk_state", None) + if disk_state is not None: + data["disk_state"] = \ + dict((key, self._ContainerToDicts(value)) + for (key, value) in disk_state.items()) + + return data + + @classmethod + def FromDict(cls, val): + """Custom function for deserializing. + + """ + obj = super(Node, cls).FromDict(val) + + if obj.hv_state is not None: + obj.hv_state = cls._ContainerFromDicts(obj.hv_state, dict, NodeHvState) + + if obj.disk_state is not None: + obj.disk_state = \ + dict((key, cls._ContainerFromDicts(value, dict, NodeDiskState)) + for (key, value) in obj.disk_state.items()) + + return obj + class NodeGroup(TaggableObject): """Config object representing a node group.""" @@ -985,7 +1377,11 @@ class NodeGroup(TaggableObject): "name", "members", "ndparams", + "diskparams", + "ipolicy", "serial_no", + "hv_state_static", + "disk_state_static", "alloc_policy", ] + _TIMESTAMPS + _UUID @@ -1024,11 +1420,16 @@ class NodeGroup(TaggableObject): if self.alloc_policy is None: self.alloc_policy = constants.ALLOC_POLICY_PREFERRED - # We only update mtime, and not ctime, since we would not be able to provide - # a correct value for creation time. + # We only update mtime, and not ctime, since we would not be able + # to provide a correct value for creation time. if self.mtime is None: self.mtime = time.time() + if self.diskparams is None: + self.diskparams = {} + if self.ipolicy is None: + self.ipolicy = MakeEmptyIPolicy() + def FillND(self, node): """Return filled out ndparams for L{objects.Node} @@ -1068,16 +1469,20 @@ class Cluster(TaggableObject): "master_node", "master_ip", "master_netdev", + "master_netmask", + "use_external_mip_script", "cluster_name", "file_storage_dir", "shared_file_storage_dir", "enabled_hypervisors", "hvparams", + "ipolicy", "os_hvp", "beparams", "osparams", "nicparams", "ndparams", + "diskparams", "candidate_pool_size", "modify_etc_hosts", "modify_ssh_setup", @@ -1088,13 +1493,15 @@ class Cluster(TaggableObject): "blacklisted_os", "primary_ip_family", "prealloc_wipe_disks", + "hv_state_static", + "disk_state_static", ] + _TIMESTAMPS + _UUID def UpgradeConfig(self): """Fill defaults for missing configuration values. """ - # pylint: disable-msg=E0203 + # pylint: disable=E0203 # because these are "defined" via slots, not manually if self.hvparams is None: self.hvparams = constants.HVC_DEFAULTS @@ -1110,11 +1517,13 @@ class Cluster(TaggableObject): if self.osparams is None: self.osparams = {} - if self.ndparams is None: - self.ndparams = constants.NDC_DEFAULTS + self.ndparams = UpgradeNDParams(self.ndparams) self.beparams = UpgradeGroupedParams(self.beparams, constants.BEC_DEFAULTS) + for beparams_group in self.beparams: + UpgradeBeParams(self.beparams[beparams_group]) + migrate_default_bridge = not self.nicparams self.nicparams = UpgradeGroupedParams(self.nicparams, constants.NICC_DEFAULTS) @@ -1167,6 +1576,10 @@ class Cluster(TaggableObject): if self.primary_ip_family is None: self.primary_ip_family = AF_INET + if self.master_netmask is None: + ipcls = netutils.IPAddress.GetClassFromIpFamily(self.primary_ip_family) + self.master_netmask = ipcls.iplen + if self.prealloc_wipe_disks is None: self.prealloc_wipe_disks = False @@ -1174,6 +1587,32 @@ class Cluster(TaggableObject): if self.shared_file_storage_dir is None: self.shared_file_storage_dir = "" + if self.use_external_mip_script is None: + self.use_external_mip_script = False + + if self.diskparams: + self.diskparams = UpgradeDiskParams(self.diskparams) + else: + self.diskparams = constants.DISK_DT_DEFAULTS.copy() + + # instance policy added before 2.6 + if self.ipolicy is None: + self.ipolicy = FillIPolicy(constants.IPOLICY_DEFAULTS, {}) + else: + # we can either make sure to upgrade the ipolicy always, or only + # do it in some corner cases (e.g. missing keys); note that this + # will break any removal of keys from the ipolicy dict + self.ipolicy = FillIPolicy(constants.IPOLICY_DEFAULTS, self.ipolicy) + + @property + def primary_hypervisor(self): + """The first hypervisor is the primary. + + Useful, for example, for L{Node}'s hv/disk state. + + """ + return self.enabled_hypervisors[0] + def ToDict(self): """Custom function for cluster. @@ -1192,6 +1631,15 @@ class Cluster(TaggableObject): obj.tcpudp_port_pool = set(obj.tcpudp_port_pool) return obj + def SimpleFillDP(self, diskparams): + """Fill a given diskparams dict with cluster defaults. + + @param diskparams: The diskparams + @return: The defaults dict + + """ + return FillDiskParams(self.diskparams, diskparams) + def GetHVDefaults(self, hypervisor, os_name=None, skip_keys=None): """Get the default hypervisor parameters for the cluster. @@ -1310,6 +1758,20 @@ class Cluster(TaggableObject): # specified params return FillDict(result, os_params) + @staticmethod + def SimpleFillHvState(hv_state): + """Fill an hv_state sub dict with cluster defaults. + + """ + return FillDict(constants.HVST_DEFAULTS, hv_state) + + @staticmethod + def SimpleFillDiskState(disk_state): + """Fill an disk_state sub dict with cluster defaults. + + """ + return FillDict(constants.DS_DEFAULTS, disk_state) + def FillND(self, node, nodegroup): """Return filled out ndparams for L{objects.NodeGroup} and L{objects.Node} @@ -1334,6 +1796,18 @@ class Cluster(TaggableObject): """ return FillDict(self.ndparams, ndparams) + def SimpleFillIPolicy(self, ipolicy): + """ Fill instance policy dict with defaults. + + @type ipolicy: dict + @param ipolicy: the dict to fill + @rtype: dict + @return: a copy of passed ipolicy with missing keys filled from + the cluster defaults + + """ + return FillIPolicy(self.ipolicy, ipolicy) + class BlockDevStatus(ConfigObject): """Config object representing the status of a block device.""" @@ -1458,17 +1932,6 @@ class _QueryResponseBase(ConfigObject): return obj -class QueryRequest(ConfigObject): - """Object holding a query request. - - """ - __slots__ = [ - "what", - "fields", - "filter", - ] - - class QueryResponse(_QueryResponseBase): """Object holding the response to a query. @@ -1501,6 +1964,17 @@ class QueryFieldsResponse(_QueryResponseBase): ] +class MigrationStatus(ConfigObject): + """Object holding the status of a migration. + + """ + __slots__ = [ + "status", + "transferred_ram", + "total_ram", + ] + + class InstanceConsole(ConfigObject): """Object describing how to access the console of an instance. @@ -1522,15 +1996,20 @@ class InstanceConsole(ConfigObject): """ assert self.kind in constants.CONS_ALL, "Unknown console type" assert self.instance, "Missing instance name" - assert self.message or self.kind in [constants.CONS_SSH, constants.CONS_VNC] + assert self.message or self.kind in [constants.CONS_SSH, + constants.CONS_SPICE, + constants.CONS_VNC] assert self.host or self.kind == constants.CONS_MESSAGE assert self.port or self.kind in [constants.CONS_MESSAGE, constants.CONS_SSH] assert self.user or self.kind in [constants.CONS_MESSAGE, + constants.CONS_SPICE, constants.CONS_VNC] assert self.command or self.kind in [constants.CONS_MESSAGE, + constants.CONS_SPICE, constants.CONS_VNC] assert self.display or self.kind in [constants.CONS_MESSAGE, + constants.CONS_SPICE, constants.CONS_SSH] return True