4 # Copyright (C) 2006, 2007 Google Inc.
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 2 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful, but
12 # WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 # General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
22 """Transportable objects for Ganeti.
24 This module provides small, mostly data-only objects which are safe to
25 pass to and from external parties.
33 from cStringIO import StringIO
35 from ganeti import errors
36 from ganeti import constants
39 __all__ = ["ConfigObject", "ConfigData", "NIC", "Disk", "Instance",
40 "OS", "Node", "Cluster", "FillDict"]
43 def FillDict(defaults_dict, custom_dict):
44 """Basic function to apply settings on top a default dict.
46 @type defaults_dict: dict
47 @param defaults_dict: dictionary holding the default values
48 @type custom_dict: dict
49 @param custom_dict: dictionary holding customized value
51 @return: dict with the 'full' values
54 ret_dict = copy.deepcopy(defaults_dict)
55 ret_dict.update(custom_dict)
59 def UpgradeGroupedParams(target, defaults):
60 """Update all groups for the target parameter.
62 @type target: dict of dicts
63 @param target: {group: {parameter: value}}
65 @param defaults: default parameter values
69 target = {constants.PP_DEFAULT: defaults}
72 target[group] = FillDict(defaults, target[group])
76 class ConfigObject(object):
77 """A generic config object.
79 It has the following properties:
81 - provides somewhat safe recursive unpickling and pickling for its classes
82 - unset attributes which are defined in slots are always returned
83 as None instead of raising an error
85 Classes derived from this must always declare __slots__ (we use many
86 config objects and the memory reduction is useful)
91 def __init__(self, **kwargs):
92 for k, v in kwargs.iteritems():
96 def __getattr__(self, name):
97 if name not in self.__slots__:
98 raise AttributeError("Invalid object attribute %s.%s" %
99 (type(self).__name__, name))
102 def __setitem__(self, key, value):
103 if key not in self.__slots__:
105 setattr(self, key, value)
107 def __setstate__(self, state):
109 if name in self.__slots__:
110 setattr(self, name, state[name])
113 """Convert to a dict holding only standard python types.
115 The generic routine just dumps all of this object's attributes in
116 a dict. It does not work if the class has children who are
117 ConfigObjects themselves (e.g. the nics list in an Instance), in
118 which case the object should subclass the function in order to
119 make sure all objects returned are only standard python types.
123 for name in self.__slots__:
124 value = getattr(self, name, None)
125 if value is not None:
129 __getstate__ = ToDict
132 def FromDict(cls, val):
133 """Create an object from a dictionary.
135 This generic routine takes a dict, instantiates a new instance of
136 the given class, and sets attributes based on the dict content.
138 As for `ToDict`, this does not work if the class has children
139 who are ConfigObjects themselves (e.g. the nics list in an
140 Instance), in which case the object should subclass the function
141 and alter the objects.
144 if not isinstance(val, dict):
145 raise errors.ConfigurationError("Invalid object passed to FromDict:"
146 " expected dict, got %s" % type(val))
147 val_str = dict([(str(k), v) for k, v in val.iteritems()])
152 def _ContainerToDicts(container):
153 """Convert the elements of a container to standard python types.
155 This method converts a container with elements derived from
156 ConfigData to standard python types. If the container is a dict,
157 we don't touch the keys, only the values.
160 if isinstance(container, dict):
161 ret = dict([(k, v.ToDict()) for k, v in container.iteritems()])
162 elif isinstance(container, (list, tuple, set, frozenset)):
163 ret = [elem.ToDict() for elem in container]
165 raise TypeError("Invalid type %s passed to _ContainerToDicts" %
170 def _ContainerFromDicts(source, c_type, e_type):
171 """Convert a container from standard python types.
173 This method converts a container with standard python types to
174 ConfigData objects. If the container is a dict, we don't touch the
175 keys, only the values.
178 if not isinstance(c_type, type):
179 raise TypeError("Container type %s passed to _ContainerFromDicts is"
180 " not a type" % type(c_type))
182 ret = dict([(k, e_type.FromDict(v)) for k, v in source.iteritems()])
183 elif c_type in (list, tuple, set, frozenset):
184 ret = c_type([e_type.FromDict(elem) for elem in source])
186 raise TypeError("Invalid container type %s passed to"
187 " _ContainerFromDicts" % c_type)
191 """Implement __repr__ for ConfigObjects."""
192 return repr(self.ToDict())
194 def UpgradeConfig(self):
195 """Fill defaults for missing configuration values.
197 This method will be called at object init time, and its implementation will
204 class TaggableObject(ConfigObject):
205 """An generic class supporting tags.
208 __slots__ = ConfigObject.__slots__ + ["tags"]
211 def ValidateTag(tag):
212 """Check if a tag is valid.
214 If the tag is invalid, an errors.TagError will be raised. The
215 function has no return value.
218 if not isinstance(tag, basestring):
219 raise errors.TagError("Invalid tag type (not a string)")
220 if len(tag) > constants.MAX_TAG_LEN:
221 raise errors.TagError("Tag too long (>%d characters)" %
222 constants.MAX_TAG_LEN)
224 raise errors.TagError("Tags cannot be empty")
225 if not re.match("^[\w.+*/:-]+$", tag):
226 raise errors.TagError("Tag contains invalid characters")
229 """Return the tags list.
232 tags = getattr(self, "tags", None)
234 tags = self.tags = set()
237 def AddTag(self, tag):
241 self.ValidateTag(tag)
242 tags = self.GetTags()
243 if len(tags) >= constants.MAX_TAGS_PER_OBJ:
244 raise errors.TagError("Too many tags")
245 self.GetTags().add(tag)
247 def RemoveTag(self, tag):
251 self.ValidateTag(tag)
252 tags = self.GetTags()
256 raise errors.TagError("Tag not found")
259 """Taggable-object-specific conversion to standard python types.
261 This replaces the tags set with a list.
264 bo = super(TaggableObject, self).ToDict()
266 tags = bo.get("tags", None)
267 if isinstance(tags, set):
268 bo["tags"] = list(tags)
272 def FromDict(cls, val):
273 """Custom function for instances.
276 obj = super(TaggableObject, cls).FromDict(val)
277 if hasattr(obj, "tags") and isinstance(obj.tags, list):
278 obj.tags = set(obj.tags)
282 class ConfigData(ConfigObject):
283 """Top-level config object."""
284 __slots__ = ["version", "cluster", "nodes", "instances", "serial_no"]
287 """Custom function for top-level config data.
289 This just replaces the list of instances, nodes and the cluster
290 with standard python types.
293 mydict = super(ConfigData, self).ToDict()
294 mydict["cluster"] = mydict["cluster"].ToDict()
295 for key in "nodes", "instances":
296 mydict[key] = self._ContainerToDicts(mydict[key])
301 def FromDict(cls, val):
302 """Custom function for top-level config data
305 obj = super(ConfigData, cls).FromDict(val)
306 obj.cluster = Cluster.FromDict(obj.cluster)
307 obj.nodes = cls._ContainerFromDicts(obj.nodes, dict, Node)
308 obj.instances = cls._ContainerFromDicts(obj.instances, dict, Instance)
312 class NIC(ConfigObject):
313 """Config object representing a network card."""
314 __slots__ = ["mac", "ip", "bridge", "nicparams"]
317 def CheckParameterSyntax(cls, nicparams):
318 """Check the given parameters for validity.
320 @type nicparams: dict
321 @param nicparams: dictionary with parameter names/value
322 @raise errors.ConfigurationError: when a parameter is not valid
325 if nicparams[constants.NIC_MODE] not in constants.NIC_VALID_MODES:
326 err = "Invalid nic mode: %s" % nicparams[constants.NIC_MODE]
327 raise errors.ConfigurationError(err)
329 if (nicparams[constants.NIC_MODE] is constants.NIC_MODE_BRIDGED and
330 not nicparams[constants.NIC_LINK]):
331 err = "Missing bridged nic link"
332 raise errors.ConfigurationError(err)
334 def UpgradeConfig(self):
335 """Fill defaults for missing configuration values.
338 if self.nicparams is None:
340 if self.bridge is not None:
341 self.nicparams[constants.NIC_MODE] = constants.NIC_MODE_BRIDGED
342 self.nicparams[constants.NIC_LINK] = self.bridge
343 # bridge is no longer used it 2.1. The slot is left there to support
344 # upgrading, but will be removed in 2.2
345 if self.bridge is not None:
349 class Disk(ConfigObject):
350 """Config object representing a block device."""
351 __slots__ = ["dev_type", "logical_id", "physical_id",
352 "children", "iv_name", "size", "mode"]
354 def CreateOnSecondary(self):
355 """Test if this device needs to be created on a secondary node."""
356 return self.dev_type in (constants.LD_DRBD8, constants.LD_LV)
358 def AssembleOnSecondary(self):
359 """Test if this device needs to be assembled on a secondary node."""
360 return self.dev_type in (constants.LD_DRBD8, constants.LD_LV)
362 def OpenOnSecondary(self):
363 """Test if this device needs to be opened on a secondary node."""
364 return self.dev_type in (constants.LD_LV,)
366 def StaticDevPath(self):
367 """Return the device path if this device type has a static one.
369 Some devices (LVM for example) live always at the same /dev/ path,
370 irrespective of their status. For such devices, we return this
371 path, for others we return None.
374 if self.dev_type == constants.LD_LV:
375 return "/dev/%s/%s" % (self.logical_id[0], self.logical_id[1])
378 def ChildrenNeeded(self):
379 """Compute the needed number of children for activation.
381 This method will return either -1 (all children) or a positive
382 number denoting the minimum number of children needed for
383 activation (only mirrored devices will usually return >=0).
385 Currently, only DRBD8 supports diskless activation (therefore we
386 return 0), for all other we keep the previous semantics and return
390 if self.dev_type == constants.LD_DRBD8:
394 def GetNodes(self, node):
395 """This function returns the nodes this device lives on.
397 Given the node on which the parent of the device lives on (or, in
398 case of a top-level device, the primary node of the devices'
399 instance), this function will return a list of nodes on which this
400 devices needs to (or can) be assembled.
403 if self.dev_type in [constants.LD_LV, constants.LD_FILE]:
405 elif self.dev_type in constants.LDS_DRBD:
406 result = [self.logical_id[0], self.logical_id[1]]
407 if node not in result:
408 raise errors.ConfigurationError("DRBD device passed unknown node")
410 raise errors.ProgrammerError("Unhandled device type %s" % self.dev_type)
413 def ComputeNodeTree(self, parent_node):
414 """Compute the node/disk tree for this disk and its children.
416 This method, given the node on which the parent disk lives, will
417 return the list of all (node, disk) pairs which describe the disk
418 tree in the most compact way. For example, a drbd/lvm stack
419 will be returned as (primary_node, drbd) and (secondary_node, drbd)
420 which represents all the top-level devices on the nodes.
423 my_nodes = self.GetNodes(parent_node)
424 result = [(node, self) for node in my_nodes]
425 if not self.children:
428 for node in my_nodes:
429 for child in self.children:
430 child_result = child.ComputeNodeTree(node)
431 if len(child_result) == 1:
432 # child (and all its descendants) is simple, doesn't split
433 # over multiple hosts, so we don't need to describe it, our
434 # own entry for this node describes it completely
437 # check if child nodes differ from my nodes; note that
438 # subdisk can differ from the child itself, and be instead
439 # one of its descendants
440 for subnode, subdisk in child_result:
441 if subnode not in my_nodes:
442 result.append((subnode, subdisk))
443 # otherwise child is under our own node, so we ignore this
444 # entry (but probably the other results in the list will
448 def RecordGrow(self, amount):
449 """Update the size of this disk after growth.
451 This method recurses over the disks's children and updates their
452 size correspondigly. The method needs to be kept in sync with the
453 actual algorithms from bdev.
456 if self.dev_type == constants.LD_LV:
458 elif self.dev_type == constants.LD_DRBD8:
460 self.children[0].RecordGrow(amount)
463 raise errors.ProgrammerError("Disk.RecordGrow called for unsupported"
464 " disk type %s" % self.dev_type)
466 def SetPhysicalID(self, target_node, nodes_ip):
467 """Convert the logical ID to the physical ID.
469 This is used only for drbd, which needs ip/port configuration.
471 The routine descends down and updates its children also, because
472 this helps when the only the top device is passed to the remote
476 - target_node: the node we wish to configure for
477 - nodes_ip: a mapping of node name to ip
479 The target_node must exist in in nodes_ip, and must be one of the
480 nodes in the logical ID for each of the DRBD devices encountered
485 for child in self.children:
486 child.SetPhysicalID(target_node, nodes_ip)
488 if self.logical_id is None and self.physical_id is not None:
490 if self.dev_type in constants.LDS_DRBD:
491 pnode, snode, port, pminor, sminor, secret = self.logical_id
492 if target_node not in (pnode, snode):
493 raise errors.ConfigurationError("DRBD device not knowing node %s" %
495 pnode_ip = nodes_ip.get(pnode, None)
496 snode_ip = nodes_ip.get(snode, None)
497 if pnode_ip is None or snode_ip is None:
498 raise errors.ConfigurationError("Can't find primary or secondary node"
499 " for %s" % str(self))
500 p_data = (pnode_ip, port)
501 s_data = (snode_ip, port)
502 if pnode == target_node:
503 self.physical_id = p_data + s_data + (pminor, secret)
504 else: # it must be secondary, we tested above
505 self.physical_id = s_data + p_data + (sminor, secret)
507 self.physical_id = self.logical_id
511 """Disk-specific conversion to standard python types.
513 This replaces the children lists of objects with lists of
514 standard python types.
517 bo = super(Disk, self).ToDict()
519 for attr in ("children",):
520 alist = bo.get(attr, None)
522 bo[attr] = self._ContainerToDicts(alist)
526 def FromDict(cls, val):
527 """Custom function for Disks
530 obj = super(Disk, cls).FromDict(val)
532 obj.children = cls._ContainerFromDicts(obj.children, list, Disk)
533 if obj.logical_id and isinstance(obj.logical_id, list):
534 obj.logical_id = tuple(obj.logical_id)
535 if obj.physical_id and isinstance(obj.physical_id, list):
536 obj.physical_id = tuple(obj.physical_id)
537 if obj.dev_type in constants.LDS_DRBD:
538 # we need a tuple of length six here
539 if len(obj.logical_id) < 6:
540 obj.logical_id += (None,) * (6 - len(obj.logical_id))
544 """Custom str() formatter for disks.
547 if self.dev_type == constants.LD_LV:
548 val = "<LogicalVolume(/dev/%s/%s" % self.logical_id
549 elif self.dev_type in constants.LDS_DRBD:
550 node_a, node_b, port, minor_a, minor_b = self.logical_id[:5]
552 if self.physical_id is None:
555 phy = ("configured as %s:%s %s:%s" %
556 (self.physical_id[0], self.physical_id[1],
557 self.physical_id[2], self.physical_id[3]))
559 val += ("hosts=%s/%d-%s/%d, port=%s, %s, " %
560 (node_a, minor_a, node_b, minor_b, port, phy))
561 if self.children and self.children.count(None) == 0:
562 val += "backend=%s, metadev=%s" % (self.children[0], self.children[1])
564 val += "no local storage"
566 val = ("<Disk(type=%s, logical_id=%s, physical_id=%s, children=%s" %
567 (self.dev_type, self.logical_id, self.physical_id, self.children))
568 if self.iv_name is None:
569 val += ", not visible"
571 val += ", visible as /dev/%s" % self.iv_name
572 if isinstance(self.size, int):
573 val += ", size=%dm)>" % self.size
575 val += ", size='%s')>" % (self.size,)
579 """Checks that this disk is correctly configured.
583 if self.mode not in constants.DISK_ACCESS_SET:
584 all_errors.append("Disk access mode '%s' is invalid" % (self.mode, ))
588 class Instance(TaggableObject):
589 """Config object representing an instance."""
590 __slots__ = TaggableObject.__slots__ + [
605 def _ComputeSecondaryNodes(self):
606 """Compute the list of secondary nodes.
608 This is a simple wrapper over _ComputeAllNodes.
611 all_nodes = set(self._ComputeAllNodes())
612 all_nodes.discard(self.primary_node)
613 return tuple(all_nodes)
615 secondary_nodes = property(_ComputeSecondaryNodes, None, None,
616 "List of secondary nodes")
618 def _ComputeAllNodes(self):
619 """Compute the list of all nodes.
621 Since the data is already there (in the drbd disks), keeping it as
622 a separate normal attribute is redundant and if not properly
623 synchronised can cause problems. Thus it's better to compute it
627 def _Helper(nodes, device):
628 """Recursively computes nodes given a top device."""
629 if device.dev_type in constants.LDS_DRBD:
630 nodea, nodeb = device.logical_id[:2]
634 for child in device.children:
635 _Helper(nodes, child)
638 all_nodes.add(self.primary_node)
639 for device in self.disks:
640 _Helper(all_nodes, device)
641 return tuple(all_nodes)
643 all_nodes = property(_ComputeAllNodes, None, None,
644 "List of all nodes of the instance")
646 def MapLVsByNode(self, lvmap=None, devs=None, node=None):
647 """Provide a mapping of nodes to LVs this instance owns.
649 This function figures out what logical volumes should belong on
650 which nodes, recursing through a device tree.
652 @param lvmap: optional dictionary to receive the
653 'node' : ['lv', ...] data.
655 @return: None if lvmap arg is given, otherwise, a dictionary
656 of the form { 'nodename' : ['volume1', 'volume2', ...], ... }
660 node = self.primary_node
663 lvmap = { node : [] }
666 if not node in lvmap:
674 if dev.dev_type == constants.LD_LV:
675 lvmap[node].append(dev.logical_id[1])
677 elif dev.dev_type in constants.LDS_DRBD:
679 self.MapLVsByNode(lvmap, dev.children, dev.logical_id[0])
680 self.MapLVsByNode(lvmap, dev.children, dev.logical_id[1])
683 self.MapLVsByNode(lvmap, dev.children, node)
687 def FindDisk(self, idx):
688 """Find a disk given having a specified index.
690 This is just a wrapper that does validation of the index.
693 @param idx: the disk index
695 @return: the corresponding disk
696 @raise errors.OpPrereqError: when the given index is not valid
701 return self.disks[idx]
702 except ValueError, err:
703 raise errors.OpPrereqError("Invalid disk index: '%s'" % str(err))
705 raise errors.OpPrereqError("Invalid disk index: %d (instace has disks"
706 " 0 to %d" % (idx, len(self.disks)))
709 """Instance-specific conversion to standard python types.
711 This replaces the children lists of objects with lists of standard
715 bo = super(Instance, self).ToDict()
717 for attr in "nics", "disks":
718 alist = bo.get(attr, None)
720 nlist = self._ContainerToDicts(alist)
727 def FromDict(cls, val):
728 """Custom function for instances.
731 obj = super(Instance, cls).FromDict(val)
732 obj.nics = cls._ContainerFromDicts(obj.nics, list, NIC)
733 obj.disks = cls._ContainerFromDicts(obj.disks, list, Disk)
737 class OS(ConfigObject):
738 """Config object representing an operating system."""
750 class Node(TaggableObject):
751 """Config object representing a node."""
752 __slots__ = TaggableObject.__slots__ + [
763 class Cluster(TaggableObject):
764 """Config object representing the cluster."""
765 __slots__ = TaggableObject.__slots__ + [
773 "default_hypervisor",
779 "enabled_hypervisors",
783 "candidate_pool_size",
787 def UpgradeConfig(self):
788 """Fill defaults for missing configuration values.
791 if self.hvparams is None:
792 self.hvparams = constants.HVC_DEFAULTS
794 for hypervisor in self.hvparams:
795 self.hvparams[hypervisor] = FillDict(
796 constants.HVC_DEFAULTS[hypervisor], self.hvparams[hypervisor])
798 self.beparams = UpgradeGroupedParams(self.beparams,
799 constants.BEC_DEFAULTS)
800 migrate_default_bridge = not self.nicparams
801 self.nicparams = UpgradeGroupedParams(self.nicparams,
802 constants.NICC_DEFAULTS)
803 if migrate_default_bridge:
804 self.nicparams[constants.PP_DEFAULT][constants.NIC_LINK] = \
807 if self.modify_etc_hosts is None:
808 self.modify_etc_hosts = True
810 # default_bridge is no longer used it 2.1. The slot is left there to
811 # support auto-upgrading, but will be removed in 2.2
812 if self.default_bridge is not None:
813 self.default_bridge = None
815 # default_hypervisor is just the first enabled one in 2.1
816 if self.default_hypervisor is not None:
817 self.enabled_hypervisors = [self.default_hypervisor] + \
818 [hvname for hvname in self.enabled_hypervisors
819 if hvname != self.default_hypervisor]
820 self.default_hypervisor = None
824 """Custom function for cluster.
827 mydict = super(Cluster, self).ToDict()
828 mydict["tcpudp_port_pool"] = list(self.tcpudp_port_pool)
832 def FromDict(cls, val):
833 """Custom function for cluster.
836 obj = super(Cluster, cls).FromDict(val)
837 if not isinstance(obj.tcpudp_port_pool, set):
838 obj.tcpudp_port_pool = set(obj.tcpudp_port_pool)
841 def FillHV(self, instance):
842 """Fill an instance's hvparams dict.
844 @type instance: L{objects.Instance}
845 @param instance: the instance parameter to fill
847 @return: a copy of the instance's hvparams with missing keys filled from
851 return FillDict(self.hvparams.get(instance.hypervisor, {}),
854 def FillBE(self, instance):
855 """Fill an instance's beparams dict.
857 @type instance: L{objects.Instance}
858 @param instance: the instance parameter to fill
860 @return: a copy of the instance's beparams with missing keys filled from
864 return FillDict(self.beparams.get(constants.PP_DEFAULT, {}),
868 class BlockDevStatus(ConfigObject):
869 """Config object representing the status of a block device."""
881 class SerializableConfigParser(ConfigParser.SafeConfigParser):
882 """Simple wrapper over ConfigParse that allows serialization.
884 This class is basically ConfigParser.SafeConfigParser with two
885 additional methods that allow it to serialize/unserialize to/from a
890 """Dump this instance and return the string representation."""
893 return buf.getvalue()
897 """Load data from a string."""
899 cfp = SerializableConfigParser()