4 # Copyright (C) 2006, 2007 Google Inc.
6 # This program is free software; you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation; either version 2 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful, but
12 # WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
14 # General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program; if not, write to the Free Software
18 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
22 """Transportable objects for Ganeti.
24 This module provides small, mostly data-only objects which are safe to
25 pass to and from external parties.
33 from cStringIO import StringIO
35 from ganeti import errors
36 from ganeti import constants
39 __all__ = ["ConfigObject", "ConfigData", "NIC", "Disk", "Instance",
40 "OS", "Node", "Cluster"]
43 class ConfigObject(object):
44 """A generic config object.
46 It has the following properties:
48 - provides somewhat safe recursive unpickling and pickling for its classes
49 - unset attributes which are defined in slots are always returned
50 as None instead of raising an error
52 Classes derived from this must always declare __slots__ (we use many
53 config objects and the memory reduction is useful)
58 def __init__(self, **kwargs):
59 for k, v in kwargs.iteritems():
62 def __getattr__(self, name):
63 if name not in self.__slots__:
64 raise AttributeError("Invalid object attribute %s.%s" %
65 (type(self).__name__, name))
68 def __setitem__(self, key, value):
69 if key not in self.__slots__:
71 setattr(self, key, value)
73 def __getstate__(self):
75 for name in self.__slots__:
76 if hasattr(self, name):
77 state[name] = getattr(self, name)
80 def __setstate__(self, state):
82 if name in self.__slots__:
83 setattr(self, name, state[name])
86 """Convert to a dict holding only standard python types.
88 The generic routine just dumps all of this object's attributes in
89 a dict. It does not work if the class has children who are
90 ConfigObjects themselves (e.g. the nics list in an Instance), in
91 which case the object should subclass the function in order to
92 make sure all objects returned are only standard python types.
95 return dict([(k, getattr(self, k, None)) for k in self.__slots__])
98 def FromDict(cls, val):
99 """Create an object from a dictionary.
101 This generic routine takes a dict, instantiates a new instance of
102 the given class, and sets attributes based on the dict content.
104 As for `ToDict`, this does not work if the class has children
105 who are ConfigObjects themselves (e.g. the nics list in an
106 Instance), in which case the object should subclass the function
107 and alter the objects.
110 if not isinstance(val, dict):
111 raise errors.ConfigurationError("Invalid object passed to FromDict:"
112 " expected dict, got %s" % type(val))
113 val_str = dict([(str(k), v) for k, v in val.iteritems()])
118 def _ContainerToDicts(container):
119 """Convert the elements of a container to standard python types.
121 This method converts a container with elements derived from
122 ConfigData to standard python types. If the container is a dict,
123 we don't touch the keys, only the values.
126 if isinstance(container, dict):
127 ret = dict([(k, v.ToDict()) for k, v in container.iteritems()])
128 elif isinstance(container, (list, tuple, set, frozenset)):
129 ret = [elem.ToDict() for elem in container]
131 raise TypeError("Invalid type %s passed to _ContainerToDicts" %
136 def _ContainerFromDicts(source, c_type, e_type):
137 """Convert a container from standard python types.
139 This method converts a container with standard python types to
140 ConfigData objects. If the container is a dict, we don't touch the
141 keys, only the values.
144 if not isinstance(c_type, type):
145 raise TypeError("Container type %s passed to _ContainerFromDicts is"
146 " not a type" % type(c_type))
148 ret = dict([(k, e_type.FromDict(v)) for k, v in source.iteritems()])
149 elif c_type in (list, tuple, set, frozenset):
150 ret = c_type([e_type.FromDict(elem) for elem in source])
152 raise TypeError("Invalid container type %s passed to"
153 " _ContainerFromDicts" % c_type)
157 """Makes a deep copy of the current object and its children.
160 dict_form = self.ToDict()
161 clone_obj = self.__class__.FromDict(dict_form)
165 """Implement __repr__ for ConfigObjects."""
166 return repr(self.ToDict())
168 def UpgradeConfig(self):
169 """Fill defaults for missing configuration values.
171 This method will be called at configuration load time, and its
172 implementation will be object dependent.
178 class TaggableObject(ConfigObject):
179 """An generic class supporting tags.
182 __slots__ = ConfigObject.__slots__ + ["tags"]
185 def ValidateTag(tag):
186 """Check if a tag is valid.
188 If the tag is invalid, an errors.TagError will be raised. The
189 function has no return value.
192 if not isinstance(tag, basestring):
193 raise errors.TagError("Invalid tag type (not a string)")
194 if len(tag) > constants.MAX_TAG_LEN:
195 raise errors.TagError("Tag too long (>%d characters)" %
196 constants.MAX_TAG_LEN)
198 raise errors.TagError("Tags cannot be empty")
199 if not re.match("^[\w.+*/:-]+$", tag):
200 raise errors.TagError("Tag contains invalid characters")
203 """Return the tags list.
206 tags = getattr(self, "tags", None)
208 tags = self.tags = set()
211 def AddTag(self, tag):
215 self.ValidateTag(tag)
216 tags = self.GetTags()
217 if len(tags) >= constants.MAX_TAGS_PER_OBJ:
218 raise errors.TagError("Too many tags")
219 self.GetTags().add(tag)
221 def RemoveTag(self, tag):
225 self.ValidateTag(tag)
226 tags = self.GetTags()
230 raise errors.TagError("Tag not found")
233 """Taggable-object-specific conversion to standard python types.
235 This replaces the tags set with a list.
238 bo = super(TaggableObject, self).ToDict()
240 tags = bo.get("tags", None)
241 if isinstance(tags, set):
242 bo["tags"] = list(tags)
246 def FromDict(cls, val):
247 """Custom function for instances.
250 obj = super(TaggableObject, cls).FromDict(val)
251 if hasattr(obj, "tags") and isinstance(obj.tags, list):
252 obj.tags = set(obj.tags)
256 class ConfigData(ConfigObject):
257 """Top-level config object."""
258 __slots__ = ["version", "cluster", "nodes", "instances", "serial_no"]
261 """Custom function for top-level config data.
263 This just replaces the list of instances, nodes and the cluster
264 with standard python types.
267 mydict = super(ConfigData, self).ToDict()
268 mydict["cluster"] = mydict["cluster"].ToDict()
269 for key in "nodes", "instances":
270 mydict[key] = self._ContainerToDicts(mydict[key])
275 def FromDict(cls, val):
276 """Custom function for top-level config data
279 obj = super(ConfigData, cls).FromDict(val)
280 obj.cluster = Cluster.FromDict(obj.cluster)
281 obj.nodes = cls._ContainerFromDicts(obj.nodes, dict, Node)
282 obj.instances = cls._ContainerFromDicts(obj.instances, dict, Instance)
285 def UpgradeConfig(self):
286 """Fill defaults for missing configuration values.
289 self.cluster.UpgradeConfig()
290 for node in self.nodes.values():
292 for instance in self.instances.values():
293 instance.UpgradeConfig()
296 class NIC(ConfigObject):
297 """Config object representing a network card."""
298 __slots__ = ["mac", "ip", "bridge"]
301 class Disk(ConfigObject):
302 """Config object representing a block device."""
303 __slots__ = ["dev_type", "logical_id", "physical_id",
304 "children", "iv_name", "size", "mode"]
306 def CreateOnSecondary(self):
307 """Test if this device needs to be created on a secondary node."""
308 return self.dev_type in (constants.LD_DRBD8, constants.LD_LV)
310 def AssembleOnSecondary(self):
311 """Test if this device needs to be assembled on a secondary node."""
312 return self.dev_type in (constants.LD_DRBD8, constants.LD_LV)
314 def OpenOnSecondary(self):
315 """Test if this device needs to be opened on a secondary node."""
316 return self.dev_type in (constants.LD_LV,)
318 def StaticDevPath(self):
319 """Return the device path if this device type has a static one.
321 Some devices (LVM for example) live always at the same /dev/ path,
322 irrespective of their status. For such devices, we return this
323 path, for others we return None.
326 if self.dev_type == constants.LD_LV:
327 return "/dev/%s/%s" % (self.logical_id[0], self.logical_id[1])
330 def ChildrenNeeded(self):
331 """Compute the needed number of children for activation.
333 This method will return either -1 (all children) or a positive
334 number denoting the minimum number of children needed for
335 activation (only mirrored devices will usually return >=0).
337 Currently, only DRBD8 supports diskless activation (therefore we
338 return 0), for all other we keep the previous semantics and return
342 if self.dev_type == constants.LD_DRBD8:
346 def GetNodes(self, node):
347 """This function returns the nodes this device lives on.
349 Given the node on which the parent of the device lives on (or, in
350 case of a top-level device, the primary node of the devices'
351 instance), this function will return a list of nodes on which this
352 devices needs to (or can) be assembled.
355 if self.dev_type in [constants.LD_LV, constants.LD_FILE]:
357 elif self.dev_type in constants.LDS_DRBD:
358 result = [self.logical_id[0], self.logical_id[1]]
359 if node not in result:
360 raise errors.ConfigurationError("DRBD device passed unknown node")
362 raise errors.ProgrammerError("Unhandled device type %s" % self.dev_type)
365 def ComputeNodeTree(self, parent_node):
366 """Compute the node/disk tree for this disk and its children.
368 This method, given the node on which the parent disk lives, will
369 return the list of all (node, disk) pairs which describe the disk
370 tree in the most compact way. For example, a drbd/lvm stack
371 will be returned as (primary_node, drbd) and (secondary_node, drbd)
372 which represents all the top-level devices on the nodes.
375 my_nodes = self.GetNodes(parent_node)
376 result = [(node, self) for node in my_nodes]
377 if not self.children:
380 for node in my_nodes:
381 for child in self.children:
382 child_result = child.ComputeNodeTree(node)
383 if len(child_result) == 1:
384 # child (and all its descendants) is simple, doesn't split
385 # over multiple hosts, so we don't need to describe it, our
386 # own entry for this node describes it completely
389 # check if child nodes differ from my nodes; note that
390 # subdisk can differ from the child itself, and be instead
391 # one of its descendants
392 for subnode, subdisk in child_result:
393 if subnode not in my_nodes:
394 result.append((subnode, subdisk))
395 # otherwise child is under our own node, so we ignore this
396 # entry (but probably the other results in the list will
400 def RecordGrow(self, amount):
401 """Update the size of this disk after growth.
403 This method recurses over the disks's children and updates their
404 size correspondigly. The method needs to be kept in sync with the
405 actual algorithms from bdev.
408 if self.dev_type == constants.LD_LV:
410 elif self.dev_type == constants.LD_DRBD8:
412 self.children[0].RecordGrow(amount)
415 raise errors.ProgrammerError("Disk.RecordGrow called for unsupported"
416 " disk type %s" % self.dev_type)
419 """Sets recursively the size to zero for the disk and its children.
423 for child in self.children:
427 def SetPhysicalID(self, target_node, nodes_ip):
428 """Convert the logical ID to the physical ID.
430 This is used only for drbd, which needs ip/port configuration.
432 The routine descends down and updates its children also, because
433 this helps when the only the top device is passed to the remote
437 - target_node: the node we wish to configure for
438 - nodes_ip: a mapping of node name to ip
440 The target_node must exist in in nodes_ip, and must be one of the
441 nodes in the logical ID for each of the DRBD devices encountered
446 for child in self.children:
447 child.SetPhysicalID(target_node, nodes_ip)
449 if self.logical_id is None and self.physical_id is not None:
451 if self.dev_type in constants.LDS_DRBD:
452 pnode, snode, port, pminor, sminor, secret = self.logical_id
453 if target_node not in (pnode, snode):
454 raise errors.ConfigurationError("DRBD device not knowing node %s" %
456 pnode_ip = nodes_ip.get(pnode, None)
457 snode_ip = nodes_ip.get(snode, None)
458 if pnode_ip is None or snode_ip is None:
459 raise errors.ConfigurationError("Can't find primary or secondary node"
460 " for %s" % str(self))
461 p_data = (pnode_ip, port)
462 s_data = (snode_ip, port)
463 if pnode == target_node:
464 self.physical_id = p_data + s_data + (pminor, secret)
465 else: # it must be secondary, we tested above
466 self.physical_id = s_data + p_data + (sminor, secret)
468 self.physical_id = self.logical_id
472 """Disk-specific conversion to standard python types.
474 This replaces the children lists of objects with lists of
475 standard python types.
478 bo = super(Disk, self).ToDict()
480 for attr in ("children",):
481 alist = bo.get(attr, None)
483 bo[attr] = self._ContainerToDicts(alist)
487 def FromDict(cls, val):
488 """Custom function for Disks
491 obj = super(Disk, cls).FromDict(val)
493 obj.children = cls._ContainerFromDicts(obj.children, list, Disk)
494 if obj.logical_id and isinstance(obj.logical_id, list):
495 obj.logical_id = tuple(obj.logical_id)
496 if obj.physical_id and isinstance(obj.physical_id, list):
497 obj.physical_id = tuple(obj.physical_id)
498 if obj.dev_type in constants.LDS_DRBD:
499 # we need a tuple of length six here
500 if len(obj.logical_id) < 6:
501 obj.logical_id += (None,) * (6 - len(obj.logical_id))
505 """Custom str() formatter for disks.
508 if self.dev_type == constants.LD_LV:
509 val = "<LogicalVolume(/dev/%s/%s" % self.logical_id
510 elif self.dev_type in constants.LDS_DRBD:
511 node_a, node_b, port, minor_a, minor_b = self.logical_id[:5]
513 if self.physical_id is None:
516 phy = ("configured as %s:%s %s:%s" %
517 (self.physical_id[0], self.physical_id[1],
518 self.physical_id[2], self.physical_id[3]))
520 val += ("hosts=%s/%d-%s/%d, port=%s, %s, " %
521 (node_a, minor_a, node_b, minor_b, port, phy))
522 if self.children and self.children.count(None) == 0:
523 val += "backend=%s, metadev=%s" % (self.children[0], self.children[1])
525 val += "no local storage"
527 val = ("<Disk(type=%s, logical_id=%s, physical_id=%s, children=%s" %
528 (self.dev_type, self.logical_id, self.physical_id, self.children))
529 if self.iv_name is None:
530 val += ", not visible"
532 val += ", visible as /dev/%s" % self.iv_name
533 if isinstance(self.size, int):
534 val += ", size=%dm)>" % self.size
536 val += ", size='%s')>" % (self.size,)
540 """Checks that this disk is correctly configured.
544 if self.mode not in constants.DISK_ACCESS_SET:
545 all_errors.append("Disk access mode '%s' is invalid" % (self.mode, ))
548 def UpgradeConfig(self):
549 """Fill defaults for missing configuration values.
553 for child in self.children:
554 child.UpgradeConfig()
555 # add here config upgrade for this disk
558 class Instance(TaggableObject):
559 """Config object representing an instance."""
560 __slots__ = TaggableObject.__slots__ + [
575 def _ComputeSecondaryNodes(self):
576 """Compute the list of secondary nodes.
578 This is a simple wrapper over _ComputeAllNodes.
581 all_nodes = set(self._ComputeAllNodes())
582 all_nodes.discard(self.primary_node)
583 return tuple(all_nodes)
585 secondary_nodes = property(_ComputeSecondaryNodes, None, None,
586 "List of secondary nodes")
588 def _ComputeAllNodes(self):
589 """Compute the list of all nodes.
591 Since the data is already there (in the drbd disks), keeping it as
592 a separate normal attribute is redundant and if not properly
593 synchronised can cause problems. Thus it's better to compute it
597 def _Helper(nodes, device):
598 """Recursively computes nodes given a top device."""
599 if device.dev_type in constants.LDS_DRBD:
600 nodea, nodeb = device.logical_id[:2]
604 for child in device.children:
605 _Helper(nodes, child)
608 all_nodes.add(self.primary_node)
609 for device in self.disks:
610 _Helper(all_nodes, device)
611 return tuple(all_nodes)
613 all_nodes = property(_ComputeAllNodes, None, None,
614 "List of all nodes of the instance")
616 def MapLVsByNode(self, lvmap=None, devs=None, node=None):
617 """Provide a mapping of nodes to LVs this instance owns.
619 This function figures out what logical volumes should belong on
620 which nodes, recursing through a device tree.
622 @param lvmap: optional dictionary to receive the
623 'node' : ['lv', ...] data.
625 @return: None if lvmap arg is given, otherwise, a dictionary
626 of the form { 'nodename' : ['volume1', 'volume2', ...], ... }
630 node = self.primary_node
633 lvmap = { node : [] }
636 if not node in lvmap:
644 if dev.dev_type == constants.LD_LV:
645 lvmap[node].append(dev.logical_id[1])
647 elif dev.dev_type in constants.LDS_DRBD:
649 self.MapLVsByNode(lvmap, dev.children, dev.logical_id[0])
650 self.MapLVsByNode(lvmap, dev.children, dev.logical_id[1])
653 self.MapLVsByNode(lvmap, dev.children, node)
657 def FindDisk(self, idx):
658 """Find a disk given having a specified index.
660 This is just a wrapper that does validation of the index.
663 @param idx: the disk index
665 @return: the corresponding disk
666 @raise errors.OpPrereqError: when the given index is not valid
671 return self.disks[idx]
672 except ValueError, err:
673 raise errors.OpPrereqError("Invalid disk index: '%s'" % str(err))
675 raise errors.OpPrereqError("Invalid disk index: %d (instace has disks"
676 " 0 to %d" % (idx, len(self.disks)))
679 """Instance-specific conversion to standard python types.
681 This replaces the children lists of objects with lists of standard
685 bo = super(Instance, self).ToDict()
687 for attr in "nics", "disks":
688 alist = bo.get(attr, None)
690 nlist = self._ContainerToDicts(alist)
697 def FromDict(cls, val):
698 """Custom function for instances.
701 obj = super(Instance, cls).FromDict(val)
702 obj.nics = cls._ContainerFromDicts(obj.nics, list, NIC)
703 obj.disks = cls._ContainerFromDicts(obj.disks, list, Disk)
706 def UpgradeConfig(self):
707 """Fill defaults for missing configuration values.
710 for nic in self.nics:
712 for disk in self.disks:
716 class OS(ConfigObject):
717 """Config object representing an operating system."""
730 def FromInvalidOS(cls, err):
731 """Create an OS from an InvalidOS error.
733 This routine knows how to convert an InvalidOS error to an OS
734 object representing the broken OS with a meaningful error message.
737 if not isinstance(err, errors.InvalidOS):
738 raise errors.ProgrammerError("Trying to initialize an OS from an"
739 " invalid object of type %s" % type(err))
741 return cls(name=err.args[0], path=err.args[1], status=err.args[2])
743 def __nonzero__(self):
744 return self.status == constants.OS_VALID_STATUS
746 __bool__ = __nonzero__
749 class Node(TaggableObject):
750 """Config object representing a node."""
751 __slots__ = TaggableObject.__slots__ + [
762 class Cluster(TaggableObject):
763 """Config object representing the cluster."""
764 __slots__ = TaggableObject.__slots__ + [
772 "default_hypervisor",
778 "enabled_hypervisors",
781 "candidate_pool_size",
785 def UpgradeConfig(self):
786 """Fill defaults for missing configuration values.
789 if self.hvparams is None:
790 self.hvparams = constants.HVC_DEFAULTS
792 for hypervisor in self.hvparams:
793 self.hvparams[hypervisor] = self.FillDict(
794 constants.HVC_DEFAULTS[hypervisor], self.hvparams[hypervisor])
796 if self.beparams is None:
797 self.beparams = {constants.BEGR_DEFAULT: constants.BEC_DEFAULTS}
799 for begroup in self.beparams:
800 self.beparams[begroup] = self.FillDict(constants.BEC_DEFAULTS,
801 self.beparams[begroup])
803 if self.modify_etc_hosts is None:
804 self.modify_etc_hosts = True
807 """Custom function for cluster.
810 mydict = super(Cluster, self).ToDict()
811 mydict["tcpudp_port_pool"] = list(self.tcpudp_port_pool)
815 def FromDict(cls, val):
816 """Custom function for cluster.
819 obj = super(Cluster, cls).FromDict(val)
820 if not isinstance(obj.tcpudp_port_pool, set):
821 obj.tcpudp_port_pool = set(obj.tcpudp_port_pool)
825 def FillDict(defaults_dict, custom_dict):
826 """Basic function to apply settings on top a default dict.
828 @type defaults_dict: dict
829 @param defaults_dict: dictionary holding the default values
830 @type custom_dict: dict
831 @param custom_dict: dictionary holding customized value
833 @return: dict with the 'full' values
836 ret_dict = copy.deepcopy(defaults_dict)
837 ret_dict.update(custom_dict)
840 def FillHV(self, instance):
841 """Fill an instance's hvparams dict.
843 @type instance: L{objects.Instance}
844 @param instance: the instance parameter to fill
846 @return: a copy of the instance's hvparams with missing keys filled from
850 return self.FillDict(self.hvparams.get(instance.hypervisor, {}),
853 def FillBE(self, instance):
854 """Fill an instance's beparams dict.
856 @type instance: L{objects.Instance}
857 @param instance: the instance parameter to fill
859 @return: a copy of the instance's beparams with missing keys filled from
863 return self.FillDict(self.beparams.get(constants.BEGR_DEFAULT, {}),
867 class SerializableConfigParser(ConfigParser.SafeConfigParser):
868 """Simple wrapper over ConfigParse that allows serialization.
870 This class is basically ConfigParser.SafeConfigParser with two
871 additional methods that allow it to serialize/unserialize to/from a
876 """Dump this instance and return the string representation."""
879 return buf.getvalue()
883 """Load data from a string."""
885 cfp = SerializableConfigParser()