X-Git-Url: https://code.grnet.gr/git/ganeti-local/blobdiff_plain/6a0f22e1e8ec6f77c0e8d0b1aaa1e3e9157454d0..3eaa6e1d6e1bf3b845d66b2343253f8e170cadd2:/qa/qa_config.py diff --git a/qa/qa_config.py b/qa/qa_config.py index afa553d..97ccaa7 100644 --- a/qa/qa_config.py +++ b/qa/qa_config.py @@ -29,60 +29,510 @@ from ganeti import constants from ganeti import utils from ganeti import serializer from ganeti import compat +from ganeti import ht import qa_error _INSTANCE_CHECK_KEY = "instance-check" _ENABLED_HV_KEY = "enabled-hypervisors" -# Key to store the cluster-wide run-time value of the exclusive storage flag -_EXCLUSIVE_STORAGE_KEY = "_exclusive_storage" +_VCLUSTER_MASTER_KEY = "vcluster-master" +_VCLUSTER_BASEDIR_KEY = "vcluster-basedir" +_ENABLED_DISK_TEMPLATES_KEY = "enabled-disk-templates" +#: QA configuration (L{_QaConfig}) +_config = None -cfg = {} -options = None +class _QaInstance(object): + __slots__ = [ + "name", + "nicmac", + "_used", + "_disk_template", + ] -def Load(path): - """Loads the passed configuration file. + def __init__(self, name, nicmac): + """Initializes instances of this class. + + """ + self.name = name + self.nicmac = nicmac + self._used = None + self._disk_template = None + + @classmethod + def FromDict(cls, data): + """Creates instance object from JSON dictionary. + + """ + nicmac = [] + + macaddr = data.get("nic.mac/0") + if macaddr: + nicmac.append(macaddr) + + return cls(name=data["name"], nicmac=nicmac) + + def __repr__(self): + status = [ + "%s.%s" % (self.__class__.__module__, self.__class__.__name__), + "name=%s" % self.name, + "nicmac=%s" % self.nicmac, + "used=%s" % self._used, + "disk_template=%s" % self._disk_template, + ] + + return "<%s at %#x>" % (" ".join(status), id(self)) + + def Use(self): + """Marks instance as being in use. + + """ + assert not self._used + assert self._disk_template is None + + self._used = True + + def Release(self): + """Releases instance and makes it available again. + + """ + assert self._used, \ + ("Instance '%s' was never acquired or released more than once" % + self.name) + + self._used = False + self._disk_template = None + + def GetNicMacAddr(self, idx, default): + """Returns MAC address for NIC. + + @type idx: int + @param idx: NIC index + @param default: Default value + + """ + if len(self.nicmac) > idx: + return self.nicmac[idx] + else: + return default + + def SetDiskTemplate(self, template): + """Set the disk template. + + """ + assert template in constants.DISK_TEMPLATES + + self._disk_template = template + + @property + def used(self): + """Returns boolean denoting whether instance is in use. + + """ + return self._used + + @property + def disk_template(self): + """Returns the current disk template. + + """ + return self._disk_template + + +class _QaNode(object): + __slots__ = [ + "primary", + "secondary", + "_added", + "_use_count", + ] + + def __init__(self, primary, secondary): + """Initializes instances of this class. + + """ + self.primary = primary + self.secondary = secondary + self._added = False + self._use_count = 0 + + @classmethod + def FromDict(cls, data): + """Creates node object from JSON dictionary. + + """ + return cls(primary=data["primary"], secondary=data.get("secondary")) + + def __repr__(self): + status = [ + "%s.%s" % (self.__class__.__module__, self.__class__.__name__), + "primary=%s" % self.primary, + "secondary=%s" % self.secondary, + "added=%s" % self._added, + "use_count=%s" % self._use_count, + ] + + return "<%s at %#x>" % (" ".join(status), id(self)) + + def Use(self): + """Marks a node as being in use. + + """ + assert self._use_count >= 0 + + self._use_count += 1 + + return self + + def Release(self): + """Release a node (opposite of L{Use}). + + """ + assert self.use_count > 0 + + self._use_count -= 1 + + def MarkAdded(self): + """Marks node as having been added to a cluster. + + """ + assert not self._added + self._added = True + + def MarkRemoved(self): + """Marks node as having been removed from a cluster. + + """ + assert self._added + self._added = False + + @property + def added(self): + """Returns whether a node is part of a cluster. + + """ + return self._added + + @property + def use_count(self): + """Returns number of current uses (controlled by L{Use} and L{Release}). + + """ + return self._use_count + + +_RESOURCE_CONVERTER = { + "instances": _QaInstance.FromDict, + "nodes": _QaNode.FromDict, + } + + +def _ConvertResources((key, value)): + """Converts cluster resources in configuration to Python objects. """ - global cfg # pylint: disable=W0603 + fn = _RESOURCE_CONVERTER.get(key, None) + if fn: + return (key, map(fn, value)) + else: + return (key, value) - cfg = serializer.LoadJson(utils.ReadFile(path)) - Validate() +class _QaConfig(object): + def __init__(self, data): + """Initializes instances of this class. + + """ + self._data = data + + #: Cluster-wide run-time value of the exclusive storage flag + self._exclusive_storage = None + + @classmethod + def Load(cls, filename): + """Loads a configuration file and produces a configuration object. + + @type filename: string + @param filename: Path to configuration file + @rtype: L{_QaConfig} + + """ + data = serializer.LoadJson(utils.ReadFile(filename)) + + result = cls(dict(map(_ConvertResources, + data.items()))) # pylint: disable=E1103 + result.Validate() + + return result + + def Validate(self): + """Validates loaded configuration data. + + """ + if not self.get("name"): + raise qa_error.Error("Cluster name is required") + + if not self.get("nodes"): + raise qa_error.Error("Need at least one node") + + if not self.get("instances"): + raise qa_error.Error("Need at least one instance") + + disks = self.GetDiskOptions() + if disks is None: + raise qa_error.Error("Config option 'disks' must exist") + else: + for d in disks: + if d.get("size") is None or d.get("growth") is None: + raise qa_error.Error("Config options `size` and `growth` must exist" + " for all `disks` items") + check = self.GetInstanceCheckScript() + if check: + try: + os.stat(check) + except EnvironmentError, err: + raise qa_error.Error("Can't find instance check script '%s': %s" % + (check, err)) + + enabled_hv = frozenset(self.GetEnabledHypervisors()) + if not enabled_hv: + raise qa_error.Error("No hypervisor is enabled") + + difference = enabled_hv - constants.HYPER_TYPES + if difference: + raise qa_error.Error("Unknown hypervisor(s) enabled: %s" % + utils.CommaJoin(difference)) + + (vc_master, vc_basedir) = self.GetVclusterSettings() + if bool(vc_master) != bool(vc_basedir): + raise qa_error.Error("All or none of the config options '%s' and '%s'" + " must be set" % + (_VCLUSTER_MASTER_KEY, _VCLUSTER_BASEDIR_KEY)) + + if vc_basedir and not utils.IsNormAbsPath(vc_basedir): + raise qa_error.Error("Path given in option '%s' must be absolute and" + " normalized" % _VCLUSTER_BASEDIR_KEY) + + def __getitem__(self, name): + """Returns configuration value. + + @type name: string + @param name: Name of configuration entry + + """ + return self._data[name] + + def __setitem__(self, key, value): + """Sets a configuration value. + + """ + self._data[key] = value + + def __delitem__(self, key): + """Deletes a value from the configuration. + + """ + del(self._data[key]) + + def __len__(self): + """Return the number of configuration items. + + """ + return len(self._data) + + def get(self, name, default=None): + """Returns configuration value. + + @type name: string + @param name: Name of configuration entry + @param default: Default value + + """ + return self._data.get(name, default) + + def GetMasterNode(self): + """Returns the default master node for the cluster. + + """ + return self["nodes"][0] + + def GetInstanceCheckScript(self): + """Returns path to instance check script or C{None}. + + """ + return self._data.get(_INSTANCE_CHECK_KEY, None) + + def GetEnabledHypervisors(self): + """Returns list of enabled hypervisors. + + @rtype: list + + """ + return self._GetStringListParameter( + _ENABLED_HV_KEY, + [constants.DEFAULT_ENABLED_HYPERVISOR]) + + def GetDefaultHypervisor(self): + """Returns the default hypervisor to be used. + + """ + return self.GetEnabledHypervisors()[0] + def GetEnabledDiskTemplates(self): + """Returns the list of enabled disk templates. -def Validate(): - if len(cfg["nodes"]) < 1: - raise qa_error.Error("Need at least one node") - if len(cfg["instances"]) < 1: - raise qa_error.Error("Need at least one instance") - if len(cfg["disk"]) != len(cfg["disk-growth"]): - raise qa_error.Error("Config options 'disk' and 'disk-growth' must have" - " the same number of items") + @rtype: list - check = GetInstanceCheckScript() - if check: + """ + return self._GetStringListParameter( + _ENABLED_DISK_TEMPLATES_KEY, + constants.DEFAULT_ENABLED_DISK_TEMPLATES) + + def GetEnabledStorageTypes(self): + """Returns the list of enabled storage types. + + @rtype: list + @returns: the list of storage types enabled for QA + + """ + enabled_disk_templates = self.GetEnabledDiskTemplates() + enabled_storage_types = list( + set([constants.MAP_DISK_TEMPLATE_STORAGE_TYPE[dt] + for dt in enabled_disk_templates])) + # Storage type 'lvm-pv' cannot be activated via a disk template, + # therefore we add it if 'lvm-vg' is present. + if constants.ST_LVM_VG in enabled_storage_types: + enabled_storage_types.append(constants.ST_LVM_PV) + return enabled_storage_types + + def GetDefaultDiskTemplate(self): + """Returns the default disk template to be used. + + """ + return self.GetEnabledDiskTemplates()[0] + + def _GetStringListParameter(self, key, default_values): + """Retrieves a parameter's value that is supposed to be a list of strings. + + @rtype: list + + """ try: - os.stat(check) - except EnvironmentError, err: - raise qa_error.Error("Can't find instance check script '%s': %s" % - (check, err)) + value = self._data[key] + except KeyError: + return default_values + else: + if value is None: + return [] + elif isinstance(value, basestring): + return value.split(",") + else: + return value - enabled_hv = frozenset(GetEnabledHypervisors()) - if not enabled_hv: - raise qa_error.Error("No hypervisor is enabled") + def SetExclusiveStorage(self, value): + """Set the expected value of the C{exclusive_storage} flag for the cluster. - difference = enabled_hv - constants.HYPER_TYPES - if difference: - raise qa_error.Error("Unknown hypervisor(s) enabled: %s" % - utils.CommaJoin(difference)) + """ + self._exclusive_storage = bool(value) + + def GetExclusiveStorage(self): + """Get the expected value of the C{exclusive_storage} flag for the cluster. + + """ + value = self._exclusive_storage + assert value is not None + return value + + def IsTemplateSupported(self, templ): + """Is the given disk template supported by the current configuration? + + """ + enabled = templ in self.GetEnabledDiskTemplates() + return enabled and (not self.GetExclusiveStorage() or + templ in constants.DTS_EXCL_STORAGE) + + def IsStorageTypeSupported(self, storage_type): + """Is the given storage type supported by the current configuration? + + This is determined by looking if at least one of the disk templates + which is associated with the storage type is enabled in the configuration. + + """ + enabled_disk_templates = self.GetEnabledDiskTemplates() + if storage_type == constants.ST_LVM_PV: + disk_templates = utils.GetDiskTemplatesOfStorageType(constants.ST_LVM_VG) + else: + disk_templates = utils.GetDiskTemplatesOfStorageType(storage_type) + return bool(set(enabled_disk_templates).intersection(set(disk_templates))) + + def AreSpindlesSupported(self): + """Are spindles supported by the current configuration? + + """ + return self.GetExclusiveStorage() + + def GetVclusterSettings(self): + """Returns settings for virtual cluster. + + """ + master = self.get(_VCLUSTER_MASTER_KEY) + basedir = self.get(_VCLUSTER_BASEDIR_KEY) + + return (master, basedir) + + def GetDiskOptions(self): + """Return options for the disks of the instances. + + Get 'disks' parameter from the configuration data. If 'disks' is missing, + try to create it from the legacy 'disk' and 'disk-growth' parameters. + + """ + try: + return self._data["disks"] + except KeyError: + pass + + # Legacy interface + sizes = self._data.get("disk") + growths = self._data.get("disk-growth") + if sizes or growths: + if (sizes is None or growths is None or len(sizes) != len(growths)): + raise qa_error.Error("Config options 'disk' and 'disk-growth' must" + " exist and have the same number of items") + disks = [] + for (size, growth) in zip(sizes, growths): + disks.append({"size": size, "growth": growth}) + return disks + else: + return None + + +def Load(path): + """Loads the passed configuration file. + + """ + global _config # pylint: disable=W0603 + + _config = _QaConfig.Load(path) + + +def GetConfig(): + """Returns the configuration object. + + """ + if _config is None: + raise RuntimeError("Configuration not yet loaded") + + return _config def get(name, default=None): - return cfg.get(name, default) + """Wrapper for L{_QaConfig.get}. + + """ + return GetConfig().get(name, default=default) class Either: @@ -132,6 +582,8 @@ def _TestEnabledInner(check_fn, names, fn): value = _TestEnabledInner(check_fn, name.tests, compat.any) elif isinstance(name, (list, tuple)): value = _TestEnabledInner(check_fn, name, compat.all) + elif callable(name): + value = name() else: value = check_fn(name) @@ -148,10 +600,12 @@ def TestEnabled(tests, _cfg=None): """ if _cfg is None: - _cfg = cfg + cfg = GetConfig() + else: + cfg = _cfg # Get settings for all tests - cfg_tests = _cfg.get("tests", {}) + cfg_tests = cfg.get("tests", {}) # Get default setting default = cfg_tests.get("default", True) @@ -160,111 +614,130 @@ def TestEnabled(tests, _cfg=None): tests, compat.all) -def GetInstanceCheckScript(): - """Returns path to instance check script or C{None}. +def GetInstanceCheckScript(*args): + """Wrapper for L{_QaConfig.GetInstanceCheckScript}. """ - return cfg.get(_INSTANCE_CHECK_KEY, None) + return GetConfig().GetInstanceCheckScript(*args) -def GetEnabledHypervisors(): - """Returns list of enabled hypervisors. +def GetEnabledHypervisors(*args): + """Wrapper for L{_QaConfig.GetEnabledHypervisors}. - @rtype: list + """ + return GetConfig().GetEnabledHypervisors(*args) + + +def GetDefaultHypervisor(*args): + """Wrapper for L{_QaConfig.GetDefaultHypervisor}. """ - try: - value = cfg[_ENABLED_HV_KEY] - except KeyError: - return [constants.DEFAULT_ENABLED_HYPERVISOR] - else: - if isinstance(value, basestring): - # The configuration key ("enabled-hypervisors") implies there can be - # multiple values. Multiple hypervisors are comma-separated on the - # command line option to "gnt-cluster init", so we need to handle them - # equally here. - return value.split(",") - else: - return value + return GetConfig().GetDefaultHypervisor(*args) + + +def GetEnabledDiskTemplates(*args): + """Wrapper for L{_QaConfig.GetEnabledDiskTemplates}. + + """ + return GetConfig().GetEnabledDiskTemplates(*args) -def GetDefaultHypervisor(): - """Returns the default hypervisor to be used. +def GetEnabledStorageTypes(*args): + """Wrapper for L{_QaConfig.GetEnabledStorageTypes}. """ - return GetEnabledHypervisors()[0] + return GetConfig().GetEnabledStorageTypes(*args) -def GetInstanceNicMac(inst, default=None): - """Returns MAC address for instance's network interface. +def GetDefaultDiskTemplate(*args): + """Wrapper for L{_QaConfig.GetDefaultDiskTemplate}. """ - return inst.get("nic.mac/0", default) + return GetConfig().GetDefaultDiskTemplate(*args) def GetMasterNode(): - return cfg["nodes"][0] + """Wrapper for L{_QaConfig.GetMasterNode}. + + """ + return GetConfig().GetMasterNode() -def AcquireInstance(): +def AcquireInstance(_cfg=None): """Returns an instance which isn't in use. """ + if _cfg is None: + cfg = GetConfig() + else: + cfg = _cfg + # Filter out unwanted instances - tmp_flt = lambda inst: not inst.get("_used", False) - instances = filter(tmp_flt, cfg["instances"]) - del tmp_flt + instances = filter(lambda inst: not inst.used, cfg["instances"]) - if len(instances) == 0: + if not instances: raise qa_error.OutOfInstancesError("No instances left") - inst = instances[0] - inst["_used"] = True - inst["_template"] = None - return inst + instance = instances[0] + instance.Use() + return instance -def ReleaseInstance(inst): - inst["_used"] = False +def SetExclusiveStorage(value): + """Wrapper for L{_QaConfig.SetExclusiveStorage}. + + """ + return GetConfig().SetExclusiveStorage(value) -def GetInstanceTemplate(inst): - """Return the disk template of an instance. + +def GetExclusiveStorage(): + """Wrapper for L{_QaConfig.GetExclusiveStorage}. """ - templ = inst["_template"] - assert templ is not None - return templ + return GetConfig().GetExclusiveStorage() -def SetInstanceTemplate(inst, template): - """Set the disk template for an instance. +def IsTemplateSupported(templ): + """Wrapper for L{_QaConfig.IsTemplateSupported}. """ - inst["_template"] = template + return GetConfig().IsTemplateSupported(templ) -def SetExclusiveStorage(value): - """Set the expected value of the exclusive_storage flag for the cluster. +def IsStorageTypeSupported(storage_type): + """Wrapper for L{_QaConfig.IsTemplateSupported}. """ - cfg[_EXCLUSIVE_STORAGE_KEY] = bool(value) + return GetConfig().IsStorageTypeSupported(storage_type) -def GetExclusiveStorage(): - """Get the expected value of the exclusive_storage flag for the cluster. +def AreSpindlesSupported(): + """Wrapper for L{_QaConfig.AreSpindlesSupported}. + + """ + return GetConfig().AreSpindlesSupported() + + +def _NodeSortKey(node): + """Returns sort key for a node. + + @type node: L{_QaNode} """ - val = cfg.get(_EXCLUSIVE_STORAGE_KEY) - assert val is not None - return val + return (node.use_count, utils.NiceSortKey(node.primary)) -def AcquireNode(exclude=None): +def AcquireNode(exclude=None, _cfg=None): """Returns the least used node. """ - master = GetMasterNode() + if _cfg is None: + cfg = GetConfig() + else: + cfg = _cfg + + master = cfg.GetMasterNode() # Filter out unwanted nodes # TODO: Maybe combine filters @@ -275,26 +748,84 @@ def AcquireNode(exclude=None): else: nodes = filter(lambda node: node != exclude, cfg["nodes"]) - tmp_flt = lambda node: node.get("_added", False) or node == master - nodes = filter(tmp_flt, nodes) - del tmp_flt + nodes = filter(lambda node: node.added or node == master, nodes) - if len(nodes) == 0: + if not nodes: raise qa_error.OutOfNodesError("No nodes left") - # Get node with least number of uses - def compare(a, b): - result = cmp(a.get("_count", 0), b.get("_count", 0)) - if result == 0: - result = cmp(a["primary"], b["primary"]) - return result + # Return node with least number of uses + return sorted(nodes, key=_NodeSortKey)[0].Use() - nodes.sort(cmp=compare) - node = nodes[0] - node["_count"] = node.get("_count", 0) + 1 - return node +def AcquireManyNodes(num, exclude=None): + """Return the least used nodes. + @type num: int + @param num: Number of nodes; can be 0. + @type exclude: list of nodes or C{None} + @param exclude: nodes to be excluded from the choice + @rtype: list of nodes + @return: C{num} different nodes -def ReleaseNode(node): - node["_count"] = node.get("_count", 0) - 1 + """ + nodes = [] + if exclude is None: + exclude = [] + elif isinstance(exclude, (list, tuple)): + # Don't modify the incoming argument + exclude = list(exclude) + else: + exclude = [exclude] + + try: + for _ in range(0, num): + n = AcquireNode(exclude=exclude) + nodes.append(n) + exclude.append(n) + except qa_error.OutOfNodesError: + ReleaseManyNodes(nodes) + raise + return nodes + + +def ReleaseManyNodes(nodes): + for node in nodes: + node.Release() + + +def GetVclusterSettings(): + """Wrapper for L{_QaConfig.GetVclusterSettings}. + + """ + return GetConfig().GetVclusterSettings() + + +def UseVirtualCluster(_cfg=None): + """Returns whether a virtual cluster is used. + + @rtype: bool + + """ + if _cfg is None: + cfg = GetConfig() + else: + cfg = _cfg + + (master, _) = cfg.GetVclusterSettings() + + return bool(master) + + +@ht.WithDesc("No virtual cluster") +def NoVirtualCluster(): + """Used to disable tests for virtual clusters. + + """ + return not UseVirtualCluster() + + +def GetDiskOptions(): + """Wrapper for L{_QaConfig.GetDiskOptions}. + + """ + return GetConfig().GetDiskOptions()