X-Git-Url: https://code.grnet.gr/git/ganeti-local/blobdiff_plain/565cb4bf23c020bbfb9745bbb0bf913fb83405b8..7488cf4b6cfefa6f2c5f1137087242eedc594025:/qa/qa_config.py diff --git a/qa/qa_config.py b/qa/qa_config.py index 6c5e27e..4bb3dce 100644 --- a/qa/qa_config.py +++ b/qa/qa_config.py @@ -29,12 +29,24 @@ from ganeti import constants from ganeti import utils from ganeti import serializer from ganeti import compat +from ganeti import ht import qa_error +import qa_logging _INSTANCE_CHECK_KEY = "instance-check" _ENABLED_HV_KEY = "enabled-hypervisors" +_VCLUSTER_MASTER_KEY = "vcluster-master" +_VCLUSTER_BASEDIR_KEY = "vcluster-basedir" +_ENABLED_DISK_TEMPLATES_KEY = "enabled-disk-templates" + +# The constants related to JSON patching (as per RFC6902) that modifies QA's +# configuration. +_QA_BASE_PATH = os.path.dirname(__file__) +_QA_DEFAULT_PATCH = "qa-patch.json" +_QA_PATCH_DIR = "patch" +_QA_PATCH_ORDER_FILE = "order" #: QA configuration (L{_QaConfig}) _config = None @@ -44,8 +56,8 @@ class _QaInstance(object): __slots__ = [ "name", "nicmac", - "used", - "disk_template", + "_used", + "_disk_template", ] def __init__(self, name, nicmac): @@ -54,8 +66,8 @@ class _QaInstance(object): """ self.name = name self.nicmac = nicmac - self.used = None - self.disk_template = None + self._used = None + self._disk_template = None @classmethod def FromDict(cls, data): @@ -70,34 +82,36 @@ class _QaInstance(object): return cls(name=data["name"], nicmac=nicmac) - def __getitem__(self, key): - """Legacy dict-like interface. + def __repr__(self): + status = [ + "%s.%s" % (self.__class__.__module__, self.__class__.__name__), + "name=%s" % self.name, + "nicmac=%s" % self.nicmac, + "used=%s" % self._used, + "disk_template=%s" % self._disk_template, + ] - """ - if key == "name": - return self.name - else: - raise KeyError(key) + return "<%s at %#x>" % (" ".join(status), id(self)) - def get(self, key, default): - """Legacy dict-like interface. + def Use(self): + """Marks instance as being in use. """ - try: - return self[key] - except KeyError: - return default + assert not self._used + assert self._disk_template is None + + self._used = True def Release(self): """Releases instance and makes it available again. """ - assert self.used, \ + assert self._used, \ ("Instance '%s' was never acquired or released more than once" % self.name) - self.used = False - self.disk_template = None + self._used = False + self._disk_template = None def GetNicMacAddr(self, idx, default): """Returns MAC address for NIC. @@ -112,6 +126,28 @@ class _QaInstance(object): else: return default + def SetDiskTemplate(self, template): + """Set the disk template. + + """ + assert template in constants.DISK_TEMPLATES + + self._disk_template = template + + @property + def used(self): + """Returns boolean denoting whether instance is in use. + + """ + return self._used + + @property + def disk_template(self): + """Returns the current disk template. + + """ + return self._disk_template + class _QaNode(object): __slots__ = [ @@ -137,25 +173,16 @@ class _QaNode(object): """ return cls(primary=data["primary"], secondary=data.get("secondary")) - def __getitem__(self, key): - """Legacy dict-like interface. - - """ - if key == "primary": - return self.primary - elif key == "secondary": - return self.secondary - else: - raise KeyError(key) + def __repr__(self): + status = [ + "%s.%s" % (self.__class__.__module__, self.__class__.__name__), + "primary=%s" % self.primary, + "secondary=%s" % self.secondary, + "added=%s" % self._added, + "use_count=%s" % self._use_count, + ] - def get(self, key, default): - """Legacy dict-like interface. - - """ - try: - return self[key] - except KeyError: - return default + return "<%s at %#x>" % (" ".join(status), id(self)) def Use(self): """Marks a node as being in use. @@ -231,6 +258,114 @@ class _QaConfig(object): #: Cluster-wide run-time value of the exclusive storage flag self._exclusive_storage = None + @staticmethod + def LoadPatch(patch_dict, rel_path): + """ Loads a single patch. + + @type patch_dict: dict of string to dict + @param patch_dict: A dictionary storing patches by relative path. + @type rel_path: string + @param rel_path: The relative path to the patch, might or might not exist. + + """ + try: + full_path = os.path.join(_QA_BASE_PATH, rel_path) + patch = serializer.LoadJson(utils.ReadFile(full_path)) + patch_dict[rel_path] = patch + except IOError: + pass + + @staticmethod + def LoadPatches(): + """ Finds and loads all patches supported by the QA. + + @rtype: dict of string to dict + @return: A dictionary of relative path to patch content. + + """ + patches = {} + _QaConfig.LoadPatch(patches, _QA_DEFAULT_PATCH) + patch_dir_path = os.path.join(_QA_BASE_PATH, _QA_PATCH_DIR) + if os.path.exists(patch_dir_path): + for filename in os.listdir(patch_dir_path): + if filename.endswith(".json"): + _QaConfig.LoadPatch(patches, os.path.join(_QA_PATCH_DIR, filename)) + return patches + + @staticmethod + def ApplyPatch(data, patch_module, patches, patch_path): + """Applies a single patch. + + @type data: dict (deserialized json) + @param data: The QA configuration + @type patch_module: module + @param patch_module: The json patch module, loaded dynamically + @type patches: dict of string to dict + @param patches: The dictionary of patch path to content + @type patch_path: string + @param patch_path: The path to the patch, relative to the QA directory + + @return: The modified configuration data. + + """ + patch_content = patches[patch_path] + print qa_logging.FormatInfo("Applying patch %s" % patch_path) + if not patch_content and patch_path != _QA_DEFAULT_PATCH: + print qa_logging.FormatWarning("The patch %s added by the user is empty" % + patch_path) + data = patch_module.apply_patch(data, patch_content) + + @staticmethod + def ApplyPatches(data, patch_module, patches): + """Applies any patches present, and returns the modified QA configuration. + + First, patches from the patch directory are applied. They are ordered + alphabetically, unless there is an ``order`` file present - any patches + listed within are applied in that order, and any remaining ones in + alphabetical order again. Finally, the default patch residing in the + top-level QA directory is applied. + + @type data: dict (deserialized json) + @param data: The QA configuration + @type patch_module: module + @param patch_module: The json patch module, loaded dynamically + @type patches: dict of string to dict + @param patches: The dictionary of patch path to content + + @return: The modified configuration data. + + """ + ordered_patches = [] + order_path = os.path.join(_QA_BASE_PATH, _QA_PATCH_DIR, + _QA_PATCH_ORDER_FILE) + if os.path.exists(order_path): + order_file = open(order_path, 'r') + ordered_patches = order_file.read().splitlines() + # Removes empty lines + ordered_patches = filter(None, ordered_patches) + + # Add the patch dir + ordered_patches = map(lambda x: os.path.join(_QA_PATCH_DIR, x), + ordered_patches) + + # First the ordered patches + for patch in ordered_patches: + if patch not in patches: + raise qa_error.Error("Patch %s specified in the ordering file does not " + "exist" % patch) + _QaConfig.ApplyPatch(data, patch_module, patches, patch) + + # Then the other non-default ones + for patch in sorted(patches): + if patch != _QA_DEFAULT_PATCH and patch not in ordered_patches: + _QaConfig.ApplyPatch(data, patch_module, patches, patch) + + # Finally the default one + if _QA_DEFAULT_PATCH in patches: + _QaConfig.ApplyPatch(data, patch_module, patches, _QA_DEFAULT_PATCH) + + return data + @classmethod def Load(cls, filename): """Loads a configuration file and produces a configuration object. @@ -242,6 +377,21 @@ class _QaConfig(object): """ data = serializer.LoadJson(utils.ReadFile(filename)) + # Patch the document using JSON Patch (RFC6902) in file _PATCH_JSON, if + # available + try: + patches = _QaConfig.LoadPatches() + # Try to use the module only if there is a non-empty patch present + if any(patches.values()): + mod = __import__("jsonpatch", fromlist=[]) + data = _QaConfig.ApplyPatches(data, mod, patches) + except IOError: + pass + except ImportError: + raise qa_error.Error("For the QA JSON patching feature to work, you " + "need to install Python modules 'jsonpatch' and " + "'jsonpointer'.") + result = cls(dict(map(_ConvertResources, data.items()))) # pylint: disable=E1103 result.Validate() @@ -252,18 +402,23 @@ class _QaConfig(object): """Validates loaded configuration data. """ + if not self.get("name"): + raise qa_error.Error("Cluster name is required") + if not self.get("nodes"): raise qa_error.Error("Need at least one node") if not self.get("instances"): raise qa_error.Error("Need at least one instance") - if (self.get("disk") is None or - self.get("disk-growth") is None or - len(self.get("disk")) != len(self.get("disk-growth"))): - raise qa_error.Error("Config options 'disk' and 'disk-growth' must exist" - " and have the same number of items") - + disks = self.GetDiskOptions() + if disks is None: + raise qa_error.Error("Config option 'disks' must exist") + else: + for d in disks: + if d.get("size") is None or d.get("growth") is None: + raise qa_error.Error("Config options `size` and `growth` must exist" + " for all `disks` items") check = self.GetInstanceCheckScript() if check: try: @@ -281,6 +436,16 @@ class _QaConfig(object): raise qa_error.Error("Unknown hypervisor(s) enabled: %s" % utils.CommaJoin(difference)) + (vc_master, vc_basedir) = self.GetVclusterSettings() + if bool(vc_master) != bool(vc_basedir): + raise qa_error.Error("All or none of the config options '%s' and '%s'" + " must be set" % + (_VCLUSTER_MASTER_KEY, _VCLUSTER_BASEDIR_KEY)) + + if vc_basedir and not utils.IsNormAbsPath(vc_basedir): + raise qa_error.Error("Path given in option '%s' must be absolute and" + " normalized" % _VCLUSTER_BASEDIR_KEY) + def __getitem__(self, name): """Returns configuration value. @@ -318,28 +483,50 @@ class _QaConfig(object): @rtype: list """ + return self._GetStringListParameter( + _ENABLED_HV_KEY, + [constants.DEFAULT_ENABLED_HYPERVISOR]) + + def GetDefaultHypervisor(self): + """Returns the default hypervisor to be used. + + """ + return self.GetEnabledHypervisors()[0] + + def GetEnabledDiskTemplates(self): + """Returns the list of enabled disk templates. + + @rtype: list + + """ + return self._GetStringListParameter( + _ENABLED_DISK_TEMPLATES_KEY, + list(constants.DEFAULT_ENABLED_DISK_TEMPLATES)) + + def GetDefaultDiskTemplate(self): + """Returns the default disk template to be used. + + """ + return self.GetEnabledDiskTemplates()[0] + + def _GetStringListParameter(self, key, default_values): + """Retrieves a parameter's value that is supposed to be a list of strings. + + @rtype: list + + """ try: - value = self._data[_ENABLED_HV_KEY] + value = self._data[key] except KeyError: - return [constants.DEFAULT_ENABLED_HYPERVISOR] + return default_values else: if value is None: return [] elif isinstance(value, basestring): - # The configuration key ("enabled-hypervisors") implies there can be - # multiple values. Multiple hypervisors are comma-separated on the - # command line option to "gnt-cluster init", so we need to handle them - # equally here. return value.split(",") else: return value - def GetDefaultHypervisor(self): - """Returns the default hypervisor to be used. - - """ - return self.GetEnabledHypervisors()[0] - def SetExclusiveStorage(self, value): """Set the expected value of the C{exclusive_storage} flag for the cluster. @@ -358,8 +545,44 @@ class _QaConfig(object): """Is the given disk template supported by the current configuration? """ - return (not self.GetExclusiveStorage() or - templ in constants.DTS_EXCL_STORAGE) + enabled = templ in self.GetEnabledDiskTemplates() + return enabled and (not self.GetExclusiveStorage() or + templ in constants.DTS_EXCL_STORAGE) + + def GetVclusterSettings(self): + """Returns settings for virtual cluster. + + """ + master = self.get(_VCLUSTER_MASTER_KEY) + basedir = self.get(_VCLUSTER_BASEDIR_KEY) + + return (master, basedir) + + def GetDiskOptions(self): + """Return options for the disks of the instances. + + Get 'disks' parameter from the configuration data. If 'disks' is missing, + try to create it from the legacy 'disk' and 'disk-growth' parameters. + + """ + try: + return self._data["disks"] + except KeyError: + pass + + # Legacy interface + sizes = self._data.get("disk") + growths = self._data.get("disk-growth") + if sizes or growths: + if (sizes is None or growths is None or len(sizes) != len(growths)): + raise qa_error.Error("Config options 'disk' and 'disk-growth' must" + " exist and have the same number of items") + disks = [] + for (size, growth) in zip(sizes, growths): + disks.append({"size": size, "growth": growth}) + return disks + else: + return None def Load(path): @@ -435,6 +658,8 @@ def _TestEnabledInner(check_fn, names, fn): value = _TestEnabledInner(check_fn, name.tests, compat.any) elif isinstance(name, (list, tuple)): value = _TestEnabledInner(check_fn, name, compat.all) + elif callable(name): + value = name() else: value = check_fn(name) @@ -486,11 +711,18 @@ def GetDefaultHypervisor(*args): return GetConfig().GetDefaultHypervisor(*args) -def GetInstanceNicMac(inst, default=None): - """Returns MAC address for instance's network interface. +def GetEnabledDiskTemplates(*args): + """Wrapper for L{_QaConfig.GetEnabledDiskTemplates}. + + """ + return GetConfig().GetEnabledDiskTemplates(*args) + + +def GetDefaultDiskTemplate(*args): + """Wrapper for L{_QaConfig.GetDefaultDiskTemplate}. """ - return inst.GetNicMacAddr(0, default) + return GetConfig().GetDefaultDiskTemplate(*args) def GetMasterNode(): @@ -515,30 +747,10 @@ def AcquireInstance(_cfg=None): if not instances: raise qa_error.OutOfInstancesError("No instances left") - inst = instances[0] - - assert not inst.used - assert inst.disk_template is None - - inst.used = True - - return inst + instance = instances[0] + instance.Use() - -def GetInstanceTemplate(inst): - """Return the disk template of an instance. - - """ - templ = inst.disk_template - assert templ is not None - return templ - - -def SetInstanceTemplate(inst, template): - """Set the disk template for an instance. - - """ - inst.disk_template = template + return instance def SetExclusiveStorage(value): @@ -556,12 +768,21 @@ def GetExclusiveStorage(): def IsTemplateSupported(templ): - """Wrapper for L{_QaConfig.GetExclusiveStorage}. + """Wrapper for L{_QaConfig.IsTemplateSupported}. """ return GetConfig().IsTemplateSupported(templ) +def _NodeSortKey(node): + """Returns sort key for a node. + + @type node: L{_QaNode} + + """ + return (node.use_count, utils.NiceSortKey(node.primary)) + + def AcquireNode(exclude=None, _cfg=None): """Returns the least used node. @@ -587,17 +808,8 @@ def AcquireNode(exclude=None, _cfg=None): if not nodes: raise qa_error.OutOfNodesError("No nodes left") - # Get node with least number of uses - # TODO: Switch to computing sort key instead of comparing directly - def compare(a, b): - result = cmp(a.use_count, b.use_count) - if result == 0: - result = cmp(a.primary, b.primary) - return result - - nodes.sort(cmp=compare) - - return nodes[0].Use() + # Return node with least number of uses + return sorted(nodes, key=_NodeSortKey)[0].Use() def AcquireManyNodes(num, exclude=None): @@ -634,3 +846,41 @@ def AcquireManyNodes(num, exclude=None): def ReleaseManyNodes(nodes): for node in nodes: node.Release() + + +def GetVclusterSettings(): + """Wrapper for L{_QaConfig.GetVclusterSettings}. + + """ + return GetConfig().GetVclusterSettings() + + +def UseVirtualCluster(_cfg=None): + """Returns whether a virtual cluster is used. + + @rtype: bool + + """ + if _cfg is None: + cfg = GetConfig() + else: + cfg = _cfg + + (master, _) = cfg.GetVclusterSettings() + + return bool(master) + + +@ht.WithDesc("No virtual cluster") +def NoVirtualCluster(): + """Used to disable tests for virtual clusters. + + """ + return not UseVirtualCluster() + + +def GetDiskOptions(): + """Wrapper for L{_QaConfig.GetDiskOptions}. + + """ + return GetConfig().GetDiskOptions()