from ganeti import utils
from ganeti import serializer
from ganeti import compat
+from ganeti import ht
import qa_error
+import qa_logging
_INSTANCE_CHECK_KEY = "instance-check"
_ENABLED_HV_KEY = "enabled-hypervisors"
+_VCLUSTER_MASTER_KEY = "vcluster-master"
+_VCLUSTER_BASEDIR_KEY = "vcluster-basedir"
+_ENABLED_DISK_TEMPLATES_KEY = "enabled-disk-templates"
+
+# The constants related to JSON patching (as per RFC6902) that modifies QA's
+# configuration.
+_QA_BASE_PATH = os.path.dirname(__file__)
+_QA_DEFAULT_PATCH = "qa-patch.json"
+_QA_PATCH_DIR = "patch"
+_QA_PATCH_ORDER_FILE = "order"
#: QA configuration (L{_QaConfig})
_config = None
__slots__ = [
"name",
"nicmac",
- "used",
- "disk_template",
+ "_used",
+ "_disk_template",
]
def __init__(self, name, nicmac):
"""
self.name = name
self.nicmac = nicmac
- self.used = None
- self.disk_template = None
+ self._used = None
+ self._disk_template = None
@classmethod
def FromDict(cls, data):
return cls(name=data["name"], nicmac=nicmac)
- def __getitem__(self, key):
- """Legacy dict-like interface.
+ def __repr__(self):
+ status = [
+ "%s.%s" % (self.__class__.__module__, self.__class__.__name__),
+ "name=%s" % self.name,
+ "nicmac=%s" % self.nicmac,
+ "used=%s" % self._used,
+ "disk_template=%s" % self._disk_template,
+ ]
- """
- if key == "name":
- return self.name
- else:
- raise KeyError(key)
+ return "<%s at %#x>" % (" ".join(status), id(self))
- def get(self, key, default):
- """Legacy dict-like interface.
+ def Use(self):
+ """Marks instance as being in use.
"""
- try:
- return self[key]
- except KeyError:
- return default
+ assert not self._used
+ assert self._disk_template is None
+
+ self._used = True
def Release(self):
"""Releases instance and makes it available again.
"""
- assert self.used, \
+ assert self._used, \
("Instance '%s' was never acquired or released more than once" %
self.name)
- self.used = False
- self.disk_template = None
+ self._used = False
+ self._disk_template = None
def GetNicMacAddr(self, idx, default):
"""Returns MAC address for NIC.
else:
return default
+ def SetDiskTemplate(self, template):
+ """Set the disk template.
+
+ """
+ assert template in constants.DISK_TEMPLATES
+
+ self._disk_template = template
+
+ @property
+ def used(self):
+ """Returns boolean denoting whether instance is in use.
+
+ """
+ return self._used
+
+ @property
+ def disk_template(self):
+ """Returns the current disk template.
+
+ """
+ return self._disk_template
+
class _QaNode(object):
__slots__ = [
"primary",
"secondary",
"_added",
- "use_count",
+ "_use_count",
]
def __init__(self, primary, secondary):
"""
self.primary = primary
self.secondary = secondary
- self.use_count = 0
self._added = False
+ self._use_count = 0
@classmethod
def FromDict(cls, data):
"""
return cls(primary=data["primary"], secondary=data.get("secondary"))
- def __getitem__(self, key):
- """Legacy dict-like interface.
-
- """
- if key == "primary":
- return self.primary
- elif key == "secondary":
- return self.secondary
- else:
- raise KeyError(key)
+ def __repr__(self):
+ status = [
+ "%s.%s" % (self.__class__.__module__, self.__class__.__name__),
+ "primary=%s" % self.primary,
+ "secondary=%s" % self.secondary,
+ "added=%s" % self._added,
+ "use_count=%s" % self._use_count,
+ ]
- def get(self, key, default):
- """Legacy dict-like interface.
-
- """
- try:
- return self[key]
- except KeyError:
- return default
+ return "<%s at %#x>" % (" ".join(status), id(self))
def Use(self):
"""Marks a node as being in use.
"""
- assert self.use_count >= 0
+ assert self._use_count >= 0
- self.use_count += 1
+ self._use_count += 1
return self
+ def Release(self):
+ """Release a node (opposite of L{Use}).
+
+ """
+ assert self.use_count > 0
+
+ self._use_count -= 1
+
def MarkAdded(self):
"""Marks node as having been added to a cluster.
"""
return self._added
+ @property
+ def use_count(self):
+ """Returns number of current uses (controlled by L{Use} and L{Release}).
+
+ """
+ return self._use_count
+
_RESOURCE_CONVERTER = {
"instances": _QaInstance.FromDict,
#: Cluster-wide run-time value of the exclusive storage flag
self._exclusive_storage = None
+ @staticmethod
+ def LoadPatch(patch_dict, rel_path):
+ """ Loads a single patch.
+
+ @type patch_dict: dict of string to dict
+ @param patch_dict: A dictionary storing patches by relative path.
+ @type rel_path: string
+ @param rel_path: The relative path to the patch, might or might not exist.
+
+ """
+ try:
+ full_path = os.path.join(_QA_BASE_PATH, rel_path)
+ patch = serializer.LoadJson(utils.ReadFile(full_path))
+ patch_dict[rel_path] = patch
+ except IOError:
+ pass
+
+ @staticmethod
+ def LoadPatches():
+ """ Finds and loads all patches supported by the QA.
+
+ @rtype: dict of string to dict
+ @return: A dictionary of relative path to patch content.
+
+ """
+ patches = {}
+ _QaConfig.LoadPatch(patches, _QA_DEFAULT_PATCH)
+ patch_dir_path = os.path.join(_QA_BASE_PATH, _QA_PATCH_DIR)
+ if os.path.exists(patch_dir_path):
+ for filename in os.listdir(patch_dir_path):
+ if filename.endswith(".json"):
+ _QaConfig.LoadPatch(patches, os.path.join(_QA_PATCH_DIR, filename))
+ return patches
+
+ @staticmethod
+ def ApplyPatch(data, patch_module, patches, patch_path):
+ """Applies a single patch.
+
+ @type data: dict (deserialized json)
+ @param data: The QA configuration
+ @type patch_module: module
+ @param patch_module: The json patch module, loaded dynamically
+ @type patches: dict of string to dict
+ @param patches: The dictionary of patch path to content
+ @type patch_path: string
+ @param patch_path: The path to the patch, relative to the QA directory
+
+ @return: The modified configuration data.
+
+ """
+ patch_content = patches[patch_path]
+ print qa_logging.FormatInfo("Applying patch %s" % patch_path)
+ if not patch_content and patch_path != _QA_DEFAULT_PATCH:
+ print qa_logging.FormatWarning("The patch %s added by the user is empty" %
+ patch_path)
+ data = patch_module.apply_patch(data, patch_content)
+
+ @staticmethod
+ def ApplyPatches(data, patch_module, patches):
+ """Applies any patches present, and returns the modified QA configuration.
+
+ First, patches from the patch directory are applied. They are ordered
+ alphabetically, unless there is an ``order`` file present - any patches
+ listed within are applied in that order, and any remaining ones in
+ alphabetical order again. Finally, the default patch residing in the
+ top-level QA directory is applied.
+
+ @type data: dict (deserialized json)
+ @param data: The QA configuration
+ @type patch_module: module
+ @param patch_module: The json patch module, loaded dynamically
+ @type patches: dict of string to dict
+ @param patches: The dictionary of patch path to content
+
+ @return: The modified configuration data.
+
+ """
+ ordered_patches = []
+ order_path = os.path.join(_QA_BASE_PATH, _QA_PATCH_DIR,
+ _QA_PATCH_ORDER_FILE)
+ if os.path.exists(order_path):
+ order_file = open(order_path, 'r')
+ ordered_patches = order_file.read().splitlines()
+ # Removes empty lines
+ ordered_patches = filter(None, ordered_patches)
+
+ # Add the patch dir
+ ordered_patches = map(lambda x: os.path.join(_QA_PATCH_DIR, x),
+ ordered_patches)
+
+ # First the ordered patches
+ for patch in ordered_patches:
+ if patch not in patches:
+ raise qa_error.Error("Patch %s specified in the ordering file does not "
+ "exist" % patch)
+ _QaConfig.ApplyPatch(data, patch_module, patches, patch)
+
+ # Then the other non-default ones
+ for patch in sorted(patches):
+ if patch != _QA_DEFAULT_PATCH and patch not in ordered_patches:
+ _QaConfig.ApplyPatch(data, patch_module, patches, patch)
+
+ # Finally the default one
+ if _QA_DEFAULT_PATCH in patches:
+ _QaConfig.ApplyPatch(data, patch_module, patches, _QA_DEFAULT_PATCH)
+
+ return data
+
@classmethod
def Load(cls, filename):
"""Loads a configuration file and produces a configuration object.
"""
data = serializer.LoadJson(utils.ReadFile(filename))
+ # Patch the document using JSON Patch (RFC6902) in file _PATCH_JSON, if
+ # available
+ try:
+ patches = _QaConfig.LoadPatches()
+ # Try to use the module only if there is a non-empty patch present
+ if any(patches.values()):
+ mod = __import__("jsonpatch", fromlist=[])
+ data = _QaConfig.ApplyPatches(data, mod, patches)
+ except IOError:
+ pass
+ except ImportError:
+ raise qa_error.Error("For the QA JSON patching feature to work, you "
+ "need to install Python modules 'jsonpatch' and "
+ "'jsonpointer'.")
+
result = cls(dict(map(_ConvertResources,
data.items()))) # pylint: disable=E1103
result.Validate()
"""Validates loaded configuration data.
"""
+ if not self.get("name"):
+ raise qa_error.Error("Cluster name is required")
+
if not self.get("nodes"):
raise qa_error.Error("Need at least one node")
if not self.get("instances"):
raise qa_error.Error("Need at least one instance")
- if (self.get("disk") is None or
- self.get("disk-growth") is None or
- len(self.get("disk")) != len(self.get("disk-growth"))):
- raise qa_error.Error("Config options 'disk' and 'disk-growth' must exist"
- " and have the same number of items")
-
+ disks = self.GetDiskOptions()
+ if disks is None:
+ raise qa_error.Error("Config option 'disks' must exist")
+ else:
+ for d in disks:
+ if d.get("size") is None or d.get("growth") is None:
+ raise qa_error.Error("Config options `size` and `growth` must exist"
+ " for all `disks` items")
check = self.GetInstanceCheckScript()
if check:
try:
raise qa_error.Error("Unknown hypervisor(s) enabled: %s" %
utils.CommaJoin(difference))
+ (vc_master, vc_basedir) = self.GetVclusterSettings()
+ if bool(vc_master) != bool(vc_basedir):
+ raise qa_error.Error("All or none of the config options '%s' and '%s'"
+ " must be set" %
+ (_VCLUSTER_MASTER_KEY, _VCLUSTER_BASEDIR_KEY))
+
+ if vc_basedir and not utils.IsNormAbsPath(vc_basedir):
+ raise qa_error.Error("Path given in option '%s' must be absolute and"
+ " normalized" % _VCLUSTER_BASEDIR_KEY)
+
def __getitem__(self, name):
"""Returns configuration value.
@rtype: list
"""
+ return self._GetStringListParameter(
+ _ENABLED_HV_KEY,
+ [constants.DEFAULT_ENABLED_HYPERVISOR])
+
+ def GetDefaultHypervisor(self):
+ """Returns the default hypervisor to be used.
+
+ """
+ return self.GetEnabledHypervisors()[0]
+
+ def GetEnabledDiskTemplates(self):
+ """Returns the list of enabled disk templates.
+
+ @rtype: list
+
+ """
+ return self._GetStringListParameter(
+ _ENABLED_DISK_TEMPLATES_KEY,
+ list(constants.DEFAULT_ENABLED_DISK_TEMPLATES))
+
+ def GetDefaultDiskTemplate(self):
+ """Returns the default disk template to be used.
+
+ """
+ return self.GetEnabledDiskTemplates()[0]
+
+ def _GetStringListParameter(self, key, default_values):
+ """Retrieves a parameter's value that is supposed to be a list of strings.
+
+ @rtype: list
+
+ """
try:
- value = self._data[_ENABLED_HV_KEY]
+ value = self._data[key]
except KeyError:
- return [constants.DEFAULT_ENABLED_HYPERVISOR]
+ return default_values
else:
if value is None:
return []
elif isinstance(value, basestring):
- # The configuration key ("enabled-hypervisors") implies there can be
- # multiple values. Multiple hypervisors are comma-separated on the
- # command line option to "gnt-cluster init", so we need to handle them
- # equally here.
return value.split(",")
else:
return value
- def GetDefaultHypervisor(self):
- """Returns the default hypervisor to be used.
-
- """
- return self.GetEnabledHypervisors()[0]
-
def SetExclusiveStorage(self, value):
"""Set the expected value of the C{exclusive_storage} flag for the cluster.
"""Is the given disk template supported by the current configuration?
"""
- return (not self.GetExclusiveStorage() or
- templ in constants.DTS_EXCL_STORAGE)
+ enabled = templ in self.GetEnabledDiskTemplates()
+ return enabled and (not self.GetExclusiveStorage() or
+ templ in constants.DTS_EXCL_STORAGE)
+
+ def GetVclusterSettings(self):
+ """Returns settings for virtual cluster.
+
+ """
+ master = self.get(_VCLUSTER_MASTER_KEY)
+ basedir = self.get(_VCLUSTER_BASEDIR_KEY)
+
+ return (master, basedir)
+
+ def GetDiskOptions(self):
+ """Return options for the disks of the instances.
+
+ Get 'disks' parameter from the configuration data. If 'disks' is missing,
+ try to create it from the legacy 'disk' and 'disk-growth' parameters.
+
+ """
+ try:
+ return self._data["disks"]
+ except KeyError:
+ pass
+
+ # Legacy interface
+ sizes = self._data.get("disk")
+ growths = self._data.get("disk-growth")
+ if sizes or growths:
+ if (sizes is None or growths is None or len(sizes) != len(growths)):
+ raise qa_error.Error("Config options 'disk' and 'disk-growth' must"
+ " exist and have the same number of items")
+ disks = []
+ for (size, growth) in zip(sizes, growths):
+ disks.append({"size": size, "growth": growth})
+ return disks
+ else:
+ return None
def Load(path):
value = _TestEnabledInner(check_fn, name.tests, compat.any)
elif isinstance(name, (list, tuple)):
value = _TestEnabledInner(check_fn, name, compat.all)
+ elif callable(name):
+ value = name()
else:
value = check_fn(name)
return GetConfig().GetDefaultHypervisor(*args)
-def GetInstanceNicMac(inst, default=None):
- """Returns MAC address for instance's network interface.
+def GetEnabledDiskTemplates(*args):
+ """Wrapper for L{_QaConfig.GetEnabledDiskTemplates}.
+
+ """
+ return GetConfig().GetEnabledDiskTemplates(*args)
+
+
+def GetDefaultDiskTemplate(*args):
+ """Wrapper for L{_QaConfig.GetDefaultDiskTemplate}.
"""
- return inst.GetNicMacAddr(0, default)
+ return GetConfig().GetDefaultDiskTemplate(*args)
def GetMasterNode():
if not instances:
raise qa_error.OutOfInstancesError("No instances left")
- inst = instances[0]
+ instance = instances[0]
+ instance.Use()
- assert not inst.used
- assert inst.disk_template is None
-
- inst.used = True
-
- return inst
-
-
-def GetInstanceTemplate(inst):
- """Return the disk template of an instance.
-
- """
- templ = inst.disk_template
- assert templ is not None
- return templ
-
-
-def SetInstanceTemplate(inst, template):
- """Set the disk template for an instance.
-
- """
- inst.disk_template = template
+ return instance
def SetExclusiveStorage(value):
def IsTemplateSupported(templ):
- """Wrapper for L{_QaConfig.GetExclusiveStorage}.
+ """Wrapper for L{_QaConfig.IsTemplateSupported}.
"""
return GetConfig().IsTemplateSupported(templ)
+def _NodeSortKey(node):
+ """Returns sort key for a node.
+
+ @type node: L{_QaNode}
+
+ """
+ return (node.use_count, utils.NiceSortKey(node.primary))
+
+
def AcquireNode(exclude=None, _cfg=None):
"""Returns the least used node.
if not nodes:
raise qa_error.OutOfNodesError("No nodes left")
- # Get node with least number of uses
- # TODO: Switch to computing sort key instead of comparing directly
- def compare(a, b):
- result = cmp(a.use_count, b.use_count)
- if result == 0:
- result = cmp(a.primary, b.primary)
- return result
-
- nodes.sort(cmp=compare)
-
- return nodes[0].Use()
+ # Return node with least number of uses
+ return sorted(nodes, key=_NodeSortKey)[0].Use()
def AcquireManyNodes(num, exclude=None):
return nodes
-def ReleaseNode(node):
- assert node.use_count > 0
+def ReleaseManyNodes(nodes):
+ for node in nodes:
+ node.Release()
- node.use_count -= 1
+def GetVclusterSettings():
+ """Wrapper for L{_QaConfig.GetVclusterSettings}.
-def ReleaseManyNodes(nodes):
- for n in nodes:
- ReleaseNode(n)
+ """
+ return GetConfig().GetVclusterSettings()
+
+
+def UseVirtualCluster(_cfg=None):
+ """Returns whether a virtual cluster is used.
+
+ @rtype: bool
+
+ """
+ if _cfg is None:
+ cfg = GetConfig()
+ else:
+ cfg = _cfg
+
+ (master, _) = cfg.GetVclusterSettings()
+
+ return bool(master)
+
+
+@ht.WithDesc("No virtual cluster")
+def NoVirtualCluster():
+ """Used to disable tests for virtual clusters.
+
+ """
+ return not UseVirtualCluster()
+
+
+def GetDiskOptions():
+ """Wrapper for L{_QaConfig.GetDiskOptions}.
+
+ """
+ return GetConfig().GetDiskOptions()
projects / ganeti-local / blobdiff