#
#
-# Copyright (C) 2006, 2007 Google Inc.
+# Copyright (C) 2006, 2007, 2011 Google Inc.
#
# This program is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
"""
import logging
+import random
+import time
from ganeti import opcodes
from ganeti import constants
from ganeti import utils
-class OpExecCbBase:
+_OP_PREFIX = "Op"
+_LU_PREFIX = "LU"
+
+
+class LockAcquireTimeout(Exception):
+ """Exception to report timeouts on acquiring locks.
+
+ """
+
+
+def _CalculateLockAttemptTimeouts():
+ """Calculate timeouts for lock attempts.
+
+ """
+ result = [constants.LOCK_ATTEMPTS_MINWAIT]
+ running_sum = result[0]
+
+ # Wait for a total of at least LOCK_ATTEMPTS_TIMEOUT before doing a
+ # blocking acquire
+ while running_sum < constants.LOCK_ATTEMPTS_TIMEOUT:
+ timeout = (result[-1] * 1.05) ** 1.25
+
+ # Cap max timeout. This gives other jobs a chance to run even if
+ # we're still trying to get our locks, before finally moving to a
+ # blocking acquire.
+ timeout = min(timeout, constants.LOCK_ATTEMPTS_MAXWAIT)
+ # And also cap the lower boundary for safety
+ timeout = max(timeout, constants.LOCK_ATTEMPTS_MINWAIT)
+
+ result.append(timeout)
+ running_sum += timeout
+
+ return result
+
+
+class LockAttemptTimeoutStrategy(object):
+ """Class with lock acquire timeout strategy.
+
+ """
+ __slots__ = [
+ "_timeouts",
+ "_random_fn",
+ "_time_fn",
+ ]
+
+ _TIMEOUT_PER_ATTEMPT = _CalculateLockAttemptTimeouts()
+
+ def __init__(self, _time_fn=time.time, _random_fn=random.random):
+ """Initializes this class.
+
+ @param _time_fn: Time function for unittests
+ @param _random_fn: Random number generator for unittests
+
+ """
+ object.__init__(self)
+
+ self._timeouts = iter(self._TIMEOUT_PER_ATTEMPT)
+ self._time_fn = _time_fn
+ self._random_fn = _random_fn
+
+ def NextAttempt(self):
+ """Returns the timeout for the next attempt.
+
+ """
+ try:
+ timeout = self._timeouts.next()
+ except StopIteration:
+ # No more timeouts, do blocking acquire
+ timeout = None
+
+ if timeout is not None:
+ # Add a small variation (-/+ 5%) to timeout. This helps in situations
+ # where two or more jobs are fighting for the same lock(s).
+ variation_range = timeout * 0.1
+ timeout += ((self._random_fn() * variation_range) -
+ (variation_range * 0.5))
+
+ return timeout
+
+
+class OpExecCbBase: # pylint: disable-msg=W0232
"""Base class for OpCode execution callbacks.
"""
"""
- def ReportLocks(self, msg):
- """Report lock operations.
+ def CheckCancel(self):
+ """Check whether job has been cancelled.
"""
+def _LUNameForOpName(opname):
+ """Computes the LU name for a given OpCode name.
+
+ """
+ assert opname.startswith(_OP_PREFIX), \
+ "Invalid OpCode name, doesn't start with %s: %s" % (_OP_PREFIX, opname)
+
+ return _LU_PREFIX + opname[len(_OP_PREFIX):]
+
+
+def _ComputeDispatchTable():
+ """Computes the opcode-to-lu dispatch table.
+
+ """
+ return dict((op, getattr(cmdlib, _LUNameForOpName(op.__name__)))
+ for op in opcodes.OP_MAPPING.values()
+ if op.WITH_LU)
+
+
class Processor(object):
"""Object which runs OpCodes"""
- DISPATCH_TABLE = {
- # Cluster
- opcodes.OpPostInitCluster: cmdlib.LUPostInitCluster,
- opcodes.OpDestroyCluster: cmdlib.LUDestroyCluster,
- opcodes.OpQueryClusterInfo: cmdlib.LUQueryClusterInfo,
- opcodes.OpVerifyCluster: cmdlib.LUVerifyCluster,
- opcodes.OpQueryConfigValues: cmdlib.LUQueryConfigValues,
- opcodes.OpRenameCluster: cmdlib.LURenameCluster,
- opcodes.OpVerifyDisks: cmdlib.LUVerifyDisks,
- opcodes.OpSetClusterParams: cmdlib.LUSetClusterParams,
- opcodes.OpRedistributeConfig: cmdlib.LURedistributeConfig,
- opcodes.OpRepairDiskSizes: cmdlib.LURepairDiskSizes,
- # node lu
- opcodes.OpAddNode: cmdlib.LUAddNode,
- opcodes.OpQueryNodes: cmdlib.LUQueryNodes,
- opcodes.OpQueryNodeVolumes: cmdlib.LUQueryNodeVolumes,
- opcodes.OpQueryNodeStorage: cmdlib.LUQueryNodeStorage,
- opcodes.OpModifyNodeStorage: cmdlib.LUModifyNodeStorage,
- opcodes.OpRepairNodeStorage: cmdlib.LURepairNodeStorage,
- opcodes.OpRemoveNode: cmdlib.LURemoveNode,
- opcodes.OpSetNodeParams: cmdlib.LUSetNodeParams,
- opcodes.OpPowercycleNode: cmdlib.LUPowercycleNode,
- opcodes.OpEvacuateNode: cmdlib.LUEvacuateNode,
- opcodes.OpMigrateNode: cmdlib.LUMigrateNode,
- # instance lu
- opcodes.OpCreateInstance: cmdlib.LUCreateInstance,
- opcodes.OpReinstallInstance: cmdlib.LUReinstallInstance,
- opcodes.OpRemoveInstance: cmdlib.LURemoveInstance,
- opcodes.OpRenameInstance: cmdlib.LURenameInstance,
- opcodes.OpActivateInstanceDisks: cmdlib.LUActivateInstanceDisks,
- opcodes.OpShutdownInstance: cmdlib.LUShutdownInstance,
- opcodes.OpStartupInstance: cmdlib.LUStartupInstance,
- opcodes.OpRebootInstance: cmdlib.LURebootInstance,
- opcodes.OpDeactivateInstanceDisks: cmdlib.LUDeactivateInstanceDisks,
- opcodes.OpReplaceDisks: cmdlib.LUReplaceDisks,
- opcodes.OpRecreateInstanceDisks: cmdlib.LURecreateInstanceDisks,
- opcodes.OpFailoverInstance: cmdlib.LUFailoverInstance,
- opcodes.OpMigrateInstance: cmdlib.LUMigrateInstance,
- opcodes.OpMoveInstance: cmdlib.LUMoveInstance,
- opcodes.OpConnectConsole: cmdlib.LUConnectConsole,
- opcodes.OpQueryInstances: cmdlib.LUQueryInstances,
- opcodes.OpQueryInstanceData: cmdlib.LUQueryInstanceData,
- opcodes.OpSetInstanceParams: cmdlib.LUSetInstanceParams,
- opcodes.OpGrowDisk: cmdlib.LUGrowDisk,
- # os lu
- opcodes.OpDiagnoseOS: cmdlib.LUDiagnoseOS,
- # exports lu
- opcodes.OpQueryExports: cmdlib.LUQueryExports,
- opcodes.OpExportInstance: cmdlib.LUExportInstance,
- opcodes.OpRemoveExport: cmdlib.LURemoveExport,
- # tags lu
- opcodes.OpGetTags: cmdlib.LUGetTags,
- opcodes.OpSearchTags: cmdlib.LUSearchTags,
- opcodes.OpAddTags: cmdlib.LUAddTags,
- opcodes.OpDelTags: cmdlib.LUDelTags,
- # test lu
- opcodes.OpTestDelay: cmdlib.LUTestDelay,
- opcodes.OpTestAllocator: cmdlib.LUTestAllocator,
- }
-
- def __init__(self, context):
+ DISPATCH_TABLE = _ComputeDispatchTable()
+
+ def __init__(self, context, ec_id):
"""Constructor for Processor
+ @type context: GanetiContext
+ @param context: global Ganeti context
+ @type ec_id: string
+ @param ec_id: execution context identifier
+
"""
self.context = context
+ self._ec_id = ec_id
self._cbs = None
- self.exclusive_BGL = False
self.rpc = rpc.RpcRunner(context.cfg)
self.hmclass = HooksMaster
- def _ReportLocks(self, level, names, shared, acquired):
- """Reports lock operations.
+ def _AcquireLocks(self, level, names, shared, timeout, priority):
+ """Acquires locks via the Ganeti lock manager.
@type level: int
@param level: Lock level
@type names: list or string
@param names: Lock names
@type shared: bool
- @param shared: Whether the lock should be acquired in shared mode
- @type acquired: bool
- @param acquired: Whether the lock has already been acquired
+ @param shared: Whether the locks should be acquired in shared mode
+ @type timeout: None or float
+ @param timeout: Timeout for acquiring the locks
+ @raise LockAcquireTimeout: In case locks couldn't be acquired in specified
+ amount of time
"""
- parts = []
-
- # Build message
- if acquired:
- parts.append("acquired")
- else:
- parts.append("waiting")
-
- parts.append(locking.LEVEL_NAMES[level])
+ if self._cbs:
+ self._cbs.CheckCancel()
- if names == locking.ALL_SET:
- parts.append("ALL")
- elif isinstance(names, basestring):
- parts.append(names)
- else:
- parts.append(",".join(names))
+ acquired = self.context.glm.acquire(level, names, shared=shared,
+ timeout=timeout, priority=priority)
- if shared:
- parts.append("shared")
- else:
- parts.append("exclusive")
+ if acquired is None:
+ raise LockAcquireTimeout()
- msg = "/".join(parts)
-
- logging.debug("LU locks %s", msg)
-
- if self._cbs:
- self._cbs.ReportLocks(msg)
+ return acquired
def _ExecLU(self, lu):
"""Logical Unit execution sequence.
hm = HooksMaster(self.rpc.call_hooks_runner, lu)
h_results = hm.RunPhase(constants.HOOKS_PHASE_PRE)
lu.HooksCallBack(constants.HOOKS_PHASE_PRE, h_results,
- self._Feedback, None)
+ self.Log, None)
if getattr(lu.op, "dry_run", False):
# in this mode, no post-hooks are run, and the config is not
return lu.dry_run_result
try:
- result = lu.Exec(self._Feedback)
+ result = lu.Exec(self.Log)
h_results = hm.RunPhase(constants.HOOKS_PHASE_POST)
result = lu.HooksCallBack(constants.HOOKS_PHASE_POST, h_results,
- self._Feedback, result)
+ self.Log, result)
finally:
# FIXME: This needs locks if not lu_class.REQ_BGL
if write_count != self.context.cfg.write_count:
return result
- def _LockAndExecLU(self, lu, level):
+ def _LockAndExecLU(self, lu, level, calc_timeout, priority):
"""Execute a Logical Unit, with the needed locks.
This is a recursive function that starts locking the given level, and
self._cbs.NotifyStart()
result = self._ExecLU(lu)
+
elif adding_locks and acquiring_locks:
# We could both acquire and add locks at the same level, but for now we
# don't need this, so we'll avoid the complicated code needed.
- raise NotImplementedError(
- "Can't declare locks to acquire when adding others")
+ raise NotImplementedError("Can't declare locks to acquire when adding"
+ " others")
+
elif adding_locks or acquiring_locks:
lu.DeclareLocks(level)
share = lu.share_locks[level]
- if acquiring_locks:
- needed_locks = lu.needed_locks[level]
-
- self._ReportLocks(level, needed_locks, share, False)
- lu.acquired_locks[level] = self.context.glm.acquire(level,
- needed_locks,
- shared=share)
- self._ReportLocks(level, needed_locks, share, True)
-
- else: # adding_locks
- add_locks = lu.add_locks[level]
- lu.remove_locks[level] = add_locks
- try:
- self.context.glm.add(level, add_locks, acquired=1, shared=share)
- except errors.LockError:
- raise errors.OpPrereqError(
- "Couldn't add locks (%s), probably because of a race condition"
- " with another job, who added them first" % add_locks)
+
try:
+ assert adding_locks ^ acquiring_locks, \
+ "Locks must be either added or acquired"
+
+ if acquiring_locks:
+ # Acquiring locks
+ needed_locks = lu.needed_locks[level]
+
+ acquired = self._AcquireLocks(level, needed_locks, share,
+ calc_timeout(), priority)
+ else:
+ # Adding locks
+ add_locks = lu.add_locks[level]
+ lu.remove_locks[level] = add_locks
+
+ try:
+ self.context.glm.add(level, add_locks, acquired=1, shared=share)
+ except errors.LockError:
+ raise errors.OpPrereqError(
+ "Couldn't add locks (%s), probably because of a race condition"
+ " with another job, who added them first" % add_locks,
+ errors.ECODE_FAULT)
+
+ acquired = add_locks
+
try:
- if adding_locks:
- lu.acquired_locks[level] = add_locks
- result = self._LockAndExecLU(lu, level + 1)
+ lu.acquired_locks[level] = acquired
+
+ result = self._LockAndExecLU(lu, level + 1, calc_timeout, priority)
finally:
if level in lu.remove_locks:
self.context.glm.remove(level, lu.remove_locks[level])
finally:
if self.context.glm.is_owned(level):
self.context.glm.release(level)
+
else:
- result = self._LockAndExecLU(lu, level + 1)
+ result = self._LockAndExecLU(lu, level + 1, calc_timeout, priority)
return result
- def ExecOpCode(self, op, cbs):
+ def ExecOpCode(self, op, cbs, timeout=None, priority=None):
"""Execute an opcode.
@type op: an OpCode instance
@param op: the opcode to be executed
@type cbs: L{OpExecCbBase}
@param cbs: Runtime callbacks
+ @type timeout: float or None
+ @param timeout: Maximum time to acquire all locks, None for no timeout
+ @type priority: number or None
+ @param priority: Priority for acquiring lock(s)
+ @raise LockAcquireTimeout: In case locks couldn't be acquired in specified
+ amount of time
"""
if not isinstance(op, opcodes.OpCode):
raise errors.ProgrammerError("Non-opcode instance passed"
" to ExecOpcode")
+ lu_class = self.DISPATCH_TABLE.get(op.__class__, None)
+ if lu_class is None:
+ raise errors.OpCodeUnknown("Unknown opcode")
+
+ if timeout is None:
+ calc_timeout = lambda: None
+ else:
+ calc_timeout = utils.RunningTimeout(timeout, False).Remaining
+
self._cbs = cbs
try:
- lu_class = self.DISPATCH_TABLE.get(op.__class__, None)
- if lu_class is None:
- raise errors.OpCodeUnknown("Unknown opcode")
-
- # Acquire the Big Ganeti Lock exclusively if this LU requires it, and in a
- # shared fashion otherwise (to prevent concurrent run with an exclusive
- # LU.
- self._ReportLocks(locking.LEVEL_CLUSTER, [locking.BGL],
- not lu_class.REQ_BGL, False)
- try:
- self.context.glm.acquire(locking.LEVEL_CLUSTER, [locking.BGL],
- shared=not lu_class.REQ_BGL)
- finally:
- self._ReportLocks(locking.LEVEL_CLUSTER, [locking.BGL],
- not lu_class.REQ_BGL, True)
+ # Acquire the Big Ganeti Lock exclusively if this LU requires it,
+ # and in a shared fashion otherwise (to prevent concurrent run with
+ # an exclusive LU.
+ self._AcquireLocks(locking.LEVEL_CLUSTER, locking.BGL,
+ not lu_class.REQ_BGL, calc_timeout(),
+ priority)
try:
- self.exclusive_BGL = lu_class.REQ_BGL
lu = lu_class(self, op, self.context, self.rpc)
lu.ExpandNames()
assert lu.needed_locks is not None, "needed_locks not set by LU"
- result = self._LockAndExecLU(lu, locking.LEVEL_INSTANCE)
+
+ try:
+ return self._LockAndExecLU(lu, locking.LEVEL_INSTANCE, calc_timeout,
+ priority)
+ finally:
+ if self._ec_id:
+ self.context.cfg.DropECReservations(self._ec_id)
finally:
self.context.glm.release(locking.LEVEL_CLUSTER)
- self.exclusive_BGL = False
finally:
self._cbs = None
- return result
-
- def _Feedback(self, *args):
+ def Log(self, *args):
"""Forward call to feedback callback function.
"""
"""
logging.debug("Step %d/%d %s", current, total, message)
- self._Feedback("STEP %d/%d %s" % (current, total, message))
+ self.Log("STEP %d/%d %s" % (current, total, message))
def LogWarning(self, message, *args, **kwargs):
"""Log a warning to the logs and the user.
message = message % tuple(args)
if message:
logging.warning(message)
- self._Feedback(" - WARNING: %s" % message)
+ self.Log(" - WARNING: %s" % message)
if "hint" in kwargs:
- self._Feedback(" Hint: %s" % kwargs["hint"])
+ self.Log(" Hint: %s" % kwargs["hint"])
def LogInfo(self, message, *args):
"""Log an informational message to the logs and the user.
if args:
message = message % tuple(args)
logging.info(message)
- self._Feedback(" - INFO: %s" % message)
+ self.Log(" - INFO: %s" % message)
+
+ def GetECId(self):
+ """Returns the current execution context ID.
+
+ """
+ if not self._ec_id:
+ raise errors.ProgrammerError("Tried to use execution context id when"
+ " not set")
+ return self._ec_id
class HooksMaster(object):
projects / ganeti-local / blobdiff