Synnefo » snf-ganeti » ganeti-local

-- | Our ADT for the OpCode status at runtime (while in a job).

data OpStatus = OP_STATUS_QUEUED

              | OP_STATUS_CANCELING

              | OP_STATUS_RUNNING

              | OP_STATUS_CANCELED

    showJSON os = showJSON w

      where w = case os of

              OP_STATUS_QUEUED -> C.jobStatusQueued

              OP_STATUS_CANCELING -> C.jobStatusCanceling

              OP_STATUS_RUNNING -> C.jobStatusRunning

              OP_STATUS_CANCELED -> C.jobStatusCanceled

              OP_STATUS_ERROR -> C.jobStatusError

    readJSON s = case readJSON s of

      J.Ok v | v == C.jobStatusQueued -> J.Ok OP_STATUS_QUEUED

             | v == C.jobStatusCanceling -> J.Ok OP_STATUS_CANCELING

             | v == C.jobStatusRunning -> J.Ok OP_STATUS_RUNNING

             | v == C.jobStatusCanceled -> J.Ok OP_STATUS_CANCELED

-- such that greater\/lesser comparison on values of this type makes

-- sense.

data JobStatus = JOB_STATUS_QUEUED

               | JOB_STATUS_RUNNING

               | JOB_STATUS_SUCCESS

               | JOB_STATUS_CANCELING

    showJSON js = showJSON w

        where w = case js of

                JOB_STATUS_QUEUED -> "queued"

                JOB_STATUS_CANCELING -> "canceling"

                JOB_STATUS_RUNNING -> "running"

                JOB_STATUS_CANCELED -> "canceled"

                JOB_STATUS_ERROR -> "error"

    readJSON s = case readJSON s of

      J.Ok "queued" -> J.Ok JOB_STATUS_QUEUED

      J.Ok "canceling" -> J.Ok JOB_STATUS_CANCELING

      J.Ok "running" -> J.Ok JOB_STATUS_RUNNING

      J.Ok "success" -> J.Ok JOB_STATUS_SUCCESS

      ToStderr("Job %s is waiting in queue", job_id)

      self.notified_queued = True

      ToStderr("Job %s is trying to acquire all necessary locks", job_id)

      self.notified_waitlock = True

    for job_data, status in zip(self.jobs, result):

      if (isinstance(status, list) and status and

          status[0] in (constants.JOB_STATUS_QUEUED,

                        constants.JOB_STATUS_CANCELING)):

        # job is still present and waiting

        continue

      logging.debug("Status of job %s is %s", job_id, status)

      if test == constants.JQT_EXPANDNAMES:

          raise errors.OpExecError("Job status while expanding names is '%s',"

                                   " not '%s' as expected" %

      elif test in (constants.JQT_EXEC, constants.JQT_LOGMSG):

        if status != constants.JOB_STATUS_RUNNING:

          raise errors.OpExecError("Job status while executing opcode is '%s',"

#: names

_USER_JOB_STATUS = {

  constants.JOB_STATUS_QUEUED: "queued",

  constants.JOB_STATUS_CANCELING: "canceling",

  constants.JOB_STATUS_RUNNING: "running",

  constants.JOB_STATUS_CANCELED: "canceled",

# Job status

JOB_STATUS_QUEUED = "queued"

JOB_STATUS_CANCELING = "canceling"

JOB_STATUS_RUNNING = "running"

JOB_STATUS_CANCELED = "canceled"

  ])

JOB_STATUS_ALL = frozenset([

  JOB_STATUS_QUEUED,

  JOB_STATUS_CANCELING,

  JOB_STATUS_RUNNING,

  ]) | JOBS_FINALIZED

# OpCode status

# not yet finalized

OP_STATUS_QUEUED = "queued"

OP_STATUS_CANCELING = "canceling"

OP_STATUS_RUNNING = "running"

# finalized

      if op.status == constants.OP_STATUS_QUEUED:

        pass

      elif op.status == constants.OP_STATUS_RUNNING:

        status = constants.JOB_STATUS_RUNNING

      elif op.status == constants.OP_STATUS_CANCELING:

      self.Finalize()

      return (True, "Job %s canceled" % self.id)

      # The worker will notice the new status and cancel the job

      self.MarkUnfinishedOps(constants.OP_STATUS_CANCELING, None)

      return (True, "Job %s will be canceled" % self.id)

    This is called from the mcpu code as a notifier function, when the LU is

    finally about to start the Exec() method. Of course, to have end-user

    visible results, the opcode must be initially (before calling into

    """

    assert self._op in self._job.ops

                               constants.OP_STATUS_CANCELING)

    # Cancel here if we were asked to

    """Check whether job has been cancelled.

    """

                               constants.OP_STATUS_CANCELING)

    # Cancel here if we were asked to

    # no changes.

    if (status not in (constants.JOB_STATUS_QUEUED,

                       constants.JOB_STATUS_RUNNING,

        job_info != self._prev_job_info or

        (log_entries and self._prev_log_serial != log_entries[0][0])):

      logging.debug("Job %s changed", job.id)

    """

    assert op in job.ops

    assert op.status in (constants.OP_STATUS_QUEUED,

    update = False

    op.result = None

    if op.status == constants.OP_STATUS_QUEUED:

      update = True

    if op.start_timestamp is None:

      job.start_timestamp = op.start_timestamp

      update = True

    return update

    """

    op = opctx.op

    timeout = opctx.GetNextLockTimeout()

      assert timeout is not None, "Received timeout for blocking acquire"

      logging.debug("Couldn't acquire locks in %0.6fs", timeout)

                           constants.OP_STATUS_CANCELING)

      # Was job cancelled while we were waiting for the lock?

        return (constants.OP_STATUS_CANCELING, None)

      # Stay in waitlock while trying to re-acquire lock

    except CancelJob:

      logging.exception("%s: Canceling job", opctx.log_prefix)

      assert op.status == constants.OP_STATUS_CANCELING

                        for i in job.ops[opctx.index + 1:])

      assert op.status in (constants.OP_STATUS_QUEUED,

                           constants.OP_STATUS_CANCELING)

      assert (op.priority <= constants.OP_PRIO_LOWEST and

      if op.status != constants.OP_STATUS_CANCELING:

        assert op.status in (constants.OP_STATUS_QUEUED,

        # Prepare to start opcode

        if self._MarkWaitlock(job, op):

          # Write to disk

          queue.UpdateJobUnlocked(job)

        assert job.start_timestamp and op.start_timestamp

        assert waitjob is None

        # Check if waiting for a job is necessary

        waitjob = self._CheckDependencies(queue, job, opctx)

                             constants.OP_STATUS_CANCELING,

                             constants.OP_STATUS_ERROR)

          assert not waitjob

          # Couldn't get locks in time

          assert not op.end_timestamp

        else:

          else:

            assert op.status in constants.OPS_FINALIZED

        finalize = False

        if not waitjob and opctx.CheckPriorityIncrease():

                op.priority >= constants.OP_PRIO_HIGHEST)

        # In no case must the status be finalized here

      else:

        # Ensure all opcodes so far have been successful

        restartjobs.append(job)

      elif status in (constants.JOB_STATUS_RUNNING,

                      constants.JOB_STATUS_CANCELING):

        logging.warning("Unfinished job %s found: %s", job.id, job)

          # Restart job

          job.MarkUnfinishedOps(constants.OP_STATUS_QUEUED, None)

          restartjobs.append(job)

NODE_ROLE_REGULAR = "regular"

JOB_STATUS_QUEUED = "queued"

JOB_STATUS_CANCELING = "canceling"

JOB_STATUS_RUNNING = "running"

JOB_STATUS_CANCELED = "canceled"

  ])

JOB_STATUS_ALL = frozenset([

  JOB_STATUS_QUEUED,

  JOB_STATUS_CANCELING,

  JOB_STATUS_RUNNING,

  ]) | JOB_STATUS_FINALIZED

# Internal constants

_REQ_DATA_VERSION_FIELD = "__version__"

_INST_CREATE_REQV1 = "instance-create-reqv1"

    shutil.rmtree(self.tmpdir)

  def _LoadWaitingJob(self):

  def _LoadLostJob(self):

    return None

    # No change

    self.assertEqual(wfjc(self.filename, self._LoadWaitingJob, ["status"],

                     constants.JOB_NOTCHANGED)

    # No previous information

    self.assertEqual(wfjc(self.filename, self._LoadWaitingJob,

                          ["status"], None, None, 1.0),

  def testLostJob(self):

    wfjc = jqueue._WaitForJobChangesHelper()

                              for op in ops))

    def _Waitlock1(ops):

    def _Waitlock2(ops):

      ops[0].status = constants.OP_STATUS_SUCCESS

      ops[1].status = constants.OP_STATUS_SUCCESS

    def _Running(ops):

      ops[0].status = constants.OP_STATUS_SUCCESS

    tests = {

      constants.JOB_STATUS_QUEUED: [_Queued],

      constants.JOB_STATUS_RUNNING: [_Running],

      constants.JOB_STATUS_CANCELING: [_Canceling1, _Canceling2],

      constants.JOB_STATUS_CANCELED: [_Canceled],

        self.assertEqual(queue.GetNextUpdate(), (job, True))

        self.assertRaises(IndexError, queue.GetNextUpdate)

        self.assertFalse(queue.IsAcquired())

        self.assertFalse(job.cur_opctx)

      def _AfterStart(op, cbs):

    self.assertEqual(job.CalcStatus(), constants.JOB_STATUS_QUEUED)

    assert len(job.ops) == 5

    # Mark as cancelling

    (success, _) = job.Cancel()

      self.assertEqual(queue.GetNextUpdate(), (job, True))

      self.assertRaises(IndexError, queue.GetNextUpdate)

      self.assertFalse(queue.IsAcquired())

      # Mark as cancelled

      (success, _) = job.Cancel()

    def _BeforeStart(timeout, priority):

      self.assertFalse(queue.IsAcquired())

      # Mark as cancelled

      (success, _) = job.Cancel()

      self.assertEqual(queue.GetNextUpdate(), (job, True))

      self.assertRaises(IndexError, queue.GetNextUpdate)

      self.assertFalse(queue.IsAcquired())

    def _AfterStart(op, cbs):

      self.assertEqual(queue.GetNextUpdate(), (job, True))

      self.assertEqual(queue.GetNextUpdate(), (job, True))

      self.assertRaises(IndexError, queue.GetNextUpdate)

      self.assertFalse(queue.IsAcquired())

      self.assertFalse(job.cur_opctx)

    def _AfterStart(op, cbs):

        self.assertEqual(queue.GetNextUpdate(), (job, True))

      self.assertRaises(IndexError, queue.GetNextUpdate)

      self.assertFalse(queue.IsAcquired())

      self.assertFalse(job.cur_opctx)

    def _AfterStart(op, cbs):

        # Simulate waiting for other job

        self.assertEqual(result, jqueue._JobProcessor.WAITDEP)

        self.assertTrue(job.cur_opctx)

        self.assertRaises(IndexError, depmgr.GetNextNotification)

        self.assert_(job.start_timestamp)

        self.assertFalse(job.end_timestamp)

        self.assertEqual(queue.GetNextUpdate(), (job, True))

      self.assertRaises(IndexError, queue.GetNextUpdate)

      self.assertFalse(queue.IsAcquired())

      self.assertFalse(job.cur_opctx)

    def _AfterStart(op, cbs):

        # Simulate waiting for other job

        self.assertEqual(result, jqueue._JobProcessor.WAITDEP)

        self.assertTrue(job.cur_opctx)

        self.assertRaises(IndexError, depmgr.GetNextNotification)

        self.assert_(job.start_timestamp)

        self.assertFalse(job.end_timestamp)

        self.assertEqual(queue.GetNextUpdate(), (job, True))

      self.assertRaises(IndexError, queue.GetNextUpdate)

      self.assertFalse(queue.IsAcquired())

      self.assertFalse(job.cur_opctx)

    def _AfterStart(op, cbs):

        # Simulate waiting for other job

        self.assertEqual(result, jqueue._JobProcessor.WAITDEP)

        self.assertTrue(job.cur_opctx)

        self.assertRaises(IndexError, depmgr.GetNextNotification)

        self.assert_(job.start_timestamp)

        self.assertFalse(job.end_timestamp)

    self.assertRaises(IndexError, self.queue.GetNextUpdate)

    self.assertFalse(self.queue.IsAcquired())

    ts = self.timeout_strategy

        self.assertEqual(job.cur_opctx._timeout_strategy._fn,

                         self.timeout_strategy.NextAttempt)

        self.assertFalse(job.ops[self.curop].exec_timestamp)

        # If priority has changed since acquiring locks, the job must've been

        # updated

    job_id = str(9610)

    dep_status = [constants.JOB_STATUS_CANCELED]

    (result, _) = self.jdm.CheckAndRegister(job, job_id, dep_status)

    self.assertEqual(result, self.jdm.WAIT)

    self.assertFalse(self._status)

    job_id = str(25519)

    dep_status = [constants.JOB_STATUS_ERROR]

    (result, _) = self.jdm.CheckAndRegister(job, job_id, dep_status)

    self.assertEqual(result, self.jdm.WAIT)

    self.assertFalse(self._status)

      job = self._FakeJob(21343)

      job_id = str(14609)

      (result, _) = self.jdm.CheckAndRegister(job, job_id, dep_status)

      self.assertEqual(result, self.jdm.WAIT)

      self.assertFalse(self._status)

    self.assertEqual(client._NODE_EVAC_RES1, rlib2._NODE_EVAC_RES1)

    self.assertEqual(client._INST_NIC_PARAMS, constants.INIC_PARAMS)

    self.assertEqual(client.JOB_STATUS_QUEUED, constants.JOB_STATUS_QUEUED)

    self.assertEqual(client.JOB_STATUS_CANCELING,

                     constants.JOB_STATUS_CANCELING)

    self.assertEqual(client.JOB_STATUS_RUNNING, constants.JOB_STATUS_RUNNING)

    self.assertEqual(client.JOB_STATUS_FINALIZED, constants.JOBS_FINALIZED)

    self.assertEqual(client.JOB_STATUS_ALL, constants.JOB_STATUS_ALL)

class RapiMockTest(unittest.TestCase):

  def test(self):

  def testWaitForJobCompletionNoChange(self):

    resp = serializer.DumpJson({

      })

    for retries in [1, 5, 25]: