74 |
74 |
return "%s%s\n" % (prefix, ('\n' + prefix).join(s.splitlines()))
|
75 |
75 |
|
76 |
76 |
|
77 |
|
def DoCmd(cmd):
|
78 |
|
"""Run a shell command.
|
79 |
|
|
80 |
|
Args:
|
81 |
|
cmd: the command to run.
|
82 |
|
|
83 |
|
Raises CommandError with verbose commentary on error.
|
84 |
|
|
85 |
|
"""
|
86 |
|
res = utils.RunCmd(cmd)
|
87 |
|
|
88 |
|
if res.failed:
|
89 |
|
msg = ("Command %s failed:\n%s\nstdout:\n%sstderr:\n%s" %
|
90 |
|
(repr(cmd),
|
91 |
|
Indent(res.fail_reason),
|
92 |
|
Indent(res.stdout),
|
93 |
|
Indent(res.stderr)))
|
94 |
|
raise errors.CommandError(msg)
|
95 |
|
|
96 |
|
return res
|
97 |
|
|
98 |
|
|
99 |
77 |
class WatcherState(object):
|
100 |
78 |
"""Interface to a state file recording restart attempts.
|
101 |
79 |
|
... | ... | |
399 |
377 |
"""Run gnt-cluster verify-disks.
|
400 |
378 |
|
401 |
379 |
"""
|
402 |
|
# TODO: What should we do here?
|
403 |
|
result = DoCmd(['gnt-cluster', 'verify-disks'])
|
404 |
|
if result.output:
|
405 |
|
logging.info(result.output)
|
|
380 |
op = opcodes.OpVerifyDisks()
|
|
381 |
result = cli.SubmitOpCode(op, cl=client)
|
|
382 |
if not isinstance(result, (tuple, list)):
|
|
383 |
logging.error("Can't get a valid result from verify-disks")
|
|
384 |
return
|
|
385 |
offline_disk_instances = result[2]
|
|
386 |
if not offline_disk_instances:
|
|
387 |
# nothing to do
|
|
388 |
return
|
|
389 |
logging.debug("Will activate disks for instances %s",
|
|
390 |
", ".join(offline_disk_instances))
|
|
391 |
# we submit only one job, and wait for it. not optimal, but spams
|
|
392 |
# less the job queue
|
|
393 |
job = [opcodes.OpActivateInstanceDisks(instance_name=name)
|
|
394 |
for name in offline_disk_instances]
|
|
395 |
job_id = cli.SendJob(job, cl=client)
|
|
396 |
|
|
397 |
cli.PollJob(job_id, cl=client, feedback_fn=logging.debug)
|
406 |
398 |
|
407 |
399 |
|
408 |
400 |
def ParseOptions():
|
... | ... | |
432 |
424 |
|
433 |
425 |
options, args = ParseOptions()
|
434 |
426 |
|
435 |
|
logger.SetupLogging(constants.LOG_WATCHER, debug=options.debug)
|
|
427 |
logger.SetupLogging(constants.LOG_WATCHER, debug=options.debug,
|
|
428 |
stderr_logging=options.debug)
|
436 |
429 |
|
437 |
430 |
try:
|
438 |
431 |
client = cli.GetClient()
|