Statistics
| Branch: | Tag: | Revision:

root / qa / qa_instance.py @ d488adb6

History | View | Annotate | Download (18.8 kB)

1
#
2
#
3

    
4
# Copyright (C) 2007, 2011, 2012 Google Inc.
5
#
6
# This program is free software; you can redistribute it and/or modify
7
# it under the terms of the GNU General Public License as published by
8
# the Free Software Foundation; either version 2 of the License, or
9
# (at your option) any later version.
10
#
11
# This program is distributed in the hope that it will be useful, but
12
# WITHOUT ANY WARRANTY; without even the implied warranty of
13
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14
# General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License
17
# along with this program; if not, write to the Free Software
18
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
19
# 02110-1301, USA.
20

    
21

    
22
"""Instance related QA tests.
23

24
"""
25

    
26
import re
27
import time
28

    
29
from ganeti import utils
30
from ganeti import constants
31
from ganeti import query
32

    
33
import qa_config
34
import qa_utils
35
import qa_error
36

    
37
from qa_utils import AssertIn, AssertCommand, AssertEqual
38
from qa_utils import InstanceCheck, INST_DOWN, INST_UP, FIRST_ARG, RETURN_VALUE
39

    
40

    
41
def _GetDiskStatePath(disk):
42
  return "/sys/block/%s/device/state" % disk
43

    
44

    
45
def _GetGenericAddParameters(inst, force_mac=None):
46
  params = ["-B"]
47
  params.append("%s=%s,%s=%s" % (constants.BE_MINMEM,
48
                                 qa_config.get(constants.BE_MINMEM),
49
                                 constants.BE_MAXMEM,
50
                                 qa_config.get(constants.BE_MAXMEM)))
51
  for idx, size in enumerate(qa_config.get("disk")):
52
    params.extend(["--disk", "%s:size=%s" % (idx, size)])
53

    
54
  # Set static MAC address if configured
55
  if force_mac:
56
    nic0_mac = force_mac
57
  else:
58
    nic0_mac = qa_config.GetInstanceNicMac(inst)
59
  if nic0_mac:
60
    params.extend(["--net", "0:mac=%s" % nic0_mac])
61

    
62
  return params
63

    
64

    
65
def _DiskTest(node, disk_template):
66
  instance = qa_config.AcquireInstance()
67
  try:
68
    cmd = (["gnt-instance", "add",
69
            "--os-type=%s" % qa_config.get("os"),
70
            "--disk-template=%s" % disk_template,
71
            "--node=%s" % node] +
72
           _GetGenericAddParameters(instance))
73
    cmd.append(instance["name"])
74

    
75
    AssertCommand(cmd)
76

    
77
    _CheckSsconfInstanceList(instance["name"])
78

    
79
    return instance
80
  except:
81
    qa_config.ReleaseInstance(instance)
82
    raise
83

    
84

    
85
@InstanceCheck(None, INST_UP, RETURN_VALUE)
86
def TestInstanceAddWithPlainDisk(node):
87
  """gnt-instance add -t plain"""
88
  return _DiskTest(node["primary"], "plain")
89

    
90

    
91
@InstanceCheck(None, INST_UP, RETURN_VALUE)
92
def TestInstanceAddWithDrbdDisk(node, node2):
93
  """gnt-instance add -t drbd"""
94
  return _DiskTest("%s:%s" % (node["primary"], node2["primary"]),
95
                   "drbd")
96

    
97

    
98
@InstanceCheck(None, INST_DOWN, FIRST_ARG)
99
def TestInstanceRemove(instance):
100
  """gnt-instance remove"""
101
  AssertCommand(["gnt-instance", "remove", "-f", instance["name"]])
102

    
103
  qa_config.ReleaseInstance(instance)
104

    
105

    
106
@InstanceCheck(INST_DOWN, INST_UP, FIRST_ARG)
107
def TestInstanceStartup(instance):
108
  """gnt-instance startup"""
109
  AssertCommand(["gnt-instance", "startup", instance["name"]])
110

    
111

    
112
@InstanceCheck(INST_UP, INST_DOWN, FIRST_ARG)
113
def TestInstanceShutdown(instance):
114
  """gnt-instance shutdown"""
115
  AssertCommand(["gnt-instance", "shutdown", instance["name"]])
116

    
117

    
118
@InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
119
def TestInstanceReboot(instance):
120
  """gnt-instance reboot"""
121
  options = qa_config.get("options", {})
122
  reboot_types = options.get("reboot-types", constants.REBOOT_TYPES)
123
  name = instance["name"]
124
  for rtype in reboot_types:
125
    AssertCommand(["gnt-instance", "reboot", "--type=%s" % rtype, name])
126

    
127
  AssertCommand(["gnt-instance", "shutdown", name])
128
  qa_utils.RunInstanceCheck(instance, False)
129
  AssertCommand(["gnt-instance", "reboot", name])
130

    
131
  master = qa_config.GetMasterNode()
132
  cmd = ["gnt-instance", "list", "--no-headers", "-o", "status", name]
133
  result_output = qa_utils.GetCommandOutput(master["primary"],
134
                                            utils.ShellQuoteArgs(cmd))
135
  AssertEqual(result_output.strip(), constants.INSTST_RUNNING)
136

    
137

    
138
@InstanceCheck(INST_DOWN, INST_DOWN, FIRST_ARG)
139
def TestInstanceReinstall(instance):
140
  """gnt-instance reinstall"""
141
  AssertCommand(["gnt-instance", "reinstall", "-f", instance["name"]])
142

    
143

    
144
def _ReadSsconfInstanceList():
145
  """Reads ssconf_instance_list from the master node.
146

147
  """
148
  master = qa_config.GetMasterNode()
149

    
150
  cmd = ["cat", utils.PathJoin(constants.DATA_DIR,
151
                               "ssconf_%s" % constants.SS_INSTANCE_LIST)]
152

    
153
  return qa_utils.GetCommandOutput(master["primary"],
154
                                   utils.ShellQuoteArgs(cmd)).splitlines()
155

    
156

    
157
def _CheckSsconfInstanceList(instance):
158
  """Checks if a certain instance is in the ssconf instance list.
159

160
  @type instance: string
161
  @param instance: Instance name
162

163
  """
164
  AssertIn(qa_utils.ResolveInstanceName(instance),
165
           _ReadSsconfInstanceList())
166

    
167

    
168
@InstanceCheck(INST_DOWN, INST_DOWN, FIRST_ARG)
169
def TestInstanceRenameAndBack(rename_source, rename_target):
170
  """gnt-instance rename
171

172
  This must leave the instance with the original name, not the target
173
  name.
174

175
  """
176
  _CheckSsconfInstanceList(rename_source)
177

    
178
  # first do a rename to a different actual name, expecting it to fail
179
  qa_utils.AddToEtcHosts(["meeeeh-not-exists", rename_target])
180
  try:
181
    AssertCommand(["gnt-instance", "rename", rename_source, rename_target],
182
                  fail=True)
183
    _CheckSsconfInstanceList(rename_source)
184
  finally:
185
    qa_utils.RemoveFromEtcHosts(["meeeeh-not-exists", rename_target])
186

    
187
  # and now rename instance to rename_target...
188
  AssertCommand(["gnt-instance", "rename", rename_source, rename_target])
189
  _CheckSsconfInstanceList(rename_target)
190
  qa_utils.RunInstanceCheck(rename_source, False)
191
  qa_utils.RunInstanceCheck(rename_target, False)
192

    
193
  # and back
194
  AssertCommand(["gnt-instance", "rename", rename_target, rename_source])
195
  _CheckSsconfInstanceList(rename_source)
196
  qa_utils.RunInstanceCheck(rename_target, False)
197

    
198

    
199
@InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
200
def TestInstanceFailover(instance):
201
  """gnt-instance failover"""
202
  cmd = ["gnt-instance", "failover", "--force", instance["name"]]
203

    
204
  # failover ...
205
  AssertCommand(cmd)
206
  qa_utils.RunInstanceCheck(instance, True)
207

    
208
  # ... and back
209
  AssertCommand(cmd)
210

    
211

    
212
@InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
213
def TestInstanceMigrate(instance):
214
  """gnt-instance migrate"""
215
  cmd = ["gnt-instance", "migrate", "--force", instance["name"]]
216

    
217
  # migrate ...
218
  AssertCommand(cmd)
219
  qa_utils.RunInstanceCheck(instance, True)
220

    
221
  # ... and back
222
  AssertCommand(cmd)
223

    
224
  # TODO: Split into multiple tests
225
  AssertCommand(["gnt-instance", "shutdown", instance["name"]])
226
  qa_utils.RunInstanceCheck(instance, False)
227
  AssertCommand(cmd, fail=True)
228
  AssertCommand(["gnt-instance", "migrate", "--force", "--allow-failover",
229
                 instance["name"]])
230
  AssertCommand(["gnt-instance", "start", instance["name"]])
231
  AssertCommand(cmd)
232
  qa_utils.RunInstanceCheck(instance, True)
233

    
234
  AssertCommand(["gnt-instance", "modify", "-B",
235
                 ("%s=%s" %
236
                  (constants.BE_ALWAYS_FAILOVER, constants.VALUE_TRUE)),
237
                 instance["name"]])
238

    
239
  AssertCommand(cmd, fail=True)
240
  qa_utils.RunInstanceCheck(instance, True)
241
  AssertCommand(["gnt-instance", "migrate", "--force", "--allow-failover",
242
                 instance["name"]])
243

    
244
  # TODO: Verify whether the default value is restored here (not hardcoded)
245
  AssertCommand(["gnt-instance", "modify", "-B",
246
                 ("%s=%s" %
247
                  (constants.BE_ALWAYS_FAILOVER, constants.VALUE_FALSE)),
248
                 instance["name"]])
249

    
250
  AssertCommand(cmd)
251
  qa_utils.RunInstanceCheck(instance, True)
252

    
253

    
254
def TestInstanceInfo(instance):
255
  """gnt-instance info"""
256
  AssertCommand(["gnt-instance", "info", instance["name"]])
257

    
258

    
259
@InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
260
def TestInstanceModify(instance):
261
  """gnt-instance modify"""
262
  default_hv = qa_config.GetDefaultHypervisor()
263

    
264
  # Assume /sbin/init exists on all systems
265
  test_kernel = "/sbin/init"
266
  test_initrd = test_kernel
267

    
268
  orig_maxmem = qa_config.get(constants.BE_MAXMEM)
269
  orig_minmem = qa_config.get(constants.BE_MINMEM)
270
  #orig_bridge = qa_config.get("bridge", "xen-br0")
271

    
272
  args = [
273
    ["-B", "%s=128" % constants.BE_MINMEM],
274
    ["-B", "%s=128" % constants.BE_MAXMEM],
275
    ["-B", "%s=%s,%s=%s" % (constants.BE_MINMEM, orig_minmem,
276
                            constants.BE_MAXMEM, orig_maxmem)],
277
    ["-B", "%s=2" % constants.BE_VCPUS],
278
    ["-B", "%s=1" % constants.BE_VCPUS],
279
    ["-B", "%s=%s" % (constants.BE_VCPUS, constants.VALUE_DEFAULT)],
280
    ["-B", "%s=%s" % (constants.BE_ALWAYS_FAILOVER, constants.VALUE_TRUE)],
281
    ["-B", "%s=%s" % (constants.BE_ALWAYS_FAILOVER, constants.VALUE_DEFAULT)],
282

    
283
    ["-H", "%s=%s" % (constants.HV_KERNEL_PATH, test_kernel)],
284
    ["-H", "%s=%s" % (constants.HV_KERNEL_PATH, constants.VALUE_DEFAULT)],
285

    
286
    # TODO: bridge tests
287
    #["--bridge", "xen-br1"],
288
    #["--bridge", orig_bridge],
289
    ]
290

    
291
  if default_hv == constants.HT_XEN_PVM:
292
    args.extend([
293
      ["-H", "%s=%s" % (constants.HV_INITRD_PATH, test_initrd)],
294
      ["-H", "no_%s" % (constants.HV_INITRD_PATH, )],
295
      ["-H", "%s=%s" % (constants.HV_INITRD_PATH, constants.VALUE_DEFAULT)],
296
      ])
297
  elif default_hv == constants.HT_XEN_HVM:
298
    args.extend([
299
      ["-H", "%s=acn" % constants.HV_BOOT_ORDER],
300
      ["-H", "%s=%s" % (constants.HV_BOOT_ORDER, constants.VALUE_DEFAULT)],
301
      ])
302

    
303
  for alist in args:
304
    AssertCommand(["gnt-instance", "modify"] + alist + [instance["name"]])
305

    
306
  # check no-modify
307
  AssertCommand(["gnt-instance", "modify", instance["name"]], fail=True)
308

    
309
  # Marking offline/online while instance is running must fail
310
  for arg in ["--online", "--offline"]:
311
    AssertCommand(["gnt-instance", "modify", arg, instance["name"]], fail=True)
312

    
313

    
314
@InstanceCheck(INST_DOWN, INST_DOWN, FIRST_ARG)
315
def TestInstanceStoppedModify(instance):
316
  """gnt-instance modify (stopped instance)"""
317
  name = instance["name"]
318

    
319
  # Instance was not marked offline; try marking it online once more
320
  AssertCommand(["gnt-instance", "modify", "--online", name])
321

    
322
  # Mark instance as offline
323
  AssertCommand(["gnt-instance", "modify", "--offline", name])
324

    
325
  # And online again
326
  AssertCommand(["gnt-instance", "modify", "--online", name])
327

    
328

    
329
@InstanceCheck(INST_DOWN, INST_DOWN, FIRST_ARG)
330
def TestInstanceConvertDisk(instance, snode):
331
  """gnt-instance modify -t"""
332
  name = instance["name"]
333
  AssertCommand(["gnt-instance", "modify", "-t", "plain", name])
334
  AssertCommand(["gnt-instance", "modify", "-t", "drbd",
335
                 "-n", snode["primary"], name])
336

    
337

    
338
@InstanceCheck(INST_DOWN, INST_DOWN, FIRST_ARG)
339
def TestInstanceGrowDisk(instance):
340
  """gnt-instance grow-disk"""
341
  name = instance["name"]
342
  all_size = qa_config.get("disk")
343
  all_grow = qa_config.get("disk-growth")
344
  if not all_grow:
345
    # missing disk sizes but instance grow disk has been enabled,
346
    # let's set fixed/nomimal growth
347
    all_grow = ["128M" for _ in all_size]
348
  for idx, (size, grow) in enumerate(zip(all_size, all_grow)):
349
    # succeed in grow by amount
350
    AssertCommand(["gnt-instance", "grow-disk", name, str(idx), grow])
351
    # fail in grow to the old size
352
    AssertCommand(["gnt-instance", "grow-disk", "--absolute", name, str(idx),
353
                   size], fail=True)
354
    # succeed to grow to old size + 2 * growth
355
    int_size = utils.ParseUnit(size)
356
    int_grow = utils.ParseUnit(grow)
357
    AssertCommand(["gnt-instance", "grow-disk", "--absolute", name, str(idx),
358
                   str(int_size + 2 * int_grow)])
359

    
360

    
361
def TestInstanceList():
362
  """gnt-instance list"""
363
  qa_utils.GenericQueryTest("gnt-instance", query.INSTANCE_FIELDS.keys())
364

    
365

    
366
def TestInstanceListFields():
367
  """gnt-instance list-fields"""
368
  qa_utils.GenericQueryFieldsTest("gnt-instance", query.INSTANCE_FIELDS.keys())
369

    
370

    
371
@InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
372
def TestInstanceConsole(instance):
373
  """gnt-instance console"""
374
  AssertCommand(["gnt-instance", "console", "--show-cmd", instance["name"]])
375

    
376

    
377
@InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
378
def TestReplaceDisks(instance, pnode, snode, othernode):
379
  """gnt-instance replace-disks"""
380
  # pylint: disable=W0613
381
  # due to unused pnode arg
382
  # FIXME: should be removed from the function completely
383
  def buildcmd(args):
384
    cmd = ["gnt-instance", "replace-disks"]
385
    cmd.extend(args)
386
    cmd.append(instance["name"])
387
    return cmd
388

    
389
  for data in [
390
    ["-p"],
391
    ["-s"],
392
    ["--new-secondary=%s" % othernode["primary"]],
393
    # and restore
394
    ["--new-secondary=%s" % snode["primary"]],
395
    ]:
396
    AssertCommand(buildcmd(data))
397

    
398
  AssertCommand(buildcmd(["-a"]))
399
  AssertCommand(["gnt-instance", "stop", instance["name"]])
400
  AssertCommand(buildcmd(["-a"]), fail=True)
401
  AssertCommand(["gnt-instance", "activate-disks", instance["name"]])
402
  AssertCommand(buildcmd(["-a"]))
403
  AssertCommand(["gnt-instance", "start", instance["name"]])
404

    
405

    
406
@InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
407
def TestInstanceExport(instance, node):
408
  """gnt-backup export -n ..."""
409
  name = instance["name"]
410
  AssertCommand(["gnt-backup", "export", "-n", node["primary"], name])
411
  return qa_utils.ResolveInstanceName(name)
412

    
413

    
414
@InstanceCheck(None, INST_DOWN, FIRST_ARG)
415
def TestInstanceExportWithRemove(instance, node):
416
  """gnt-backup export --remove-instance"""
417
  AssertCommand(["gnt-backup", "export", "-n", node["primary"],
418
                 "--remove-instance", instance["name"]])
419

    
420

    
421
@InstanceCheck(INST_UP, INST_UP, FIRST_ARG)
422
def TestInstanceExportNoTarget(instance):
423
  """gnt-backup export (without target node, should fail)"""
424
  AssertCommand(["gnt-backup", "export", instance["name"]], fail=True)
425

    
426

    
427
@InstanceCheck(None, INST_DOWN, FIRST_ARG)
428
def TestInstanceImport(newinst, node, expnode, name):
429
  """gnt-backup import"""
430
  cmd = (["gnt-backup", "import",
431
          "--disk-template=plain",
432
          "--no-ip-check",
433
          "--src-node=%s" % expnode["primary"],
434
          "--src-dir=%s/%s" % (constants.EXPORT_DIR, name),
435
          "--node=%s" % node["primary"]] +
436
         _GetGenericAddParameters(newinst, force_mac=constants.VALUE_GENERATE))
437
  cmd.append(newinst["name"])
438
  AssertCommand(cmd)
439

    
440

    
441
def TestBackupList(expnode):
442
  """gnt-backup list"""
443
  AssertCommand(["gnt-backup", "list", "--node=%s" % expnode["primary"]])
444

    
445
  qa_utils.GenericQueryTest("gnt-backup", query.EXPORT_FIELDS.keys(),
446
                            namefield=None, test_unknown=False)
447

    
448

    
449
def TestBackupListFields():
450
  """gnt-backup list-fields"""
451
  qa_utils.GenericQueryFieldsTest("gnt-backup", query.EXPORT_FIELDS.keys())
452

    
453

    
454
def _TestInstanceDiskFailure(instance, node, node2, onmaster):
455
  """Testing disk failure."""
456
  master = qa_config.GetMasterNode()
457
  sq = utils.ShellQuoteArgs
458

    
459
  instance_full = qa_utils.ResolveInstanceName(instance["name"])
460
  node_full = qa_utils.ResolveNodeName(node)
461
  node2_full = qa_utils.ResolveNodeName(node2)
462

    
463
  print qa_utils.FormatInfo("Getting physical disk names")
464
  cmd = ["gnt-node", "volumes", "--separator=|", "--no-headers",
465
         "--output=node,phys,instance",
466
         node["primary"], node2["primary"]]
467
  output = qa_utils.GetCommandOutput(master["primary"], sq(cmd))
468

    
469
  # Get physical disk names
470
  re_disk = re.compile(r"^/dev/([a-z]+)\d+$")
471
  node2disk = {}
472
  for line in output.splitlines():
473
    (node_name, phys, inst) = line.split("|")
474
    if inst == instance_full:
475
      if node_name not in node2disk:
476
        node2disk[node_name] = []
477

    
478
      m = re_disk.match(phys)
479
      if not m:
480
        raise qa_error.Error("Unknown disk name format: %s" % phys)
481

    
482
      name = m.group(1)
483
      if name not in node2disk[node_name]:
484
        node2disk[node_name].append(name)
485

    
486
  if [node2_full, node_full][int(onmaster)] not in node2disk:
487
    raise qa_error.Error("Couldn't find physical disks used on"
488
                         " %s node" % ["secondary", "master"][int(onmaster)])
489

    
490
  print qa_utils.FormatInfo("Checking whether nodes have ability to stop"
491
                            " disks")
492
  for node_name, disks in node2disk.iteritems():
493
    cmds = []
494
    for disk in disks:
495
      cmds.append(sq(["test", "-f", _GetDiskStatePath(disk)]))
496
    AssertCommand(" && ".join(cmds), node=node_name)
497

    
498
  print qa_utils.FormatInfo("Getting device paths")
499
  cmd = ["gnt-instance", "activate-disks", instance["name"]]
500
  output = qa_utils.GetCommandOutput(master["primary"], sq(cmd))
501
  devpath = []
502
  for line in output.splitlines():
503
    (_, _, tmpdevpath) = line.split(":")
504
    devpath.append(tmpdevpath)
505
  print devpath
506

    
507
  print qa_utils.FormatInfo("Getting drbd device paths")
508
  cmd = ["gnt-instance", "info", instance["name"]]
509
  output = qa_utils.GetCommandOutput(master["primary"], sq(cmd))
510
  pattern = (r"\s+-\s+sd[a-z]+,\s+type:\s+drbd8?,\s+.*$"
511
             r"\s+primary:\s+(/dev/drbd\d+)\s+")
512
  drbddevs = re.findall(pattern, output, re.M)
513
  print drbddevs
514

    
515
  halted_disks = []
516
  try:
517
    print qa_utils.FormatInfo("Deactivating disks")
518
    cmds = []
519
    for name in node2disk[[node2_full, node_full][int(onmaster)]]:
520
      halted_disks.append(name)
521
      cmds.append(sq(["echo", "offline"]) + " >%s" % _GetDiskStatePath(name))
522
    AssertCommand(" && ".join(cmds), node=[node2, node][int(onmaster)])
523

    
524
    print qa_utils.FormatInfo("Write to disks and give some time to notice"
525
                              " the problem")
526
    cmds = []
527
    for disk in devpath:
528
      cmds.append(sq(["dd", "count=1", "bs=512", "conv=notrunc",
529
                      "if=%s" % disk, "of=%s" % disk]))
530
    for _ in (0, 1, 2):
531
      AssertCommand(" && ".join(cmds), node=node)
532
      time.sleep(3)
533

    
534
    print qa_utils.FormatInfo("Debugging info")
535
    for name in drbddevs:
536
      AssertCommand(["drbdsetup", name, "show"], node=node)
537

    
538
    AssertCommand(["gnt-instance", "info", instance["name"]])
539

    
540
  finally:
541
    print qa_utils.FormatInfo("Activating disks again")
542
    cmds = []
543
    for name in halted_disks:
544
      cmds.append(sq(["echo", "running"]) + " >%s" % _GetDiskStatePath(name))
545
    AssertCommand("; ".join(cmds), node=[node2, node][int(onmaster)])
546

    
547
  if onmaster:
548
    for name in drbddevs:
549
      AssertCommand(["drbdsetup", name, "detach"], node=node)
550
  else:
551
    for name in drbddevs:
552
      AssertCommand(["drbdsetup", name, "disconnect"], node=node2)
553

    
554
  # TODO
555
  #AssertCommand(["vgs"], [node2, node][int(onmaster)])
556

    
557
  print qa_utils.FormatInfo("Making sure disks are up again")
558
  AssertCommand(["gnt-instance", "replace-disks", instance["name"]])
559

    
560
  print qa_utils.FormatInfo("Restarting instance")
561
  AssertCommand(["gnt-instance", "shutdown", instance["name"]])
562
  AssertCommand(["gnt-instance", "startup", instance["name"]])
563

    
564
  AssertCommand(["gnt-cluster", "verify"])
565

    
566

    
567
def TestInstanceMasterDiskFailure(instance, node, node2):
568
  """Testing disk failure on master node."""
569
  # pylint: disable=W0613
570
  # due to unused args
571
  print qa_utils.FormatError("Disk failure on primary node cannot be"
572
                             " tested due to potential crashes.")
573
  # The following can cause crashes, thus it's disabled until fixed
574
  #return _TestInstanceDiskFailure(instance, node, node2, True)
575

    
576

    
577
def TestInstanceSecondaryDiskFailure(instance, node, node2):
578
  """Testing disk failure on secondary node."""
579
  return _TestInstanceDiskFailure(instance, node, node2, False)