Statistics
| Branch: | Tag: | Revision:

root / qa / qa_instance.py @ 8a4e8898

History | View | Annotate | Download (13.1 kB)

1
#
2
#
3

    
4
# Copyright (C) 2007 Google Inc.
5
#
6
# This program is free software; you can redistribute it and/or modify
7
# it under the terms of the GNU General Public License as published by
8
# the Free Software Foundation; either version 2 of the License, or
9
# (at your option) any later version.
10
#
11
# This program is distributed in the hope that it will be useful, but
12
# WITHOUT ANY WARRANTY; without even the implied warranty of
13
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14
# General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License
17
# along with this program; if not, write to the Free Software
18
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
19
# 02110-1301, USA.
20

    
21

    
22
"""Instance related QA tests.
23

24
"""
25

    
26
import re
27
import time
28

    
29
from ganeti import utils
30
from ganeti import constants
31

    
32
import qa_config
33
import qa_utils
34
import qa_error
35

    
36
from qa_utils import AssertEqual, AssertNotEqual, StartSSH
37

    
38

    
39
def _GetDiskStatePath(disk):
40
  return "/sys/block/%s/device/state" % disk
41

    
42

    
43
def _GetGenericAddParameters():
44
  return ['--os-size=%s' % qa_config.get('os-size'),
45
          '--swap-size=%s' % qa_config.get('swap-size'),
46
          '--memory=%s' % qa_config.get('mem')]
47

    
48

    
49
def _DiskTest(node, disk_template):
50
  master = qa_config.GetMasterNode()
51

    
52
  instance = qa_config.AcquireInstance()
53
  try:
54
    cmd = (['gnt-instance', 'add',
55
            '--os-type=%s' % qa_config.get('os'),
56
            '--disk-template=%s' % disk_template,
57
            '--node=%s' % node] +
58
           _GetGenericAddParameters())
59
    cmd.append(instance['name'])
60

    
61
    AssertEqual(StartSSH(master['primary'],
62
                         utils.ShellQuoteArgs(cmd)).wait(), 0)
63
    return instance
64
  except:
65
    qa_config.ReleaseInstance(instance)
66
    raise
67

    
68

    
69
@qa_utils.DefineHook('instance-add-plain-disk')
70
def TestInstanceAddWithPlainDisk(node):
71
  """gnt-instance add -t plain"""
72
  return _DiskTest(node['primary'], 'plain')
73

    
74

    
75
@qa_utils.DefineHook('instance-add-drbd-disk')
76
def TestInstanceAddWithDrbdDisk(node, node2):
77
  """gnt-instance add -t drbd"""
78
  return _DiskTest("%s:%s" % (node['primary'], node2['primary']),
79
                   'drbd')
80

    
81

    
82
@qa_utils.DefineHook('instance-remove')
83
def TestInstanceRemove(instance):
84
  """gnt-instance remove"""
85
  master = qa_config.GetMasterNode()
86

    
87
  cmd = ['gnt-instance', 'remove', '-f', instance['name']]
88
  AssertEqual(StartSSH(master['primary'],
89
                       utils.ShellQuoteArgs(cmd)).wait(), 0)
90

    
91
  qa_config.ReleaseInstance(instance)
92

    
93

    
94
@qa_utils.DefineHook('instance-startup')
95
def TestInstanceStartup(instance):
96
  """gnt-instance startup"""
97
  master = qa_config.GetMasterNode()
98

    
99
  cmd = ['gnt-instance', 'startup', instance['name']]
100
  AssertEqual(StartSSH(master['primary'],
101
                       utils.ShellQuoteArgs(cmd)).wait(), 0)
102

    
103

    
104
@qa_utils.DefineHook('instance-shutdown')
105
def TestInstanceShutdown(instance):
106
  """gnt-instance shutdown"""
107
  master = qa_config.GetMasterNode()
108

    
109
  cmd = ['gnt-instance', 'shutdown', instance['name']]
110
  AssertEqual(StartSSH(master['primary'],
111
                       utils.ShellQuoteArgs(cmd)).wait(), 0)
112

    
113

    
114
@qa_utils.DefineHook('instance-reboot')
115
def TestInstanceReboot(instance):
116
  """gnt-instance reboot"""
117
  master = qa_config.GetMasterNode()
118

    
119
  for reboottype in ["soft", "hard", "full"]:
120
    cmd = ['gnt-instance', 'reboot', '--type=%s' % reboottype,
121
           instance['name']]
122
    AssertEqual(StartSSH(master['primary'],
123
                         utils.ShellQuoteArgs(cmd)).wait(), 0)
124

    
125

    
126
@qa_utils.DefineHook('instance-reinstall')
127
def TestInstanceReinstall(instance):
128
  """gnt-instance reinstall"""
129
  master = qa_config.GetMasterNode()
130

    
131
  cmd = ['gnt-instance', 'reinstall', '-f', instance['name']]
132
  AssertEqual(StartSSH(master['primary'],
133
                       utils.ShellQuoteArgs(cmd)).wait(), 0)
134

    
135

    
136
@qa_utils.DefineHook('instance-failover')
137
def TestInstanceFailover(instance):
138
  """gnt-instance failover"""
139
  master = qa_config.GetMasterNode()
140

    
141
  cmd = ['gnt-instance', 'failover', '--force', instance['name']]
142
  AssertEqual(StartSSH(master['primary'],
143
                       utils.ShellQuoteArgs(cmd)).wait(), 0)
144

    
145

    
146
@qa_utils.DefineHook('instance-info')
147
def TestInstanceInfo(instance):
148
  """gnt-instance info"""
149
  master = qa_config.GetMasterNode()
150

    
151
  cmd = ['gnt-instance', 'info', instance['name']]
152
  AssertEqual(StartSSH(master['primary'],
153
                       utils.ShellQuoteArgs(cmd)).wait(), 0)
154

    
155

    
156
@qa_utils.DefineHook('instance-modify')
157
def TestInstanceModify(instance):
158
  """gnt-instance modify"""
159
  master = qa_config.GetMasterNode()
160

    
161
  orig_memory = qa_config.get('mem')
162
  orig_bridge = qa_config.get('bridge', 'xen-br0')
163
  args = [
164
    ["--memory", "128"],
165
    ["--memory", str(orig_memory)],
166
    ["--cpu", "2"],
167
    ["--cpu", "1"],
168
    ["--bridge", "xen-br1"],
169
    ["--bridge", orig_bridge],
170
    ["--kernel", "/dev/null"],
171
    ["--kernel", "default"],
172
    ["--initrd", "/dev/null"],
173
    ["--initrd", "none"],
174
    ["--initrd", "default"],
175
    ["--hvm-boot-order", "acn"],
176
    ["--hvm-boot-order", "default"],
177
    ]
178
  for alist in args:
179
    cmd = ['gnt-instance', 'modify'] + alist + [instance['name']]
180
    AssertEqual(StartSSH(master['primary'],
181
                         utils.ShellQuoteArgs(cmd)).wait(), 0)
182

    
183
  # check no-modify
184
  cmd = ['gnt-instance', 'modify', instance['name']]
185
  AssertNotEqual(StartSSH(master['primary'],
186
                          utils.ShellQuoteArgs(cmd)).wait(), 0)
187

    
188

    
189
@qa_utils.DefineHook('instance-list')
190
def TestInstanceList():
191
  """gnt-instance list"""
192
  master = qa_config.GetMasterNode()
193

    
194
  cmd = ['gnt-instance', 'list']
195
  AssertEqual(StartSSH(master['primary'],
196
                       utils.ShellQuoteArgs(cmd)).wait(), 0)
197

    
198

    
199
@qa_utils.DefineHook('instance-console')
200
def TestInstanceConsole(instance):
201
  """gnt-instance console"""
202
  master = qa_config.GetMasterNode()
203

    
204
  cmd = ['gnt-instance', 'console', '--show-cmd', instance['name']]
205
  AssertEqual(StartSSH(master['primary'],
206
                       utils.ShellQuoteArgs(cmd)).wait(), 0)
207

    
208

    
209
@qa_utils.DefineHook('instance-replace-disks')
210
def TestReplaceDisks(instance, pnode, snode, othernode):
211
  """gnt-instance replace-disks"""
212
  master = qa_config.GetMasterNode()
213

    
214
  def buildcmd(args):
215
    cmd = ['gnt-instance', 'replace-disks']
216
    cmd.extend(args)
217
    cmd.append(instance["name"])
218
    return cmd
219

    
220
  cmd = buildcmd(["-p"])
221
  AssertEqual(StartSSH(master['primary'],
222
                       utils.ShellQuoteArgs(cmd)).wait(), 0)
223

    
224
  cmd = buildcmd(["-s"])
225
  AssertEqual(StartSSH(master['primary'],
226
                       utils.ShellQuoteArgs(cmd)).wait(), 0)
227

    
228
  cmd = buildcmd(["--new-secondary=%s" % othernode["primary"]])
229
  AssertEqual(StartSSH(master['primary'],
230
                       utils.ShellQuoteArgs(cmd)).wait(), 0)
231

    
232
  # Restore
233
  cmd = buildcmd(["--new-secondary=%s" % snode["primary"]])
234
  AssertEqual(StartSSH(master['primary'],
235
                       utils.ShellQuoteArgs(cmd)).wait(), 0)
236

    
237

    
238
@qa_utils.DefineHook('backup-export')
239
def TestInstanceExport(instance, node):
240
  """gnt-backup export"""
241
  master = qa_config.GetMasterNode()
242

    
243
  cmd = ['gnt-backup', 'export', '-n', node['primary'], instance['name']]
244
  AssertEqual(StartSSH(master['primary'],
245
                       utils.ShellQuoteArgs(cmd)).wait(), 0)
246

    
247
  return qa_utils.ResolveInstanceName(instance)
248

    
249

    
250
@qa_utils.DefineHook('backup-import')
251
def TestInstanceImport(node, newinst, expnode, name):
252
  """gnt-backup import"""
253
  master = qa_config.GetMasterNode()
254

    
255
  cmd = (['gnt-backup', 'import',
256
          '--disk-template=plain',
257
          '--no-ip-check',
258
          '--src-node=%s' % expnode['primary'],
259
          '--src-dir=%s/%s' % (constants.EXPORT_DIR, name),
260
          '--node=%s' % node['primary']] +
261
         _GetGenericAddParameters())
262
  cmd.append(newinst['name'])
263
  AssertEqual(StartSSH(master['primary'],
264
                       utils.ShellQuoteArgs(cmd)).wait(), 0)
265

    
266

    
267
@qa_utils.DefineHook('backup-list')
268
def TestBackupList(expnode):
269
  """gnt-backup list"""
270
  master = qa_config.GetMasterNode()
271

    
272
  cmd = ['gnt-backup', 'list', '--node=%s' % expnode['primary']]
273
  AssertEqual(StartSSH(master['primary'],
274
                       utils.ShellQuoteArgs(cmd)).wait(), 0)
275

    
276

    
277
def _TestInstanceDiskFailure(instance, node, node2, onmaster):
278
  """Testing disk failure."""
279
  master = qa_config.GetMasterNode()
280
  sq = utils.ShellQuoteArgs
281

    
282
  instance_full = qa_utils.ResolveInstanceName(instance)
283
  node_full = qa_utils.ResolveNodeName(node)
284
  node2_full = qa_utils.ResolveNodeName(node2)
285

    
286
  print qa_utils.FormatInfo("Getting physical disk names")
287
  cmd = ['gnt-node', 'volumes', '--separator=|', '--no-headers',
288
         '--output=node,phys,instance',
289
         node['primary'], node2['primary']]
290
  output = qa_utils.GetCommandOutput(master['primary'], sq(cmd))
291

    
292
  # Get physical disk names
293
  re_disk = re.compile(r'^/dev/([a-z]+)\d+$')
294
  node2disk = {}
295
  for line in output.splitlines():
296
    (node_name, phys, inst) = line.split('|')
297
    if inst == instance_full:
298
      if node_name not in node2disk:
299
        node2disk[node_name] = []
300

    
301
      m = re_disk.match(phys)
302
      if not m:
303
        raise qa_error.Error("Unknown disk name format: %s" % disk)
304

    
305
      name = m.group(1)
306
      if name not in node2disk[node_name]:
307
        node2disk[node_name].append(name)
308

    
309
  if [node2_full, node_full][int(onmaster)] not in node2disk:
310
    raise qa_error.Error("Couldn't find physical disks used on"
311
                         " %s node" % ["secondary", "master"][int(onmaster)])
312

    
313
  print qa_utils.FormatInfo("Checking whether nodes have ability to stop"
314
                            " disks")
315
  for node_name, disks in node2disk.iteritems():
316
    cmds = []
317
    for disk in disks:
318
      cmds.append(sq(["test", "-f", _GetDiskStatePath(disk)]))
319
    AssertEqual(StartSSH(node_name, ' && '.join(cmds)).wait(), 0)
320

    
321
  print qa_utils.FormatInfo("Getting device paths")
322
  cmd = ['gnt-instance', 'activate-disks', instance['name']]
323
  output = qa_utils.GetCommandOutput(master['primary'], sq(cmd))
324
  devpath = []
325
  for line in output.splitlines():
326
    (_, _, tmpdevpath) = line.split(':')
327
    devpath.append(tmpdevpath)
328
  print devpath
329

    
330
  print qa_utils.FormatInfo("Getting drbd device paths")
331
  cmd = ['gnt-instance', 'info', instance['name']]
332
  output = qa_utils.GetCommandOutput(master['primary'], sq(cmd))
333
  pattern = (r'\s+-\s+sd[a-z]+,\s+type:\s+drbd8?,\s+.*$'
334
             r'\s+primary:\s+(/dev/drbd\d+)\s+')
335
  drbddevs = re.findall(pattern, output, re.M)
336
  print drbddevs
337

    
338
  halted_disks = []
339
  try:
340
    print qa_utils.FormatInfo("Deactivating disks")
341
    cmds = []
342
    for name in node2disk[[node2_full, node_full][int(onmaster)]]:
343
      halted_disks.append(name)
344
      cmds.append(sq(["echo", "offline"]) + " >%s" % _GetDiskStatePath(name))
345
    AssertEqual(StartSSH([node2, node][int(onmaster)]['primary'],
346
                         ' && '.join(cmds)).wait(), 0)
347

    
348
    print qa_utils.FormatInfo("Write to disks and give some time to notice"
349
                              " to notice the problem")
350
    cmds = []
351
    for disk in devpath:
352
      cmds.append(sq(["dd", "count=1", "bs=512", "conv=notrunc",
353
                      "if=%s" % disk, "of=%s" % disk]))
354
    for _ in (0, 1, 2):
355
      AssertEqual(StartSSH(node['primary'], ' && '.join(cmds)).wait(), 0)
356
      time.sleep(3)
357

    
358
    print qa_utils.FormatInfo("Debugging info")
359
    for name in drbddevs:
360
      cmd = ['drbdsetup', name, 'show']
361
      AssertEqual(StartSSH(node['primary'], sq(cmd)).wait(), 0)
362

    
363
    cmd = ['gnt-instance', 'info', instance['name']]
364
    AssertEqual(StartSSH(master['primary'], sq(cmd)).wait(), 0)
365

    
366
  finally:
367
    print qa_utils.FormatInfo("Activating disks again")
368
    cmds = []
369
    for name in halted_disks:
370
      cmds.append(sq(["echo", "running"]) + " >%s" % _GetDiskStatePath(name))
371
    AssertEqual(StartSSH([node2, node][int(onmaster)]['primary'],
372
                         '; '.join(cmds)).wait(), 0)
373

    
374
  if onmaster:
375
    for name in drbddevs:
376
      cmd = ['drbdsetup', name, 'detach']
377
      AssertEqual(StartSSH(node['primary'], sq(cmd)).wait(), 0)
378
  else:
379
    for name in drbddevs:
380
      cmd = ['drbdsetup', name, 'disconnect']
381
      AssertEqual(StartSSH(node2['primary'], sq(cmd)).wait(), 0)
382

    
383
  # TODO
384
  #cmd = ['vgs']
385
  #AssertEqual(StartSSH([node2, node][int(onmaster)]['primary'],
386
  #                     sq(cmd)).wait(), 0)
387

    
388
  print qa_utils.FormatInfo("Making sure disks are up again")
389
  cmd = ['gnt-instance', 'replace-disks', instance['name']]
390
  AssertEqual(StartSSH(master['primary'], sq(cmd)).wait(), 0)
391

    
392
  print qa_utils.FormatInfo("Restarting instance")
393
  cmd = ['gnt-instance', 'shutdown', instance['name']]
394
  AssertEqual(StartSSH(master['primary'], sq(cmd)).wait(), 0)
395

    
396
  cmd = ['gnt-instance', 'startup', instance['name']]
397
  AssertEqual(StartSSH(master['primary'], sq(cmd)).wait(), 0)
398

    
399
  cmd = ['gnt-cluster', 'verify']
400
  AssertEqual(StartSSH(master['primary'], sq(cmd)).wait(), 0)
401

    
402

    
403
def TestInstanceMasterDiskFailure(instance, node, node2):
404
  """Testing disk failure on master node."""
405
  print qa_utils.FormatError("Disk failure on primary node cannot be"
406
                             " tested due to potential crashes.")
407
  # The following can cause crashes, thus it's disabled until fixed
408
  #return _TestInstanceDiskFailure(instance, node, node2, True)
409

    
410

    
411
def TestInstanceSecondaryDiskFailure(instance, node, node2):
412
  """Testing disk failure on secondary node."""
413
  return _TestInstanceDiskFailure(instance, node, node2, False)