Statistics
| Branch: | Tag: | Revision:

root / qa / qa_instance.py @ c68d1f43

History | View | Annotate | Download (11.5 kB)

1
#
2
#
3

    
4
# Copyright (C) 2007 Google Inc.
5
#
6
# This program is free software; you can redistribute it and/or modify
7
# it under the terms of the GNU General Public License as published by
8
# the Free Software Foundation; either version 2 of the License, or
9
# (at your option) any later version.
10
#
11
# This program is distributed in the hope that it will be useful, but
12
# WITHOUT ANY WARRANTY; without even the implied warranty of
13
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
14
# General Public License for more details.
15
#
16
# You should have received a copy of the GNU General Public License
17
# along with this program; if not, write to the Free Software
18
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA
19
# 02110-1301, USA.
20

    
21

    
22
"""Instance related QA tests.
23

24
"""
25

    
26
import re
27
import time
28

    
29
from ganeti import utils
30
from ganeti import constants
31

    
32
import qa_config
33
import qa_utils
34
import qa_error
35

    
36
from qa_utils import AssertEqual, AssertNotEqual, StartSSH
37

    
38

    
39
def _GetDiskStatePath(disk):
40
  return "/sys/block/%s/device/state" % disk
41

    
42

    
43
def _GetGenericAddParameters():
44
  return ['--os-size=%s' % qa_config.get('os-size'),
45
          '--swap-size=%s' % qa_config.get('swap-size'),
46
          '--memory=%s' % qa_config.get('mem')]
47

    
48

    
49
def _DiskTest(node, disk_template):
50
  master = qa_config.GetMasterNode()
51

    
52
  instance = qa_config.AcquireInstance()
53
  try:
54
    cmd = (['gnt-instance', 'add',
55
            '--os-type=%s' % qa_config.get('os'),
56
            '--disk-template=%s' % disk_template,
57
            '--node=%s' % node] +
58
           _GetGenericAddParameters())
59
    cmd.append(instance['name'])
60

    
61
    AssertEqual(StartSSH(master['primary'],
62
                         utils.ShellQuoteArgs(cmd)).wait(), 0)
63
    return instance
64
  except:
65
    qa_config.ReleaseInstance(instance)
66
    raise
67

    
68

    
69
@qa_utils.DefineHook('instance-add-plain-disk')
70
def TestInstanceAddWithPlainDisk(node):
71
  """gnt-instance add -t plain"""
72
  return _DiskTest(node['primary'], 'plain')
73

    
74

    
75
@qa_utils.DefineHook('instance-add-local-mirror-disk')
76
def TestInstanceAddWithLocalMirrorDisk(node):
77
  """gnt-instance add -t local_raid1"""
78
  return _DiskTest(node['primary'], 'local_raid1')
79

    
80

    
81
@qa_utils.DefineHook('instance-add-remote-raid-disk')
82
def TestInstanceAddWithRemoteRaidDisk(node, node2):
83
  """gnt-instance add -t remote_raid1"""
84
  return _DiskTest("%s:%s" % (node['primary'], node2['primary']),
85
                   'remote_raid1')
86

    
87

    
88
@qa_utils.DefineHook('instance-add-drbd-disk')
89
def TestInstanceAddWithDrbdDisk(node, node2):
90
  """gnt-instance add -t drbd"""
91
  return _DiskTest("%s:%s" % (node['primary'], node2['primary']),
92
                   'drbd')
93

    
94

    
95
@qa_utils.DefineHook('instance-remove')
96
def TestInstanceRemove(instance):
97
  """gnt-instance remove"""
98
  master = qa_config.GetMasterNode()
99

    
100
  cmd = ['gnt-instance', 'remove', '-f', instance['name']]
101
  AssertEqual(StartSSH(master['primary'],
102
                       utils.ShellQuoteArgs(cmd)).wait(), 0)
103

    
104
  qa_config.ReleaseInstance(instance)
105

    
106

    
107
@qa_utils.DefineHook('instance-startup')
108
def TestInstanceStartup(instance):
109
  """gnt-instance startup"""
110
  master = qa_config.GetMasterNode()
111

    
112
  cmd = ['gnt-instance', 'startup', instance['name']]
113
  AssertEqual(StartSSH(master['primary'],
114
                       utils.ShellQuoteArgs(cmd)).wait(), 0)
115

    
116

    
117
@qa_utils.DefineHook('instance-shutdown')
118
def TestInstanceShutdown(instance):
119
  """gnt-instance shutdown"""
120
  master = qa_config.GetMasterNode()
121

    
122
  cmd = ['gnt-instance', 'shutdown', instance['name']]
123
  AssertEqual(StartSSH(master['primary'],
124
                       utils.ShellQuoteArgs(cmd)).wait(), 0)
125

    
126

    
127
@qa_utils.DefineHook('instance-reinstall')
128
def TestInstanceReinstall(instance):
129
  """gnt-instance reinstall"""
130
  master = qa_config.GetMasterNode()
131

    
132
  cmd = ['gnt-instance', 'reinstall', '-f', instance['name']]
133
  AssertEqual(StartSSH(master['primary'],
134
                       utils.ShellQuoteArgs(cmd)).wait(), 0)
135

    
136

    
137
@qa_utils.DefineHook('instance-failover')
138
def TestInstanceFailover(instance):
139
  """gnt-instance failover"""
140
  master = qa_config.GetMasterNode()
141

    
142
  cmd = ['gnt-instance', 'failover', '--force', instance['name']]
143
  AssertEqual(StartSSH(master['primary'],
144
                       utils.ShellQuoteArgs(cmd)).wait(), 0)
145

    
146

    
147
@qa_utils.DefineHook('instance-info')
148
def TestInstanceInfo(instance):
149
  """gnt-instance info"""
150
  master = qa_config.GetMasterNode()
151

    
152
  cmd = ['gnt-instance', 'info', instance['name']]
153
  AssertEqual(StartSSH(master['primary'],
154
                       utils.ShellQuoteArgs(cmd)).wait(), 0)
155

    
156

    
157
@qa_utils.DefineHook('instance-modify')
158
def TestInstanceModify(instance):
159
  """gnt-instance modify"""
160
  master = qa_config.GetMasterNode()
161

    
162
  orig_memory = qa_config.get('mem')
163
  orig_bridge = qa_config.get('bridge', 'xen-br0')
164
  args = [
165
    ["--memory", "128"],
166
    ["--memory", str(orig_memory)],
167
    ["--cpu", "2"],
168
    ["--cpu", "1"],
169
    ["--bridge", "xen-br1"],
170
    ["--bridge", orig_bridge],
171
    ["--kernel", "/dev/null"],
172
    ["--kernel", "default"],
173
    ["--initrd", "/dev/null"],
174
    ["--initrd", "none"],
175
    ["--initrd", "default"],
176
    ["--hvm-boot-order", "acn"],
177
    ["--hvm-boot-order", "default"],
178
    ]
179
  for alist in args:
180
    cmd = ['gnt-instance', 'modify'] + alist + [instance['name']]
181
    AssertEqual(StartSSH(master['primary'],
182
                         utils.ShellQuoteArgs(cmd)).wait(), 0)
183

    
184
  # check no-modify
185
  cmd = ['gnt-instance', 'modify', instance['name']]
186
  AssertNotEqual(StartSSH(master['primary'],
187
                          utils.ShellQuoteArgs(cmd)).wait(), 0)
188

    
189

    
190
@qa_utils.DefineHook('instance-list')
191
def TestInstanceList():
192
  """gnt-instance list"""
193
  master = qa_config.GetMasterNode()
194

    
195
  cmd = ['gnt-instance', 'list']
196
  AssertEqual(StartSSH(master['primary'],
197
                       utils.ShellQuoteArgs(cmd)).wait(), 0)
198

    
199

    
200
@qa_utils.DefineHook('backup-export')
201
def TestInstanceExport(instance, node):
202
  """gnt-backup export"""
203
  master = qa_config.GetMasterNode()
204

    
205
  cmd = ['gnt-backup', 'export', '-n', node['primary'], instance['name']]
206
  AssertEqual(StartSSH(master['primary'],
207
                       utils.ShellQuoteArgs(cmd)).wait(), 0)
208

    
209
  return qa_utils.ResolveInstanceName(instance)
210

    
211

    
212
@qa_utils.DefineHook('backup-import')
213
def TestInstanceImport(node, newinst, expnode, name):
214
  """gnt-backup import"""
215
  master = qa_config.GetMasterNode()
216

    
217
  cmd = (['gnt-backup', 'import',
218
          '--disk-template=plain',
219
          '--no-ip-check',
220
          '--src-node=%s' % expnode['primary'],
221
          '--src-dir=%s/%s' % (constants.EXPORT_DIR, name),
222
          '--node=%s' % node['primary']] +
223
         _GetGenericAddParameters())
224
  cmd.append(newinst['name'])
225
  AssertEqual(StartSSH(master['primary'],
226
                       utils.ShellQuoteArgs(cmd)).wait(), 0)
227

    
228

    
229
@qa_utils.DefineHook('backup-list')
230
def TestBackupList(expnode):
231
  """gnt-backup list"""
232
  master = qa_config.GetMasterNode()
233

    
234
  cmd = ['gnt-backup', 'list', '--node=%s' % expnode['primary']]
235
  AssertEqual(StartSSH(master['primary'],
236
                       utils.ShellQuoteArgs(cmd)).wait(), 0)
237

    
238

    
239
def _TestInstanceDiskFailure(instance, node, node2, onmaster):
240
  """Testing disk failure."""
241
  master = qa_config.GetMasterNode()
242
  sq = utils.ShellQuoteArgs
243

    
244
  instance_full = qa_utils.ResolveInstanceName(instance)
245
  node_full = qa_utils.ResolveNodeName(node)
246
  node2_full = qa_utils.ResolveNodeName(node2)
247

    
248
  cmd = ['gnt-node', 'volumes', '--separator=|', '--no-headers',
249
         '--output=node,phys,instance',
250
         node['primary'], node2['primary']]
251
  output = qa_utils.GetCommandOutput(master['primary'], sq(cmd))
252

    
253
  # Get physical disk names
254
  re_disk = re.compile(r'^/dev/([a-z]+)\d+$')
255
  node2disk = {}
256
  for line in output.splitlines():
257
    (node_name, phys, inst) = line.split('|')
258
    if inst == instance_full:
259
      if node_name not in node2disk:
260
        node2disk[node_name] = []
261

    
262
      m = re_disk.match(phys)
263
      if not m:
264
        raise qa_error.Error("Unknown disk name format: %s" % disk)
265

    
266
      name = m.group(1)
267
      if name not in node2disk[node_name]:
268
        node2disk[node_name].append(name)
269

    
270
  if [node2_full, node_full][int(onmaster)] not in node2disk:
271
    raise qa_error.Error("Couldn't find physical disks used on"
272
                         " %s node" % ["secondary", "master"][int(onmaster)])
273

    
274
  # Check whether nodes have ability to stop disks
275
  for node_name, disks in node2disk.iteritems():
276
    cmds = []
277
    for disk in disks:
278
      cmds.append(sq(["test", "-f", _GetDiskStatePath(disk)]))
279
    AssertEqual(StartSSH(node_name, ' && '.join(cmds)).wait(), 0)
280

    
281
  # Get device paths
282
  cmd = ['gnt-instance', 'activate-disks', instance['name']]
283
  output = qa_utils.GetCommandOutput(master['primary'], sq(cmd))
284
  devpath = []
285
  for line in output.splitlines():
286
    (_, _, tmpdevpath) = line.split(':')
287
    devpath.append(tmpdevpath)
288

    
289
  # Get drbd device paths
290
  cmd = ['gnt-instance', 'info', instance['name']]
291
  output = qa_utils.GetCommandOutput(master['primary'], sq(cmd))
292
  pattern = (r'\s+-\s+type:\s+drbd,\s+.*$'
293
             r'\s+primary:\s+(/dev/drbd\d+)\s+')
294
  drbddevs = re.findall(pattern, output, re.M)
295

    
296
  halted_disks = []
297
  try:
298
    # Deactivate disks
299
    cmds = []
300
    for name in node2disk[[node2_full, node_full][int(onmaster)]]:
301
      halted_disks.append(name)
302
      cmds.append(sq(["echo", "offline"]) + " >%s" % _GetDiskStatePath(name))
303
    AssertEqual(StartSSH([node2, node][int(onmaster)]['primary'],
304
                         ' && '.join(cmds)).wait(), 0)
305

    
306
    # Write something to the disks and give some time to notice the problem
307
    cmds = []
308
    for disk in devpath:
309
      cmds.append(sq(["dd", "count=1", "bs=512", "conv=notrunc",
310
                      "if=%s" % disk, "of=%s" % disk]))
311
    for _ in (0, 1, 2):
312
      AssertEqual(StartSSH(node['primary'], ' && '.join(cmds)).wait(), 0)
313
      time.sleep(3)
314

    
315
    for name in drbddevs:
316
      cmd = ['drbdsetup', name, 'show']
317
      AssertEqual(StartSSH(node['primary'], sq(cmd)).wait(), 0)
318

    
319
    # For manual checks
320
    cmd = ['gnt-instance', 'info', instance['name']]
321
    AssertEqual(StartSSH(master['primary'], sq(cmd)).wait(), 0)
322

    
323
  finally:
324
    # Activate disks again
325
    cmds = []
326
    for name in halted_disks:
327
      cmds.append(sq(["echo", "running"]) + " >%s" % _GetDiskStatePath(name))
328
    AssertEqual(StartSSH([node2, node][int(onmaster)]['primary'],
329
                         '; '.join(cmds)).wait(), 0)
330

    
331
  if onmaster:
332
    for name in drbddevs:
333
      cmd = ['drbdsetup', name, 'detach']
334
      AssertEqual(StartSSH(node['primary'], sq(cmd)).wait(), 0)
335
  else:
336
    for name in drbddevs:
337
      cmd = ['drbdsetup', name, 'disconnect']
338
      AssertEqual(StartSSH(node2['primary'], sq(cmd)).wait(), 0)
339

    
340
  # Make sure disks are up again
341
  #cmd = ['gnt-instance', 'activate-disks', instance['name']]
342
  #AssertEqual(StartSSH(master['primary'], sq(cmd)).wait(), 0)
343

    
344
  # Restart instance
345
  cmd = ['gnt-instance', 'shutdown', instance['name']]
346
  AssertEqual(StartSSH(master['primary'], sq(cmd)).wait(), 0)
347

    
348
  #cmd = ['gnt-instance', 'startup', '--force', instance['name']]
349
  cmd = ['gnt-instance', 'startup', instance['name']]
350
  AssertEqual(StartSSH(master['primary'], sq(cmd)).wait(), 0)
351

    
352
  cmd = ['gnt-cluster', 'verify']
353
  AssertEqual(StartSSH(master['primary'], sq(cmd)).wait(), 0)
354

    
355

    
356
def TestInstanceMasterDiskFailure(instance, node, node2):
357
  """Testing disk failure on master node."""
358
  print qa_utils.FormatError("Disk failure on primary node cannot be"
359
                             " tested due to potential crashes.")
360
  # The following can cause crashes, thus it's disabled until fixed
361
  #return _TestInstanceDiskFailure(instance, node, node2, True)
362

    
363

    
364
def TestInstanceSecondaryDiskFailure(instance, node, node2):
365
  """Testing disk failure on secondary node."""
366
  return _TestInstanceDiskFailure(instance, node, node2, False)