Revision b1ffe1eb

b/qa/ganeti-qa.py
60 60
  return fn(*args)
61 61

  
62 62

  
63
def main():
64
  """Main program.
63
def RunEnvTests():
64
  """Run several environment tests.
65 65

  
66 66
  """
67
  parser = OptionParser(usage="%prog [options] <config-file> "
68
                              "<known-hosts-file>")
69
  parser.add_option('--dry-run', dest='dry_run',
70
      action="store_true",
71
      help="Show what would be done")
72
  parser.add_option('--yes-do-it', dest='yes_do_it',
73
      action="store_true",
74
      help="Really execute the tests")
75
  (qa_config.options, args) = parser.parse_args()
67
  if not qa_config.TestEnabled('env'):
68
    return
76 69

  
77
  if len(args) == 2:
78
    (config_file, known_hosts_file) = args
79
  else:
80
    parser.error("Not enough arguments.")
81

  
82
  if not qa_config.options.yes_do_it:
83
    print ("Executing this script irreversibly destroys any Ganeti\n"
84
           "configuration on all nodes involved. If you really want\n"
85
           "to start testing, supply the --yes-do-it option.")
86
    sys.exit(1)
87

  
88
  qa_config.Load(config_file)
70
  RunTest(qa_env.TestSshConnection)
71
  RunTest(qa_env.TestIcmpPing)
72
  RunTest(qa_env.TestGanetiCommands)
89 73

  
90
  RunTest(qa_other.TestUploadKnownHostsFile, known_hosts_file)
91 74

  
92
  if qa_config.TestEnabled('env'):
93
    RunTest(qa_env.TestSshConnection)
94
    RunTest(qa_env.TestIcmpPing)
95
    RunTest(qa_env.TestGanetiCommands)
75
def SetupCluster():
76
  """Initializes the cluster.
96 77

  
78
  """
97 79
  RunTest(qa_cluster.TestClusterInit)
98

  
99 80
  RunTest(qa_node.TestNodeAddAll)
81
  if qa_config.TestEnabled('node-info'):
82
    RunTest(qa_node.TestNodeInfo)
83

  
84

  
85
def RunClusterTests():
86
  """Runs tests related to gnt-cluster.
100 87

  
88
  """
101 89
  if qa_config.TestEnabled('cluster-verify'):
102 90
    RunTest(qa_cluster.TestClusterVerify)
103 91

  
......
113 101
  if qa_config.TestEnabled('cluster-copyfile'):
114 102
    RunTest(qa_cluster.TestClusterCopyfile)
115 103

  
116
  if qa_config.TestEnabled('node-info'):
117
    RunTest(qa_node.TestNodeInfo)
118

  
119 104
  if qa_config.TestEnabled('cluster-burnin'):
120 105
    RunTest(qa_cluster.TestClusterBurnin)
121 106

  
122 107
  if qa_config.TestEnabled('cluster-master-failover'):
123 108
    RunTest(qa_cluster.TestClusterMasterFailover)
124 109

  
125
  if qa_config.TestEnabled('os'):
126
    RunTest(qa_os.TestOsList)
127
    RunTest(qa_os.TestOsDiagnose)
128
    RunTest(qa_os.TestOsValid)
129
    RunTest(qa_os.TestOsInvalid)
130
    RunTest(qa_os.TestOsPartiallyValid)
131 110

  
132
  node = qa_config.AcquireNode()
133
  try:
134
    if qa_config.TestEnabled('instance-add-plain-disk'):
135
      instance = RunTest(qa_instance.TestInstanceAddWithPlainDisk, node)
111
def RunOsTests():
112
  """Runs all tests related to gnt-os.
136 113

  
137
      if qa_config.TestEnabled('instance-shutdown'):
138
        RunTest(qa_instance.TestInstanceShutdown, instance)
139
        RunTest(qa_instance.TestInstanceStartup, instance)
114
  """
115
  if not qa_config.TestEnabled('os'):
116
    return
117

  
118
  RunTest(qa_os.TestOsList)
119
  RunTest(qa_os.TestOsDiagnose)
120
  RunTest(qa_os.TestOsValid)
121
  RunTest(qa_os.TestOsInvalid)
122
  RunTest(qa_os.TestOsPartiallyValid)
123

  
124

  
125
def RunCommonInstanceTests(instance):
126
  """Runs a few tests that are common to all disk types.
127

  
128
  """
129
  if qa_config.TestEnabled('instance-shutdown'):
130
    RunTest(qa_instance.TestInstanceShutdown, instance)
131
    RunTest(qa_instance.TestInstanceStartup, instance)
140 132

  
141
      if qa_config.TestEnabled('instance-list'):
142
        RunTest(qa_instance.TestInstanceList)
133
  if qa_config.TestEnabled('instance-list'):
134
    RunTest(qa_instance.TestInstanceList)
143 135

  
144
      if qa_config.TestEnabled('instance-info'):
145
        RunTest(qa_instance.TestInstanceInfo, instance)
136
  if qa_config.TestEnabled('instance-info'):
137
    RunTest(qa_instance.TestInstanceInfo, instance)
146 138

  
147
      automatic_restart = \
148
        qa_config.TestEnabled('instance-automatic-restart')
149
      consecutive_failures = \
150
        qa_config.TestEnabled('instance-consecutive-failures')
139
  if qa_config.TestEnabled('instance-reinstall'):
140
    RunTest(qa_instance.TestInstanceShutdown, instance)
141
    RunTest(qa_instance.TestInstanceReinstall, instance)
142
    RunTest(qa_instance.TestInstanceStartup, instance)
151 143

  
152
      if automatic_restart or consecutive_failures:
153
        qa_daemon.PrintCronWarning()
144
  if qa_config.TestEnabled('node-volumes'):
145
    RunTest(qa_node.TestNodeVolumes)
154 146

  
155
        if automatic_restart:
156
          RunTest(qa_daemon.TestInstanceAutomaticRestart, node, instance)
157 147

  
158
        if consecutive_failures:
159
          RunTest(qa_daemon.TestInstanceConsecutiveFailures, node, instance)
148
def RunExportImportTests(instance, pnode):
149
  """Tries to export and import the instance.
160 150

  
161
      if qa_config.TestEnabled('instance-export'):
162
        expnode = qa_config.AcquireNode(exclude=node)
151
  """
152
  if qa_config.TestEnabled('instance-export'):
153
    expnode = qa_config.AcquireNode(exclude=pnode)
154
    try:
155
      name = RunTest(qa_instance.TestInstanceExport, instance, expnode)
156

  
157
      RunTest(qa_instance.TestBackupList, expnode)
158

  
159
      if qa_config.TestEnabled('instance-import'):
160
        newinst = qa_config.AcquireInstance()
163 161
        try:
164
          name = RunTest(qa_instance.TestInstanceExport, instance, expnode)
165

  
166
          RunTest(qa_instance.TestBackupList, expnode)
167

  
168
          if qa_config.TestEnabled('instance-import'):
169
            newinst = qa_config.AcquireInstance()
170
            try:
171
              RunTest(qa_instance.TestInstanceImport, node, newinst,
172
                      expnode, name)
173
              RunTest(qa_instance.TestInstanceRemove, newinst)
174
            finally:
175
              qa_config.ReleaseInstance(newinst)
162
          RunTest(qa_instance.TestInstanceImport, pnode, newinst,
163
                  expnode, name)
164
          RunTest(qa_instance.TestInstanceRemove, newinst)
176 165
        finally:
177
          qa_config.ReleaseNode(expnode)
166
          qa_config.ReleaseInstance(newinst)
167
    finally:
168
      qa_config.ReleaseNode(expnode)
178 169

  
179
      if qa_config.TestEnabled('instance-reinstall'):
180
        RunTest(qa_instance.TestInstanceShutdown, instance)
181
        RunTest(qa_instance.TestInstanceReinstall, instance)
182
        RunTest(qa_instance.TestInstanceStartup, instance)
183 170

  
184
      if qa_config.TestEnabled('node-volumes'):
185
        RunTest(qa_node.TestNodeVolumes)
171
def RunDaemonTests(instance, pnode):
172
  """Test the ganeti-watcher script.
186 173

  
187
      RunTest(qa_instance.TestInstanceRemove, instance)
188
      del instance
174
  """
175
  automatic_restart = \
176
    qa_config.TestEnabled('instance-automatic-restart')
177
  consecutive_failures = \
178
    qa_config.TestEnabled('instance-consecutive-failures')
189 179

  
190
    if qa_config.TestEnabled('instance-add-local-mirror-disk'):
191
      instance = RunTest(qa_instance.TestInstanceAddWithLocalMirrorDisk, node)
180
  if automatic_restart or consecutive_failures:
181
    qa_daemon.PrintCronWarning()
192 182

  
193
      if qa_config.TestEnabled('instance-shutdown'):
194
        RunTest(qa_instance.TestInstanceShutdown, instance)
195
        RunTest(qa_instance.TestInstanceStartup, instance)
183
    if automatic_restart:
184
      RunTest(qa_daemon.TestInstanceAutomaticRestart, pnode, instance)
196 185

  
197
      if qa_config.TestEnabled('instance-info'):
198
        RunTest(qa_instance.TestInstanceInfo, instance)
186
    if consecutive_failures:
187
      RunTest(qa_daemon.TestInstanceConsecutiveFailures, node, instance)
199 188

  
200
      if qa_config.TestEnabled('node-volumes'):
201
        RunTest(qa_node.TestNodeVolumes)
202 189

  
203
      RunTest(qa_instance.TestInstanceRemove, instance)
204
      del instance
190
def RunHardwareFailureTests(instance, pnode, snode):
191
  """Test cluster internal hardware failure recovery.
205 192

  
206
    if qa_config.TestEnabled('instance-add-remote-raid-disk'):
207
      node2 = qa_config.AcquireNode(exclude=node)
208
      try:
209
        instance = RunTest(qa_instance.TestInstanceAddWithRemoteRaidDisk,
210
                           node, node2)
193
  """
194
  if qa_config.TestEnabled('instance-failover'):
195
    RunTest(qa_instance.TestInstanceFailover, instance)
196

  
197
  if qa_config.TestEnabled('node-evacuate'):
198
    RunTest(qa_node.TestNodeEvacuate, pnode, snode)
199

  
200
  if qa_config.TestEnabled('node-failover'):
201
    RunTest(qa_node.TestNodeFailover, pnode, snode)
202

  
203
  if qa_config.TestEnabled('instance-disk-failure'):
204
    RunTest(qa_instance.TestInstanceMasterDiskFailure,
205
            instance, pnode, snode)
206
    RunTest(qa_instance.TestInstanceSecondaryDiskFailure,
207
            instance, pnode, snode)
208

  
209

  
210
def main():
211
  """Main program.
212

  
213
  """
214
  parser = OptionParser(usage="%prog [options] <config-file> "
215
                              "<known-hosts-file>")
216
  parser.add_option('--dry-run', dest='dry_run',
217
      action="store_true",
218
      help="Show what would be done")
219
  parser.add_option('--yes-do-it', dest='yes_do_it',
220
      action="store_true",
221
      help="Really execute the tests")
222
  (qa_config.options, args) = parser.parse_args()
211 223

  
212
        if qa_config.TestEnabled('instance-shutdown'):
213
          RunTest(qa_instance.TestInstanceShutdown, instance)
214
          RunTest(qa_instance.TestInstanceStartup, instance)
224
  if len(args) == 2:
225
    (config_file, known_hosts_file) = args
226
  else:
227
    parser.error("Not enough arguments.")
215 228

  
216
        if qa_config.TestEnabled('instance-info'):
217
          RunTest(qa_instance.TestInstanceInfo, instance)
229
  if not qa_config.options.yes_do_it:
230
    print ("Executing this script irreversibly destroys any Ganeti\n"
231
           "configuration on all nodes involved. If you really want\n"
232
           "to start testing, supply the --yes-do-it option.")
233
    sys.exit(1)
218 234

  
219
        if qa_config.TestEnabled('instance-failover'):
220
          RunTest(qa_instance.TestInstanceFailover, instance)
235
  qa_config.Load(config_file)
221 236

  
222
        if qa_config.TestEnabled('node-evacuate'):
223
          RunTest(qa_node.TestNodeEvacuate, node, node2)
237
  RunTest(qa_other.UploadKnownHostsFile, known_hosts_file)
224 238

  
225
        if qa_config.TestEnabled('node-failover'):
226
          RunTest(qa_node.TestNodeFailover, node, node2)
239
  RunEnvTests()
240
  SetupCluster()
241
  RunClusterTests()
242
  RunOsTests()
227 243

  
228
        if qa_config.TestEnabled('node-volumes'):
229
          RunTest(qa_node.TestNodeVolumes)
244
  pnode = qa_config.AcquireNode()
245
  try:
246
    if qa_config.TestEnabled('instance-add-plain-disk'):
247
      instance = RunTest(qa_instance.TestInstanceAddWithPlainDisk, pnode)
248
      RunCommonInstanceTests(instance)
249
      RunExportImportTests(instance, pnode)
250
      RunDaemonTests(instance, pnode)
251
      RunTest(qa_instance.TestInstanceRemove, instance)
252
      del instance
230 253

  
231
        if qa_config.TestEnabled('instance-disk-failure'):
232
          RunTest(qa_instance.TestInstanceMasterDiskFailure,
233
                  instance, node, node2)
234
          RunTest(qa_instance.TestInstanceSecondaryDiskFailure,
235
                  instance, node, node2)
254
    if qa_config.TestEnabled('instance-add-local-mirror-disk'):
255
      instance = RunTest(qa_instance.TestInstanceAddWithLocalMirrorDisk, pnode)
256
      RunCommonInstanceTests(instance)
257
      RunExportImportTests(instance, pnode)
258
      RunTest(qa_instance.TestInstanceRemove, instance)
259
      del instance
236 260

  
261
    if qa_config.TestEnabled('instance-add-remote-raid-disk'):
262
      snode = qa_config.AcquireNode(exclude=pnode)
263
      try:
264
        instance = RunTest(qa_instance.TestInstanceAddWithRemoteRaidDisk,
265
                           pnode, snode)
266
        RunCommonInstanceTests(instance)
267
        RunExportImportTests(instance, pnode)
268
        RunHardwareFailureTests(instance, pnode, snode)
237 269
        RunTest(qa_instance.TestInstanceRemove, instance)
238 270
        del instance
239 271
      finally:
240
        qa_config.ReleaseNode(node2)
272
        qa_config.ReleaseNode(snode)
241 273

  
242 274
  finally:
243
    qa_config.ReleaseNode(node)
275
    qa_config.ReleaseNode(pnode)
244 276

  
245 277
  RunTest(qa_node.TestNodeRemoveAll)
246 278

  
b/qa/qa_instance.py
239 239
             r'\s+primary:\s+(/dev/drbd\d+)\s+')
240 240
  drbddevs = re.findall(pattern, output, re.M)
241 241

  
242
  # Deactivate disks on secondary node
243 242
  halted_disks = []
244
  cmds = []
245
  for name in node2disk[[node2_full, node_full][int(onmaster)]]:
246
    halted_disks.append(name)
247
    cmds.append(sq(["echo", "offline"]) + " >%s" % _GetDiskStatePath(name))
248
  AssertEqual(StartSSH([node2, node][int(onmaster)]['primary'],
249
                       '; '.join(cmds)).wait(), 0)
250 243
  try:
244
    # Deactivate disks
245
    cmds = []
246
    for name in node2disk[[node2_full, node_full][int(onmaster)]]:
247
      halted_disks.append(name)
248
      cmds.append(sq(["echo", "offline"]) + " >%s" % _GetDiskStatePath(name))
249
    AssertEqual(StartSSH([node2, node][int(onmaster)]['primary'],
250
                         ' && '.join(cmds)).wait(), 0)
251

  
251 252
    # Write something to the disks and give some time to notice the problem
252 253
    cmds = []
253 254
    for disk in devpath:
......
257 258
      AssertEqual(StartSSH(node['primary'], ' && '.join(cmds)).wait(), 0)
258 259
      time.sleep(3)
259 260

  
261
    for name in drbddevs:
262
      cmd = ['drbdsetup', name, 'show']
263
      AssertEqual(StartSSH(node['primary'], sq(cmd)).wait(), 0)
264

  
260 265
    # For manual checks
261 266
    cmd = ['gnt-instance', 'info', instance['name']]
262 267
    AssertEqual(StartSSH(master['primary'], sq(cmd)).wait(), 0)
......
269 274
    AssertEqual(StartSSH([node2, node][int(onmaster)]['primary'],
270 275
                         '; '.join(cmds)).wait(), 0)
271 276

  
277
  if onmaster:
278
    for name in drbddevs:
279
      cmd = ['drbdsetup', name, 'detach']
280
      AssertEqual(StartSSH(node['primary'], sq(cmd)).wait(), 0)
281
  else:
282
    for name in drbddevs:
283
      cmd = ['drbdsetup', name, 'disconnect']
284
      AssertEqual(StartSSH(node2['primary'], sq(cmd)).wait(), 0)
285

  
286
  # Make sure disks are up again
287
  #cmd = ['gnt-instance', 'activate-disks', instance['name']]
288
  #AssertEqual(StartSSH(master['primary'], sq(cmd)).wait(), 0)
289

  
272 290
  # Restart instance
273 291
  cmd = ['gnt-instance', 'shutdown', instance['name']]
274 292
  AssertEqual(StartSSH(master['primary'], sq(cmd)).wait(), 0)
275 293

  
276
  cmd = ['gnt-instance', 'startup', '--force', instance['name']]
277
  AssertEqual(StartSSH(master['primary'], sq(cmd)).wait(), 0)
278

  
279
  # Make sure disks are up again
280
  cmd = ['gnt-instance', 'activate-disks', instance['name']]
294
  #cmd = ['gnt-instance', 'startup', '--force', instance['name']]
295
  cmd = ['gnt-instance', 'startup', instance['name']]
281 296
  AssertEqual(StartSSH(master['primary'], sq(cmd)).wait(), 0)
282 297

  
283 298
  cmd = ['gnt-cluster', 'verify']
......
289 304
  qa_utils.PrintError("Disk failure on primary node cannot be "
290 305
                      "tested due to potential crashes.")
291 306
  # The following can cause crashes, thus it's disabled until fixed
292
  #return _TestInstanceDiskFailure(instance, node, node2, True)
307
  return _TestInstanceDiskFailure(instance, node, node2, True)
293 308

  
294 309

  
295 310
def TestInstanceSecondaryDiskFailure(instance, node, node2):
b/qa/qa_node.py
89 89
  master = qa_config.GetMasterNode()
90 90

  
91 91
  if qa_utils.GetNodeInstances(node2, secondaries=False):
92
    raise qa_errors.UnusableNodeError("Secondary node has at least one "
93
                                      "primary instance. This test requires "
94
                                      "it to have no primary instances.")
92
    raise qa_error.UnusableNodeError("Secondary node has at least one "
93
                                     "primary instance. This test requires "
94
                                     "it to have no primary instances.")
95 95

  
96 96
  # Fail over to secondary node
97 97
  cmd = ['gnt-node', 'failover', '-f', node['primary']]
......
111 111
  node3 = qa_config.AcquireNode(exclude=[node, node2])
112 112
  try:
113 113
    if qa_utils.GetNodeInstances(node3, secondaries=True):
114
      raise qa_errors.UnusableNodeError("Evacuation node has at least one "
115
                                        "secondary instance. This test requires "
116
                                        "it to have no secondary instances.")
114
      raise qa_error.UnusableNodeError("Evacuation node has at least one "
115
                                       "secondary instance. This test requires "
116
                                       "it to have no secondary instances.")
117 117

  
118 118
    # Evacuate all secondary instances
119 119
    cmd = ['gnt-node', 'evacuate', '-f', node2['primary'], node3['primary']]
b/qa/qa_other.py
25 25
from qa_utils import AssertEqual, StartSSH
26 26

  
27 27

  
28
def TestUploadKnownHostsFile(localpath):
28
def UploadKnownHostsFile(localpath):
29 29
  """Uploading known_hosts file.
30 30

  
31 31
  """

Also available in: Unified diff