Revision 2aaa1336 snf-tools/synnefo_tools/burnin.py
| b/snf-tools/synnefo_tools/burnin.py | ||
|---|---|---|
| 87 | 87 |
green = '\x1b[32m' |
| 88 | 88 |
normal = '\x1b[0m' |
| 89 | 89 |
|
| 90 |
|
|
| 91 | 90 |
class burninFormatter(logging.Formatter): |
| 92 | 91 |
|
| 93 | 92 |
err_fmt = red + "ERROR: %(msg)s" + normal |
| ... | ... | |
| 116 | 115 |
|
| 117 | 116 |
return result |
| 118 | 117 |
|
| 119 |
|
|
| 120 | 118 |
log = logging.getLogger("burnin")
|
| 121 | 119 |
log.setLevel(logging.DEBUG) |
| 122 | 120 |
handler = logging.StreamHandler() |
| ... | ... | |
| 260 | 258 |
def _get_ipv4(self, server): |
| 261 | 259 |
"""Get the public IPv4 of a server from the detailed server info""" |
| 262 | 260 |
|
| 263 |
public_addrs = filter(lambda x: x["id"] == "public", |
|
| 264 |
server["addresses"]["values"]) |
|
| 261 |
public_addrs = filter(lambda x: x["network_id"] == "public", |
|
| 262 |
server["attachments"]["values"]) |
|
| 263 |
|
|
| 265 | 264 |
self.assertEqual(len(public_addrs), 1) |
| 266 |
ipv4_addrs = filter(lambda x: x["version"] == 4, |
|
| 267 |
public_addrs[0]["values"])
|
|
| 268 |
self.assertEqual(len(ipv4_addrs), 1) |
|
| 269 |
return ipv4_addrs[0]["addr"]
|
|
| 265 |
|
|
| 266 |
self.assertTrue(public_addrs[0]['ipv4'] != None)
|
|
| 267 |
|
|
| 268 |
return public_addrs[0]['ipv4']
|
|
| 270 | 269 |
|
| 271 | 270 |
def _get_ipv6(self, server): |
| 272 | 271 |
"""Get the public IPv6 of a server from the detailed server info""" |
| 273 |
public_addrs = filter(lambda x: x["id"] == "public", |
|
| 274 |
server["addresses"]["values"]) |
|
| 272 |
|
|
| 273 |
public_addrs = filter(lambda x: x["network_id"] == "public", |
|
| 274 |
server["attachments"]["values"]) |
|
| 275 |
|
|
| 275 | 276 |
self.assertEqual(len(public_addrs), 1) |
| 276 |
ipv6_addrs = filter(lambda x: x["version"] == 6, |
|
| 277 |
public_addrs[0]["values"]) |
|
| 278 |
self.assertEqual(len(ipv6_addrs), 1) |
|
| 279 |
return ipv6_addrs[0]["addr"] |
|
| 277 |
|
|
| 278 |
self.assertTrue(public_addrs[0]['ipv6'] != None) |
|
| 279 |
|
|
| 280 |
return public_addrs[0]['ipv6'] |
|
| 281 |
|
|
| 282 |
|
|
| 280 | 283 |
|
| 281 | 284 |
def _connect_loginname(self, os): |
| 282 | 285 |
"""Return the login name for connections based on the server OS""" |
| ... | ... | |
| 446 | 449 |
|
| 447 | 450 |
servers = self.client.list_servers(detail=True) |
| 448 | 451 |
servers = filter(lambda x: x["name"] == self.servername, servers) |
| 449 |
self.assertEqual(len(servers), 1) |
|
| 452 |
|
|
| 450 | 453 |
server = servers[0] |
| 451 | 454 |
self.assertEqual(server["name"], self.servername) |
| 452 | 455 |
self.assertEqual(server["flavorRef"], self.flavorid) |
| ... | ... | |
| 506 | 509 |
self._insist_on_status_transition("BUILD", "ACTIVE",
|
| 507 | 510 |
self.build_fail, self.build_warning) |
| 508 | 511 |
|
| 509 |
def test_003a_get_server_oob_console(self): |
|
| 510 |
"""Test getting OOB server console over VNC |
|
| 512 |
# def test_003a_get_server_oob_console(self):
|
|
| 513 |
# """Test getting OOB server console over VNC
|
|
| 511 | 514 |
|
| 512 |
Implementation of RFB protocol follows |
|
| 513 |
http://www.realvnc.com/docs/rfbproto.pdf. |
|
| 515 |
# Implementation of RFB protocol follows
|
|
| 516 |
# http://www.realvnc.com/docs/rfbproto.pdf.
|
|
| 514 | 517 |
|
| 515 |
""" |
|
| 516 |
console = self.cyclades.get_server_console(self.serverid) |
|
| 517 |
self.assertEquals(console['type'], "vnc") |
|
| 518 |
sock = self._insist_on_tcp_connection(socket.AF_INET, |
|
| 519 |
console["host"], console["port"]) |
|
| 520 |
|
|
| 521 |
# Step 1. ProtocolVersion message (par. 6.1.1) |
|
| 522 |
version = sock.recv(1024) |
|
| 523 |
self.assertEquals(version, 'RFB 003.008\n') |
|
| 524 |
sock.send(version) |
|
| 525 |
|
|
| 526 |
# Step 2. Security (par 6.1.2): Only VNC Authentication supported |
|
| 527 |
sec = sock.recv(1024) |
|
| 528 |
self.assertEquals(list(sec), ['\x01', '\x02']) |
|
| 529 |
|
|
| 530 |
# Step 3. Request VNC Authentication (par 6.1.2) |
|
| 531 |
sock.send('\x02')
|
|
| 532 |
|
|
| 533 |
# Step 4. Receive Challenge (par 6.2.2) |
|
| 534 |
challenge = sock.recv(1024) |
|
| 535 |
self.assertEquals(len(challenge), 16) |
|
| 536 |
|
|
| 537 |
# Step 5. DES-Encrypt challenge, use password as key (par 6.2.2) |
|
| 538 |
response = d3des_generate_response( |
|
| 539 |
(console["password"] + '\0' * 8)[:8], challenge) |
|
| 540 |
sock.send(response) |
|
| 541 |
|
|
| 542 |
# Step 6. SecurityResult (par 6.1.3) |
|
| 543 |
result = sock.recv(4) |
|
| 544 |
self.assertEquals(list(result), ['\x00', '\x00', '\x00', '\x00']) |
|
| 545 |
sock.close() |
|
| 518 |
# """
|
|
| 519 |
# console = self.cyclades.get_server_console(self.serverid)
|
|
| 520 |
# self.assertEquals(console['type'], "vnc")
|
|
| 521 |
# sock = self._insist_on_tcp_connection(socket.AF_INET,
|
|
| 522 |
# console["host"], console["port"])
|
|
| 523 |
|
|
| 524 |
# # Step 1. ProtocolVersion message (par. 6.1.1)
|
|
| 525 |
# version = sock.recv(1024)
|
|
| 526 |
# self.assertEquals(version, 'RFB 003.008\n')
|
|
| 527 |
# sock.send(version)
|
|
| 528 |
|
|
| 529 |
# # Step 2. Security (par 6.1.2): Only VNC Authentication supported
|
|
| 530 |
# sec = sock.recv(1024)
|
|
| 531 |
# self.assertEquals(list(sec), ['\x01', '\x02'])
|
|
| 532 |
|
|
| 533 |
# # Step 3. Request VNC Authentication (par 6.1.2)
|
|
| 534 |
# sock.send('\x02')
|
|
| 535 |
|
|
| 536 |
# # Step 4. Receive Challenge (par 6.2.2)
|
|
| 537 |
# challenge = sock.recv(1024)
|
|
| 538 |
# self.assertEquals(len(challenge), 16)
|
|
| 539 |
|
|
| 540 |
# # Step 5. DES-Encrypt challenge, use password as key (par 6.2.2)
|
|
| 541 |
# response = d3des_generate_response(
|
|
| 542 |
# (console["password"] + '\0' * 8)[:8], challenge)
|
|
| 543 |
# sock.send(response)
|
|
| 544 |
|
|
| 545 |
# # Step 6. SecurityResult (par 6.1.3)
|
|
| 546 |
# result = sock.recv(4)
|
|
| 547 |
# self.assertEquals(list(result), ['\x00', '\x00', '\x00', '\x00'])
|
|
| 548 |
# sock.close()
|
|
| 546 | 549 |
|
| 547 | 550 |
def test_004_server_has_ipv4(self): |
| 548 | 551 |
"""Test active server has a valid IPv4 address""" |
| ... | ... | |
| 737 | 740 |
def _get_ipv4(self, server): |
| 738 | 741 |
"""Get the public IPv4 of a server from the detailed server info""" |
| 739 | 742 |
|
| 740 |
public_addrs = filter(lambda x: x["id"] == "public", |
|
| 741 |
server["addresses"]["values"]) |
|
| 743 |
public_addrs = filter(lambda x: x["network_id"] == "public", |
|
| 744 |
server["attachments"]["values"]) |
|
| 745 |
|
|
| 742 | 746 |
self.assertEqual(len(public_addrs), 1) |
| 743 |
ipv4_addrs = filter(lambda x: x["version"] == 4, |
|
| 744 |
public_addrs[0]["values"])
|
|
| 745 |
self.assertEqual(len(ipv4_addrs), 1) |
|
| 746 |
return ipv4_addrs[0]["addr"]
|
|
| 747 |
|
|
| 748 |
self.assertTrue(public_addrs[0]['ipv4'] != None)
|
|
| 749 |
|
|
| 750 |
return public_addrs[0]['ipv4']
|
|
| 747 | 751 |
|
| 748 | 752 |
def _connect_loginname(self, os): |
| 749 | 753 |
"""Return the login name for connections based on the server OS""" |
| ... | ... | |
| 852 | 856 |
|
| 853 | 857 |
name = SNF_TEST_PREFIX + TEST_RUN_ID |
| 854 | 858 |
previous_num = len(self.client.list_networks()) |
| 855 |
network = self.client.create_network(name) |
|
| 859 |
network = self.client.create_network(name,cidr='10.0.0.1/28')
|
|
| 856 | 860 |
|
| 857 | 861 |
#Test if right name is assigned |
| 858 | 862 |
self.assertEqual(network['name'], name) |
| ... | ... | |
| 862 | 866 |
cls.networkid = network['id'] |
| 863 | 867 |
networks = self.client.list_networks() |
| 864 | 868 |
|
| 869 |
fail_tmout = time.time() + self.action_timeout |
|
| 870 |
|
|
| 865 | 871 |
#Test if new network is created |
| 866 |
self.assertTrue(len(networks) > previous_num) |
|
| 872 |
while True: |
|
| 873 |
d = self.client.get_network_details(network['id']) |
|
| 874 |
if d['status'] == 'ACTIVE': |
|
| 875 |
connected = True |
|
| 876 |
break |
|
| 877 |
elif time.time() > fail_tmout: |
|
| 878 |
self.assertLess(time.time(), fail_tmout) |
|
| 879 |
else: |
|
| 880 |
log.info("Waiting for network to become ACTIVE")
|
|
| 881 |
time.sleep(self.query_interval) |
|
| 882 |
|
|
| 883 |
self.assertTrue(connected) |
|
| 867 | 884 |
|
| 868 | 885 |
def test_002_connect_to_network(self): |
| 869 | 886 |
"""Test connect VMs to network""" |
| ... | ... | |
| 877 | 894 |
fail_tmout = time.time() + self.action_timeout |
| 878 | 895 |
|
| 879 | 896 |
while True: |
| 880 |
connected = (self.client.get_network_details(self.networkid)) |
|
| 881 |
connections = connected['servers']['values'] |
|
| 882 |
if (self.serverid['A'] in connections) \ |
|
| 883 |
and (self.serverid['B'] in connections): |
|
| 897 |
|
|
| 898 |
netsA=[x['network_id'] for x in self.client.get_server_details(self.serverid['A'])['attachments']['values']] |
|
| 899 |
netsB=[x['network_id'] for x in self.client.get_server_details(self.serverid['B'])['attachments']['values']] |
|
| 900 |
|
|
| 901 |
if (self.networkid in netsA) and (self.networkid in netsB): |
|
| 884 | 902 |
conn_exists = True |
| 885 | 903 |
break |
| 886 | 904 |
elif time.time() > fail_tmout: |
| ... | ... | |
| 1042 | 1060 |
user=loginname, password=myPass |
| 1043 | 1061 |
): |
| 1044 | 1062 |
|
| 1045 |
if len(sudo('ifconfig eth1 192.168.0.12')) == 0:
|
|
| 1063 |
if len(sudo('ifconfig eth1 10.0.0.5')) == 0:
|
|
| 1046 | 1064 |
res = True |
| 1047 | 1065 |
|
| 1048 | 1066 |
else: |
| ... | ... | |
| 1053 | 1071 |
user=loginname, password=myPass |
| 1054 | 1072 |
): |
| 1055 | 1073 |
|
| 1056 |
if len(run('ifconfig eth1 192.168.0.12')) == 0:
|
|
| 1074 |
if len(run('ifconfig eth1 10.0.0.5')) == 0:
|
|
| 1057 | 1075 |
res = True |
| 1058 | 1076 |
|
| 1059 | 1077 |
self.assertTrue(res) |
| ... | ... | |
| 1094 | 1112 |
user=loginname, password=myPass |
| 1095 | 1113 |
): |
| 1096 | 1114 |
|
| 1097 |
if len(sudo('ifconfig eth1 192.168.0.13')) == 0:
|
|
| 1115 |
if len(sudo('ifconfig eth1 10.0.0.6')) == 0:
|
|
| 1098 | 1116 |
res = True |
| 1099 | 1117 |
|
| 1100 | 1118 |
else: |
| ... | ... | |
| 1105 | 1123 |
user=loginname, password=myPass |
| 1106 | 1124 |
): |
| 1107 | 1125 |
|
| 1108 |
if len(run('ifconfig eth1 192.168.0.13')) == 0:
|
|
| 1126 |
if len(run('ifconfig eth1 10.0.0.6')) == 0:
|
|
| 1109 | 1127 |
res = True |
| 1110 | 1128 |
|
| 1111 | 1129 |
self.assertTrue(res) |
| ... | ... | |
| 1141 | 1159 |
except socket.error: |
| 1142 | 1160 |
raise AssertionError |
| 1143 | 1161 |
|
| 1144 |
cmd = "if ping -c 2 -w 3 192.168.0.13 >/dev/null; \
|
|
| 1162 |
cmd = "if ping -c 2 -w 3 10.0.0.6 >/dev/null; \
|
|
| 1145 | 1163 |
then echo \'True\'; fi;" |
| 1146 | 1164 |
stdin, stdout, stderr = ssh.exec_command(cmd) |
| 1147 | 1165 |
lines = stdout.readlines() |
| ... | ... | |
| 1159 | 1177 |
log.info("Disconnecting servers from private network")
|
| 1160 | 1178 |
|
| 1161 | 1179 |
prev_state = self.client.get_network_details(self.networkid) |
| 1162 |
prev_conn = len(prev_state['servers']['values']) |
|
| 1180 |
prev_nics = prev_state['attachments']['values'] |
|
| 1181 |
prev_conn = len(prev_nics) |
|
| 1182 |
|
|
| 1183 |
nicsA=[x['id'] for x in self.client.get_server_details(self.serverid['A'])['attachments']['values']] |
|
| 1184 |
nicsB=[x['id'] for x in self.client.get_server_details(self.serverid['B'])['attachments']['values']] |
|
| 1185 |
|
|
| 1186 |
for nic in prev_nics: |
|
| 1187 |
if nic in nicsA: |
|
| 1188 |
self.client.disconnect_server(self.serverid['A'], nic) |
|
| 1189 |
if nic in nicsB: |
|
| 1190 |
self.client.disconnect_server(self.serverid['B'], nic) |
|
| 1163 | 1191 |
|
| 1164 |
self.client.disconnect_server(self.serverid['A'], self.networkid) |
|
| 1165 |
self.client.disconnect_server(self.serverid['B'], self.networkid) |
|
| 1166 | 1192 |
|
| 1167 | 1193 |
#Insist on deleting until action timeout |
| 1168 | 1194 |
fail_tmout = time.time() + self.action_timeout |
| 1169 | 1195 |
|
| 1170 | 1196 |
while True: |
| 1197 |
|
|
| 1198 |
netsA=[x['network_id'] for x in self.client.get_server_details(self.serverid['A'])['attachments']['values']] |
|
| 1199 |
netsB=[x['network_id'] for x in self.client.get_server_details(self.serverid['B'])['attachments']['values']] |
|
| 1200 |
|
|
| 1201 |
|
|
| 1171 | 1202 |
connected = (self.client.get_network_details(self.networkid)) |
| 1172 |
connections = connected['servers']['values'] |
|
| 1173 |
if ((self.serverid['A'] not in connections) and |
|
| 1174 |
(self.serverid['B'] not in connections)): |
|
| 1203 |
connections = connected['attachments']['values'] |
|
| 1204 |
if (self.networkid not in netsA) and (self.networkid not in netsB): |
|
| 1175 | 1205 |
conn_exists = False |
| 1176 | 1206 |
break |
| 1177 | 1207 |
elif time.time() > fail_tmout: |
| ... | ... | |
| 1187 | 1217 |
log.info("Submitting delete network request")
|
| 1188 | 1218 |
|
| 1189 | 1219 |
self.client.delete_network(self.networkid) |
| 1190 |
networks = self.client.list_networks() |
|
| 1220 |
|
|
| 1221 |
fail_tmout = time.time() + self.action_timeout |
|
| 1222 |
|
|
| 1223 |
while True: |
|
| 1191 | 1224 |
|
| 1192 |
curr_net = [] |
|
| 1193 |
for net in networks: |
|
| 1194 |
curr_net.append(net['id']) |
|
| 1225 |
curr_net = [] |
|
| 1226 |
networks = self.client.list_networks() |
|
| 1227 |
|
|
| 1228 |
for net in networks: |
|
| 1229 |
curr_net.append(net['id']) |
|
| 1230 |
|
|
| 1231 |
if self.networkid not in curr_net: |
|
| 1232 |
self.assertTrue(self.networkid not in curr_net) |
|
| 1233 |
break |
|
| 1195 | 1234 |
|
| 1196 |
self.assertTrue(self.networkid not in curr_net) |
|
| 1235 |
elif time.time() > fail_tmout: |
|
| 1236 |
self.assertLess(time.time(), fail_tmout) |
|
| 1237 |
|
|
| 1238 |
else: |
|
| 1239 |
time.sleep(self.query_interval) |
|
| 1240 |
|
|
| 1241 |
|
|
| 1197 | 1242 |
|
| 1198 | 1243 |
def test_006_cleanup_servers(self): |
| 1199 | 1244 |
"""Cleanup servers created for this test""" |
| ... | ... | |
| 1230 | 1275 |
Process.__init__(self, **kw) |
| 1231 | 1276 |
kwargs = kw["kwargs"] |
| 1232 | 1277 |
self.testq = kwargs["testq"] |
| 1233 |
self.runner = kwargs["runner"]
|
|
| 1278 |
self.worker_folder = kwargs["worker_folder"]
|
|
| 1234 | 1279 |
|
| 1235 | 1280 |
def run(self): |
| 1236 | 1281 |
# Make sure this test runner process dies with the parent |
| ... | ... | |
| 1238 | 1283 |
# |
| 1239 | 1284 |
# WARNING: This uses the prctl(2) call and is |
| 1240 | 1285 |
# Linux-specific. |
| 1286 |
|
|
| 1241 | 1287 |
prctl.set_pdeathsig(signal.SIGHUP) |
| 1242 | 1288 |
|
| 1289 |
multi = logging.getLogger("multiprocess")
|
|
| 1290 |
|
|
| 1243 | 1291 |
while True: |
| 1244 |
log.debug("I am process %d, GETting from queue is %s",
|
|
| 1245 |
os.getpid(), self.testq)
|
|
| 1292 |
multi.debug("I am process %d, GETting from queue is %s" %
|
|
| 1293 |
(os.getpid(), self.testq))
|
|
| 1246 | 1294 |
msg = self.testq.get() |
| 1247 |
log.debug("Dequeued msg: %s", msg)
|
|
| 1295 |
|
|
| 1296 |
multi.debug("Dequeued msg: %s" % msg)
|
|
| 1248 | 1297 |
|
| 1249 | 1298 |
if msg == "TEST_RUNNER_TERMINATE": |
| 1250 | 1299 |
raise SystemExit |
| 1300 |
|
|
| 1251 | 1301 |
elif issubclass(msg, unittest.TestCase): |
| 1252 | 1302 |
# Assemble a TestSuite, and run it |
| 1303 |
|
|
| 1304 |
log_file = os.path.join(self.worker_folder, 'details_' + |
|
| 1305 |
(msg.__name__) + "_" + |
|
| 1306 |
TEST_RUN_ID + '.log') |
|
| 1307 |
|
|
| 1308 |
fail_file = os.path.join(self.worker_folder, 'failed_' + |
|
| 1309 |
(msg.__name__) + "_" + |
|
| 1310 |
TEST_RUN_ID + '.log') |
|
| 1311 |
error_file = os.path.join(self.worker_folder, 'error_' + |
|
| 1312 |
(msg.__name__) + "_" + |
|
| 1313 |
TEST_RUN_ID + '.log') |
|
| 1314 |
|
|
| 1315 |
f = open(log_file, 'w') |
|
| 1316 |
fail = open(fail_file,'w') |
|
| 1317 |
error = open(error_file, 'w') |
|
| 1318 |
|
|
| 1319 |
log.info(yellow + '* Starting testcase: %s' % msg + normal) |
|
| 1320 |
|
|
| 1321 |
runner = unittest.TextTestRunner(f, verbosity=2, failfast = True) |
|
| 1253 | 1322 |
suite = unittest.TestLoader().loadTestsFromTestCase(msg) |
| 1254 |
self.runner.run(suite) |
|
| 1323 |
result = runner.run(suite) |
|
| 1324 |
|
|
| 1325 |
for res in result.errors: |
|
| 1326 |
log.error("snf-burnin encountered an error in " \
|
|
| 1327 |
"testcase: %s" %msg) |
|
| 1328 |
log.error("See log for details")
|
|
| 1329 |
error.write(str(res[0]) + '\n') |
|
| 1330 |
error.write(str(res[0].shortDescription()) + '\n') |
|
| 1331 |
error.write('\n')
|
|
| 1332 |
|
|
| 1333 |
for res in result.failures: |
|
| 1334 |
log.error("snf-burnin failed in testcase: %s" %msg)
|
|
| 1335 |
log.error("See log for details")
|
|
| 1336 |
fail.write(str(res[0]) + '\n') |
|
| 1337 |
fail.write(str(res[0].shortDescription()) + '\n') |
|
| 1338 |
fail.write('\n')
|
|
| 1339 |
if NOFAILFAST == False: |
|
| 1340 |
sys.exit() |
|
| 1341 |
|
|
| 1342 |
if (len(result.failures) == 0) and (len(result.errors) == 0): |
|
| 1343 |
log.debug("Passed testcase: %s" %msg)
|
|
| 1344 |
|
|
| 1345 |
f.close() |
|
| 1346 |
fail.close() |
|
| 1347 |
error.close() |
|
| 1348 |
|
|
| 1349 |
|
|
| 1255 | 1350 |
else: |
| 1256 | 1351 |
raise Exception("Cannot handle msg: %s" % msg)
|
| 1257 | 1352 |
|
| 1258 | 1353 |
|
| 1259 |
def _run_cases_in_parallel(cases, fanout=1, runner=None):
|
|
| 1354 |
def _run_cases_in_parallel(cases, fanout, image_folder):
|
|
| 1260 | 1355 |
"""Run instances of TestCase in parallel, in a number of distinct processes |
| 1261 | 1356 |
|
| 1262 | 1357 |
The cases iterable specifies the TestCases to be executed in parallel, |
| ... | ... | |
| 1267 | 1362 |
runner process. |
| 1268 | 1363 |
|
| 1269 | 1364 |
""" |
| 1270 |
if runner is None: |
|
| 1271 |
runner = unittest.TextTestRunner(verbosity=2, failfast=True) |
|
| 1365 |
|
|
| 1366 |
multi = logging.getLogger("multiprocess")
|
|
| 1367 |
handler = logging.StreamHandler() |
|
| 1368 |
multi.addHandler(handler) |
|
| 1272 | 1369 |
|
| 1273 |
# testq: The master process enqueues TestCase objects into this queue, |
|
| 1274 |
# test runner processes pick them up for execution, in parallel. |
|
| 1275 |
testq = Queue() |
|
| 1370 |
if VERBOSE: |
|
| 1371 |
multi.setLevel(logging.DEBUG) |
|
| 1372 |
else: |
|
| 1373 |
multi.setLevel(logging.INFO) |
|
| 1374 |
|
|
| 1375 |
testq = [] |
|
| 1376 |
worker_folder = [] |
|
| 1276 | 1377 |
runners = [] |
| 1378 |
|
|
| 1379 |
for i in xrange(0,fanout): |
|
| 1380 |
testq.append(Queue()) |
|
| 1381 |
worker_folder.append(os.path.join(image_folder, 'process'+str(i))) |
|
| 1382 |
os.mkdir(worker_folder[i]) |
|
| 1383 |
|
|
| 1277 | 1384 |
for i in xrange(0, fanout): |
| 1278 |
kwargs = dict(testq=testq, runner=runner)
|
|
| 1385 |
kwargs = dict(testq=testq[i], worker_folder=worker_folder[i])
|
|
| 1279 | 1386 |
runners.append(TestRunnerProcess(kwargs=kwargs)) |
| 1280 | 1387 |
|
| 1281 |
log.info("Spawning %d test runner processes", len(runners))
|
|
| 1388 |
multi.debug("Spawning %d test runner processes" %len(runners))
|
|
| 1389 |
|
|
| 1282 | 1390 |
for p in runners: |
| 1283 | 1391 |
p.start() |
| 1284 |
log.debug("Spawned %d test runners, PIDs are %s",
|
|
| 1285 |
len(runners), [p.pid for p in runners]) |
|
| 1286 | 1392 |
|
| 1287 | 1393 |
# Enqueue test cases |
| 1288 |
map(testq.put, cases) |
|
| 1289 |
map(testq.put, ["TEST_RUNNER_TERMINATE"] * len(runners)) |
|
| 1394 |
for i in xrange(0, fanout): |
|
| 1395 |
map(testq[i].put, cases) |
|
| 1396 |
testq[i].put("TEST_RUNNER_TERMINATE")
|
|
| 1397 |
|
|
| 1398 |
multi.debug("Spawned %d test runners, PIDs are %s" %
|
|
| 1399 |
(len(runners), [p.pid for p in runners])) |
|
| 1290 | 1400 |
|
| 1291 |
log.debug("Joining %d processes", len(runners))
|
|
| 1401 |
multi.debug("Joining %d processes" % len(runners))
|
|
| 1402 |
|
|
| 1292 | 1403 |
for p in runners: |
| 1293 | 1404 |
p.join() |
| 1294 |
log.debug("Done joining %d processes", len(runners))
|
|
| 1405 |
|
|
| 1406 |
multi.debug("Done joining %d processes" % len(runners))
|
|
| 1295 | 1407 |
|
| 1296 | 1408 |
|
| 1297 | 1409 |
def _spawn_server_test_case(**kwargs): |
| ... | ... | |
| 1308 | 1420 |
|
| 1309 | 1421 |
# Make sure the class can be pickled, by listing it among |
| 1310 | 1422 |
# the attributes of __main__. A PicklingError is raised otherwise. |
| 1311 |
setattr(__main__, name, cls) |
|
| 1423 |
|
|
| 1424 |
thismodule = sys.modules[__name__] |
|
| 1425 |
setattr(thismodule, name, cls) |
|
| 1312 | 1426 |
return cls |
| 1313 | 1427 |
|
| 1314 | 1428 |
|
| ... | ... | |
| 1320 | 1434 |
|
| 1321 | 1435 |
# Make sure the class can be pickled, by listing it among |
| 1322 | 1436 |
# the attributes of __main__. A PicklingError is raised otherwise. |
| 1323 |
setattr(__main__, name, cls) |
|
| 1437 |
|
|
| 1438 |
thismodule = sys.modules[__name__] |
|
| 1439 |
setattr(thismodule, name, cls) |
|
| 1324 | 1440 |
return cls |
| 1325 | 1441 |
|
| 1326 | 1442 |
|
| ... | ... | |
| 1347 | 1463 |
print >> sys.stderr, "Use --delete-stale to delete them." |
| 1348 | 1464 |
|
| 1349 | 1465 |
|
| 1350 |
def cleanup_networks(delete_stale=False): |
|
| 1466 |
def cleanup_networks(action_timeout, query_interval,delete_stale=False): |
|
| 1467 |
def isSnfTest(s): |
|
| 1468 |
if s.find(SNF_TEST_PREFIX) == -1: |
|
| 1469 |
return False |
|
| 1470 |
else: |
|
| 1471 |
return True |
|
| 1351 | 1472 |
|
| 1352 | 1473 |
c = CycladesClient(API, TOKEN) |
| 1353 |
|
|
| 1474 |
|
|
| 1354 | 1475 |
networks = c.list_networks() |
| 1355 | 1476 |
stale = [n for n in networks if n["name"].startswith(SNF_TEST_PREFIX)] |
| 1356 | 1477 |
|
| 1357 | 1478 |
if len(stale) == 0: |
| 1358 | 1479 |
return |
| 1480 |
|
|
| 1481 |
fail_tmout = time.time() + action_timeout |
|
| 1482 |
while True: |
|
| 1483 |
servers = c.list_servers() |
|
| 1484 |
staleServers = [s for s in servers if s["name"].startswith(SNF_TEST_PREFIX)] |
|
| 1485 |
if len(staleServers) == 0: |
|
| 1486 |
break |
|
| 1487 |
elif time.time() > fail_tmout: |
|
| 1488 |
log.error("Stale servers not deleted from previous run")
|
|
| 1489 |
sys.exit() |
|
| 1490 |
else: |
|
| 1491 |
time.sleep(query_interval) |
|
| 1359 | 1492 |
|
| 1360 | 1493 |
print >> sys.stderr, yellow + "Found these stale networks from previous runs:" + normal |
| 1361 | 1494 |
print " " + \ |
| ... | ... | |
| 1480 | 1613 |
help="Define the absolute path where the output \ |
| 1481 | 1614 |
log is stored. ", |
| 1482 | 1615 |
default="/var/log/burnin/") |
| 1616 |
parser.add_option("--verbose", "-V",
|
|
| 1617 |
action="store_true", dest="verbose", |
|
| 1618 |
help="Print detailed output about multiple processes spawning", |
|
| 1619 |
default=False) |
|
| 1483 | 1620 |
parser.add_option("--set-tests",
|
| 1484 | 1621 |
action="callback", |
| 1485 | 1622 |
dest="tests", |
| ... | ... | |
| 1492 | 1629 |
default='all', |
| 1493 | 1630 |
callback=parse_comma) |
| 1494 | 1631 |
|
| 1495 |
# FIXME: Change the default for build-fanout to 10 |
|
| 1496 |
# FIXME: Allow the user to specify a specific set of Images to test |
|
| 1497 |
|
|
| 1498 | 1632 |
(opts, args) = parser.parse_args(args) |
| 1499 | 1633 |
|
| 1500 | 1634 |
# Verify arguments |
| ... | ... | |
| 1538 | 1672 |
|
| 1539 | 1673 |
(opts, args) = parse_arguments(sys.argv[1:]) |
| 1540 | 1674 |
|
| 1541 |
global API, TOKEN, PLANKTON, PLANKTON_USER, NO_IPV6 |
|
| 1675 |
global API, TOKEN, PLANKTON, PLANKTON_USER, NO_IPV6, VERBOSE, NOFAILFAST
|
|
| 1542 | 1676 |
API = opts.api |
| 1543 | 1677 |
TOKEN = opts.token |
| 1544 | 1678 |
PLANKTON = opts.plankton |
| 1545 | 1679 |
PLANKTON_USER = opts.plankton_user |
| 1546 | 1680 |
NO_IPV6 = opts.no_ipv6 |
| 1681 |
VERBOSE = opts.verbose |
|
| 1682 |
NOFAILFAST = opts.nofailfast |
|
| 1547 | 1683 |
|
| 1548 | 1684 |
# Cleanup stale servers from previous runs |
| 1549 | 1685 |
if opts.show_stale: |
| 1550 | 1686 |
cleanup_servers(delete_stale=opts.delete_stale) |
| 1551 |
cleanup_networks(delete_stale=opts.delete_stale) |
|
| 1687 |
cleanup_networks(opts.action_timeout, opts.query_interval, delete_stale=opts.delete_stale)
|
|
| 1552 | 1688 |
return 0 |
| 1553 | 1689 |
|
| 1554 | 1690 |
# Initialize a kamaki instance, get flavors, images |
| ... | ... | |
| 1616 | 1752 |
query_interval=opts.query_interval, |
| 1617 | 1753 |
) |
| 1618 | 1754 |
|
| 1755 |
|
|
| 1619 | 1756 |
NetworkTestCase = _spawn_network_test_case( |
| 1620 | 1757 |
action_timeout=opts.action_timeout, |
| 1621 | 1758 |
imageid=imageid, |
| ... | ... | |
| 1643 | 1780 |
image_folder = os.path.join(test_folder, imageid) |
| 1644 | 1781 |
os.mkdir(image_folder) |
| 1645 | 1782 |
|
| 1646 |
for case in seq_cases: |
|
| 1647 |
|
|
| 1648 |
test = (key for key, value in test_dict.items() |
|
| 1649 |
if value == case).next() |
|
| 1650 |
|
|
| 1651 |
log.info(yellow + '* Starting testcase: %s' %test + normal) |
|
| 1652 |
log_file = os.path.join(image_folder, 'details_' + |
|
| 1653 |
(case.__name__) + "_" + |
|
| 1654 |
TEST_RUN_ID + '.log') |
|
| 1655 |
fail_file = os.path.join(image_folder, 'failed_' + |
|
| 1656 |
(case.__name__) + "_" + |
|
| 1657 |
TEST_RUN_ID + '.log') |
|
| 1658 |
error_file = os.path.join(image_folder, 'error_' + |
|
| 1659 |
(case.__name__) + "_" + |
|
| 1660 |
TEST_RUN_ID + '.log') |
|
| 1661 |
|
|
| 1662 |
f = open(log_file, "w") |
|
| 1663 |
fail = open(fail_file, "w") |
|
| 1664 |
error = open(error_file, "w") |
|
| 1665 |
|
|
| 1666 |
suite = unittest.TestLoader().loadTestsFromTestCase(case) |
|
| 1667 |
runner = unittest.TextTestRunner(f, verbosity=2, failfast=True) |
|
| 1668 |
result = runner.run(suite) |
|
| 1669 |
|
|
| 1670 |
for res in result.errors: |
|
| 1671 |
log.error("snf-burnin encountered an error in " \
|
|
| 1672 |
"testcase: %s" %test) |
|
| 1673 |
log.error("See log for details")
|
|
| 1674 |
error.write(str(res[0]) + '\n') |
|
| 1675 |
error.write(str(res[0].shortDescription()) + '\n') |
|
| 1676 |
error.write('\n')
|
|
| 1677 |
|
|
| 1678 |
for res in result.failures: |
|
| 1679 |
log.error("snf-burnin failed in testcase: %s" %test)
|
|
| 1680 |
log.error("See log for details")
|
|
| 1681 |
fail.write(str(res[0]) + '\n') |
|
| 1682 |
fail.write(str(res[0].shortDescription()) + '\n') |
|
| 1683 |
fail.write('\n')
|
|
| 1684 |
if opts.nofailfast == False: |
|
| 1685 |
sys.exit() |
|
| 1686 |
|
|
| 1687 |
if (len(result.failures) == 0) and (len(result.errors) == 0): |
|
| 1688 |
log.debug("Passed testcase: %s" %test)
|
|
| 1783 |
log.info('Parallel spawn:')
|
|
| 1784 |
|
|
| 1785 |
_run_cases_in_parallel(seq_cases, opts.fanout, image_folder) |
|
| 1689 | 1786 |
|
| 1690 | 1787 |
if __name__ == "__main__": |
| 1691 | 1788 |
sys.exit(main()) |
Also available in: Unified diff