Revision 6c78720b snf-tools/synnefo_tools/burnin/common.py
b/snf-tools/synnefo_tools/burnin/common.py | ||
---|---|---|
38 | 38 |
|
39 | 39 |
import os |
40 | 40 |
import re |
41 |
import time |
|
41 | 42 |
import shutil |
43 |
import socket |
|
44 |
import random |
|
42 | 45 |
import unittest |
43 | 46 |
import datetime |
44 | 47 |
import tempfile |
45 | 48 |
import traceback |
49 |
import subprocess |
|
46 | 50 |
|
51 |
from kamaki.clients.cyclades import CycladesClient |
|
47 | 52 |
from kamaki.clients.astakos import AstakosClient |
48 | 53 |
from kamaki.clients.compute import ComputeClient |
49 | 54 |
from kamaki.clients.pithos import PithosClient |
... | ... | |
104 | 109 |
|
105 | 110 |
# -------------------------------------------------------------------- |
106 | 111 |
# BurninTests class |
107 |
# Too few public methods (0/2). pylint: disable-msg=R0903 |
|
112 |
# Too few public methods. pylint: disable-msg=R0903 |
|
113 |
# Too many instance attributes. pylint: disable-msg=R0902 |
|
108 | 114 |
class Clients(object): |
109 | 115 |
"""Our kamaki clients""" |
110 | 116 |
auth_url = None |
... | ... | |
135 | 141 |
action_warning = None |
136 | 142 |
query_interval = None |
137 | 143 |
system_user = None |
144 |
images = None |
|
145 |
flavors = None |
|
138 | 146 |
|
139 | 147 |
@classmethod |
140 | 148 |
def setUpClass(cls): # noqa |
... | ... | |
160 | 168 |
self.clients.compute_url, self.clients.token) |
161 | 169 |
self.clients.compute.CONNECTION_RETRY_LIMIT = self.clients.retry |
162 | 170 |
|
171 |
self.clients.cyclades = CycladesClient( |
|
172 |
self.clients.compute_url, self.clients.token) |
|
173 |
self.clients.cyclades.CONNECTION_RETRY_LIMIT = self.clients.retry |
|
174 |
|
|
163 | 175 |
self.clients.pithos_url = self.clients.astakos.\ |
164 | 176 |
get_service_endpoints('object-store')['publicURL'] |
165 | 177 |
self.info("Pithos url is %s", self.clients.pithos_url) |
... | ... | |
268 | 280 |
self.warning("No system user found") |
269 | 281 |
return None |
270 | 282 |
|
283 |
def _try_until_timeout_expires(self, opmsg, check_fun): |
|
284 |
"""Try to perform an action until timeout expires""" |
|
285 |
assert callable(check_fun), "Not a function" |
|
286 |
|
|
287 |
action_timeout = self.action_timeout |
|
288 |
action_warning = self.action_warning |
|
289 |
if action_warning > action_timeout: |
|
290 |
action_warning = action_timeout |
|
291 |
|
|
292 |
start_time = time.time() |
|
293 |
while (start_time + action_warning) > time.time(): |
|
294 |
try: |
|
295 |
return check_fun() |
|
296 |
except Retry: |
|
297 |
time.sleep(self.query_interval) |
|
298 |
self.warning("Operation `%s' is taking too long", opmsg) |
|
299 |
while (start_time + action_timeout) > time.time(): |
|
300 |
try: |
|
301 |
return check_fun() |
|
302 |
except Retry: |
|
303 |
time.sleep(self.query_interval) |
|
304 |
self.error("Operation `%s' timed out", opmsg) |
|
305 |
self.fail("time out") |
|
306 |
|
|
307 |
def _skip_if(self, condition, msg): |
|
308 |
"""Skip tests""" |
|
309 |
if condition: |
|
310 |
self.info("Test skipped: %s" % msg) |
|
311 |
self.skipTest(msg) |
|
312 |
|
|
271 | 313 |
# ---------------------------------- |
272 | 314 |
# Flavors |
273 | 315 |
def _get_list_of_flavors(self, detail=False): |
... | ... | |
279 | 321 |
flavors = self.clients.compute.list_flavors(detail=detail) |
280 | 322 |
return flavors |
281 | 323 |
|
324 |
def _find_flavors(self, patterns, flavors=None): |
|
325 |
"""Find a list of suitable flavors to use |
|
326 |
|
|
327 |
The patterns is a list of `typed_options'. A list of all flavors |
|
328 |
matching this patterns will be returned. |
|
329 |
|
|
330 |
""" |
|
331 |
if flavors is None: |
|
332 |
flavors = self._get_list_of_flavors(detail=True) |
|
333 |
|
|
334 |
ret_flavors = [] |
|
335 |
for ptrn in patterns: |
|
336 |
parsed_ptrn = parse_typed_option(ptrn) |
|
337 |
if parsed_ptrn is None: |
|
338 |
msg = "Invalid flavor format: %s. Must be [id|name]:.+" |
|
339 |
self.warning(msg, ptrn) |
|
340 |
continue |
|
341 |
flv_type, flv_value = parsed_ptrn |
|
342 |
if flv_type == "name": |
|
343 |
# Filter flavor by name |
|
344 |
msg = "Trying to find a flavor with name %s" |
|
345 |
self.info(msg, flv_value) |
|
346 |
filtered_flvs = \ |
|
347 |
[f for f in flavors if |
|
348 |
re.search(flv_value, f['name'], flags=re.I) is not None] |
|
349 |
elif flv_type == "id": |
|
350 |
# Filter flavors by id |
|
351 |
msg = "Trying to find a flavor with id %s" |
|
352 |
self.info(msg, flv_value) |
|
353 |
filtered_flvs = \ |
|
354 |
[f for f in flavors if str(f['id']) == flv_value] |
|
355 |
else: |
|
356 |
self.error("Unrecognized flavor type %s", flv_type) |
|
357 |
self.fail("Unrecognized flavor type") |
|
358 |
|
|
359 |
# Append and continue |
|
360 |
ret_flavors.extend(filtered_flvs) |
|
361 |
|
|
362 |
self.assertGreater(len(ret_flavors), 0, |
|
363 |
"No matching flavors found") |
|
364 |
return ret_flavors |
|
365 |
|
|
282 | 366 |
# ---------------------------------- |
283 | 367 |
# Images |
284 | 368 |
def _get_list_of_images(self, detail=False): |
... | ... | |
306 | 390 |
|
307 | 391 |
return ret_images |
308 | 392 |
|
309 |
def _find_image(self, patterns, images=None): |
|
310 |
"""Find a suitable image to use
|
|
393 |
def _find_images(self, patterns, images=None):
|
|
394 |
"""Find a list of suitable images to use
|
|
311 | 395 |
|
312 |
The patterns is a list of `typed_options'. The first pattern to
|
|
313 |
match an image will be the one that will be returned.
|
|
396 |
The patterns is a list of `typed_options'. A list of all images
|
|
397 |
matching this patterns will be returned.
|
|
314 | 398 |
|
315 | 399 |
""" |
316 | 400 |
if images is None: |
317 | 401 |
images = self._get_list_of_sys_images() |
318 | 402 |
|
403 |
ret_images = [] |
|
319 | 404 |
for ptrn in patterns: |
320 | 405 |
parsed_ptrn = parse_typed_option(ptrn) |
321 | 406 |
if parsed_ptrn is None: |
... | ... | |
341 | 426 |
self.error("Unrecognized image type %s", img_type) |
342 | 427 |
self.fail("Unrecognized image type") |
343 | 428 |
|
344 |
# Check if we found one |
|
345 |
if filtered_imgs: |
|
346 |
img = filtered_imgs[0] |
|
347 |
self.info("Will use %s with id %s", img['name'], img['id']) |
|
348 |
return img |
|
429 |
# Append and continue |
|
430 |
ret_images.extend(filtered_imgs) |
|
349 | 431 |
|
350 |
# We didn't found one |
|
351 |
err = "No matching image found" |
|
352 |
self.error(err) |
|
353 |
self.fail(err) |
|
432 |
self.assertGreater(len(ret_images), 0, |
|
433 |
"No matching images found") |
|
434 |
return ret_images |
|
354 | 435 |
|
355 | 436 |
# ---------------------------------- |
356 | 437 |
# Pithos |
... | ... | |
387 | 468 |
self.clients.pithos.container = container |
388 | 469 |
self.clients.pithos.container_put() |
389 | 470 |
|
471 |
# ---------------------------------- |
|
472 |
# Servers |
|
473 |
def _get_list_of_servers(self, detail=False): |
|
474 |
"""Get (detailed) list of servers""" |
|
475 |
if detail: |
|
476 |
self.info("Getting detailed list of servers") |
|
477 |
else: |
|
478 |
self.info("Getting simple list of servers") |
|
479 |
return self.clients.cyclades.list_servers(detail=detail) |
|
480 |
|
|
481 |
def _get_server_details(self, server): |
|
482 |
"""Get details for a server""" |
|
483 |
self.info("Getting details for server %s with id %s", |
|
484 |
server['name'], server['id']) |
|
485 |
return self.clients.cyclades.get_server_details(server['id']) |
|
486 |
|
|
487 |
def _create_server(self, name, image, flavor): |
|
488 |
"""Create a new server""" |
|
489 |
self.info("Creating a server with name %s", name) |
|
490 |
self.info("Using image %s with id %s", image['name'], image['id']) |
|
491 |
self.info("Using flavor %s with id %s", flavor['name'], flavor['id']) |
|
492 |
server = self.clients.cyclades.create_server( |
|
493 |
name, flavor['id'], image['id']) |
|
494 |
|
|
495 |
self.info("Server id: %s", server['id']) |
|
496 |
self.info("Server password: %s", server['adminPass']) |
|
497 |
|
|
498 |
self.assertEqual(server['name'], name) |
|
499 |
self.assertEqual(server['flavor']['id'], flavor['id']) |
|
500 |
self.assertEqual(server['image']['id'], image['id']) |
|
501 |
self.assertEqual(server['status'], "BUILD") |
|
502 |
|
|
503 |
return server |
|
504 |
|
|
505 |
def _get_connection_username(self, server): |
|
506 |
"""Determine the username to use to connect to the server""" |
|
507 |
users = server['metadata'].get("users", None) |
|
508 |
ret_user = None |
|
509 |
if users is not None: |
|
510 |
user_list = users.split() |
|
511 |
if "root" in user_list: |
|
512 |
ret_user = "root" |
|
513 |
else: |
|
514 |
ret_user = random.choice(user_list) |
|
515 |
else: |
|
516 |
# Return the login name for connections based on the server OS |
|
517 |
self.info("Could not find `users' metadata in server. Let's guess") |
|
518 |
os_value = server['metadata'].get("os") |
|
519 |
if os_value in ("Ubuntu", "Kubuntu", "Fedora"): |
|
520 |
ret_user = "user" |
|
521 |
elif os_value in ("windows", "windows_alpha1"): |
|
522 |
ret_user = "Administrator" |
|
523 |
else: |
|
524 |
ret_user = "root" |
|
525 |
|
|
526 |
self.assertIsNotNone(ret_user) |
|
527 |
self.info("User's login name: %s", ret_user) |
|
528 |
return ret_user |
|
529 |
|
|
530 |
def _insist_on_server_transition(self, server, curr_status, new_status): |
|
531 |
"""Insist on server transiting from curr_status to new_status""" |
|
532 |
def check_fun(): |
|
533 |
"""Check server status""" |
|
534 |
srv = self.clients.cyclades.get_server_details(server['id']) |
|
535 |
if srv['status'] == curr_status: |
|
536 |
raise Retry() |
|
537 |
elif srv['status'] == new_status: |
|
538 |
return |
|
539 |
else: |
|
540 |
msg = "Server %s went to unexpected status %s" |
|
541 |
self.error(msg, server['name'], srv['status']) |
|
542 |
self.fail(msg % (server['name'], srv['status'])) |
|
543 |
opmsg = "Waiting for server %s to transit from %s to %s" |
|
544 |
self.info(opmsg, server['name'], curr_status, new_status) |
|
545 |
opmsg = opmsg % (server['name'], curr_status, new_status) |
|
546 |
self._try_until_timeout_expires(opmsg, check_fun) |
|
547 |
|
|
548 |
def _insist_on_tcp_connection(self, family, host, port): |
|
549 |
"""Insist on tcp connection""" |
|
550 |
def check_fun(): |
|
551 |
"""Get a connected socket from the specified family to host:port""" |
|
552 |
sock = None |
|
553 |
for res in socket.getaddrinfo(host, port, family, |
|
554 |
socket.SOCK_STREAM, 0, |
|
555 |
socket.AI_PASSIVE): |
|
556 |
fam, socktype, proto, _, saddr = res |
|
557 |
try: |
|
558 |
sock = socket.socket(fam, socktype, proto) |
|
559 |
except socket.error: |
|
560 |
sock = None |
|
561 |
continue |
|
562 |
try: |
|
563 |
sock.connect(saddr) |
|
564 |
except socket.error: |
|
565 |
sock.close() |
|
566 |
sock = None |
|
567 |
continue |
|
568 |
if sock is None: |
|
569 |
raise Retry |
|
570 |
return sock |
|
571 |
familystr = {socket.AF_INET: "IPv4", socket.AF_INET6: "IPv6", |
|
572 |
socket.AF_UNSPEC: "Unspecified-IPv4/6"} |
|
573 |
opmsg = "Connecting over %s to %s:%s" |
|
574 |
self.info(opmsg, familystr.get(family, "Unknown"), host, port) |
|
575 |
opmsg = opmsg % (familystr.get(family, "Unknown"), host, port) |
|
576 |
return self._try_until_timeout_expires(opmsg, check_fun) |
|
577 |
|
|
578 |
def _get_ip(self, server, version): |
|
579 |
"""Get the public IP of a server from the detailed server info""" |
|
580 |
assert version in (4, 6) |
|
581 |
|
|
582 |
nics = server['attachments'] |
|
583 |
public_addrs = None |
|
584 |
for nic in nics: |
|
585 |
net_id = nic['network_id'] |
|
586 |
if self.clients.cyclades.get_network_details(net_id)['public']: |
|
587 |
public_addrs = nic['ipv' + str(version)] |
|
588 |
|
|
589 |
self.assertIsNotNone(public_addrs) |
|
590 |
msg = "Servers %s public IPv%s is %s" |
|
591 |
self.info(msg, server['name'], version, public_addrs) |
|
592 |
return public_addrs |
|
593 |
|
|
594 |
def _insist_on_ping(self, ip_addr, version): |
|
595 |
"""Test server responds to a single IPv4 of IPv6 ping""" |
|
596 |
def check_fun(): |
|
597 |
"""Ping to server""" |
|
598 |
assert version in (4, 6) |
|
599 |
cmd = ("ping%s -c 3 -w 20 %s" % |
|
600 |
("6" if version == 6 else "", ip_addr)) |
|
601 |
ping = subprocess.Popen( |
|
602 |
cmd, shell=True, stdout=subprocess.PIPE, |
|
603 |
stderr=subprocess.PIPE) |
|
604 |
ping.communicate() |
|
605 |
ret = ping.wait() |
|
606 |
if ret != 0: |
|
607 |
raise Retry |
|
608 |
opmsg = "Sent IPv%s ping requests to %s" |
|
609 |
self.info(opmsg, version, ip_addr) |
|
610 |
opmsg = opmsg % (version, ip_addr) |
|
611 |
self._try_until_timeout_expires(opmsg, check_fun) |
|
612 |
|
|
390 | 613 |
|
391 | 614 |
# -------------------------------------------------------------------- |
392 | 615 |
# Initialize Burnin |
... | ... | |
412 | 635 |
BurninTests.action_warning = opts.action_warning |
413 | 636 |
BurninTests.query_interval = opts.query_interval |
414 | 637 |
BurninTests.system_user = opts.system_user |
638 |
BurninTests.flavors = opts.flavors |
|
639 |
BurninTests.images = opts.images |
|
415 | 640 |
BurninTests.run_id = SNF_TEST_PREFIX + \ |
416 | 641 |
datetime.datetime.strftime(datetime.datetime.now(), "%Y%m%d%H%M%S") |
417 | 642 |
|
... | ... | |
427 | 652 |
|
428 | 653 |
# -------------------------------------------------------------------- |
429 | 654 |
# Run Burnin |
430 |
def run(testsuites, failfast=False, final_report=False): |
|
655 |
def run_burnin(testsuites, failfast=False, final_report=False):
|
|
431 | 656 |
"""Run burnin testsuites""" |
432 | 657 |
global logger # Using global. pylint: disable-msg=C0103,W0603,W0602 |
433 | 658 |
|
434 | 659 |
success = True |
435 | 660 |
for tcase in testsuites: |
436 |
tsuite = unittest.TestLoader().loadTestsFromTestCase(tcase) |
|
437 |
results = tsuite.run(BurninTestResult()) |
|
438 |
|
|
439 |
was_success = was_successful(tcase.__name__, results.wasSuccessful()) |
|
661 |
was_success = run_test(tcase) |
|
440 | 662 |
success = success and was_success |
441 | 663 |
if failfast and not success: |
442 | 664 |
break |
... | ... | |
451 | 673 |
return 0 if success else 1 |
452 | 674 |
|
453 | 675 |
|
676 |
def run_test(tcase): |
|
677 |
"""Run a testcase""" |
|
678 |
tsuite = unittest.TestLoader().loadTestsFromTestCase(tcase) |
|
679 |
results = tsuite.run(BurninTestResult()) |
|
680 |
|
|
681 |
return was_successful(tcase.__name__, results.wasSuccessful()) |
|
682 |
|
|
683 |
|
|
454 | 684 |
# -------------------------------------------------------------------- |
455 | 685 |
# Helper functions |
456 | 686 |
def was_successful(tsuite, success): |
... | ... | |
494 | 724 |
|
495 | 725 |
def __set__(self, obj, value): |
496 | 726 |
self.val = value |
727 |
|
|
728 |
|
|
729 |
class Retry(Exception): |
|
730 |
"""Retry the action |
|
731 |
|
|
732 |
This is used by _try_unit_timeout_expires method. |
|
733 |
|
|
734 |
""" |
Also available in: Unified diff