root / snf-tools / synnefo_tools / burnin.py @ 6e168615
History | View | Annotate | Download (71.3 kB)
| 1 |
#!/usr/bin/env python
|
|---|---|
| 2 |
|
| 3 |
# Copyright 2011 GRNET S.A. All rights reserved.
|
| 4 |
#
|
| 5 |
# Redistribution and use in source and binary forms, with or
|
| 6 |
# without modification, are permitted provided that the following
|
| 7 |
# conditions are met:
|
| 8 |
#
|
| 9 |
# 1. Redistributions of source code must retain the above
|
| 10 |
# copyright notice, this list of conditions and the following
|
| 11 |
# disclaimer.
|
| 12 |
#
|
| 13 |
# 2. Redistributions in binary form must reproduce the above
|
| 14 |
# copyright notice, this list of conditions and the following
|
| 15 |
# disclaimer in the documentation and/or other materials
|
| 16 |
# provided with the distribution.
|
| 17 |
#
|
| 18 |
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
|
| 19 |
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
| 20 |
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
|
| 21 |
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
|
| 22 |
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
| 23 |
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
| 24 |
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
|
| 25 |
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
|
| 26 |
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
| 27 |
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
|
| 28 |
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
| 29 |
# POSSIBILITY OF SUCH DAMAGE.
|
| 30 |
#
|
| 31 |
# The views and conclusions contained in the software and
|
| 32 |
# documentation are those of the authors and should not be
|
| 33 |
# interpreted as representing official policies, either expressed
|
| 34 |
# or implied, of GRNET S.A.
|
| 35 |
|
| 36 |
"""Perform integration testing on a running Synnefo deployment"""
|
| 37 |
|
| 38 |
#import __main__
|
| 39 |
import datetime |
| 40 |
import inspect |
| 41 |
import logging |
| 42 |
import os |
| 43 |
import paramiko |
| 44 |
import prctl |
| 45 |
import subprocess |
| 46 |
import signal |
| 47 |
import socket |
| 48 |
import sys |
| 49 |
import time |
| 50 |
from base64 import b64encode |
| 51 |
from IPy import IP |
| 52 |
from multiprocessing import Process, Queue |
| 53 |
from random import choice |
| 54 |
from optparse import OptionParser, OptionValueError |
| 55 |
|
| 56 |
from kamaki.clients.compute import ComputeClient |
| 57 |
from kamaki.clients.cyclades import CycladesClient |
| 58 |
from kamaki.clients.image import ImageClient |
| 59 |
from kamaki.clients import ClientError |
| 60 |
|
| 61 |
import fabric.api as fabric |
| 62 |
|
| 63 |
from vncauthproxy.d3des import generate_response as d3des_generate_response |
| 64 |
|
| 65 |
# Use backported unittest functionality if Python < 2.7
|
| 66 |
try:
|
| 67 |
import unittest2 as unittest |
| 68 |
except ImportError: |
| 69 |
if sys.version_info < (2, 7): |
| 70 |
raise Exception("The unittest2 package is required for Python < 2.7") |
| 71 |
import unittest |
| 72 |
|
| 73 |
# --------------------------------------------------------------------
|
| 74 |
# Global Variables
|
| 75 |
API = None
|
| 76 |
TOKEN = None
|
| 77 |
PLANKTON = None
|
| 78 |
PLANKTON_USER = None
|
| 79 |
NO_IPV6 = None
|
| 80 |
DEFAULT_API = "https://cyclades.okeanos.grnet.gr/api/v1.1"
|
| 81 |
DEFAULT_PLANKTON = "https://cyclades.okeanos.grnet.gr/plankton"
|
| 82 |
DEFAULT_PLANKTON_USER = "images@okeanos.grnet.gr"
|
| 83 |
NOFAILFAST = None
|
| 84 |
VERBOSE = None
|
| 85 |
|
| 86 |
# A unique id identifying this test run
|
| 87 |
TEST_RUN_ID = datetime.datetime.strftime(datetime.datetime.now(), |
| 88 |
"%Y%m%d%H%M%S")
|
| 89 |
SNF_TEST_PREFIX = "snf-test-"
|
| 90 |
|
| 91 |
red = '\x1b[31m'
|
| 92 |
yellow = '\x1b[33m'
|
| 93 |
green = '\x1b[32m'
|
| 94 |
normal = '\x1b[0m'
|
| 95 |
|
| 96 |
|
| 97 |
# --------------------------------------------------------------------
|
| 98 |
# BurninTestReulst class
|
| 99 |
class BurninTestResult(unittest.TextTestResult): |
| 100 |
def addSuccess(self, test): |
| 101 |
super(BurninTestResult, self).addSuccess(test) |
| 102 |
if self.showAll: |
| 103 |
if hasattr(test, 'result_dict'): |
| 104 |
run_details = test.result_dict |
| 105 |
|
| 106 |
self.stream.write("\n") |
| 107 |
for i in run_details: |
| 108 |
self.stream.write("%s : %s \n" % (i, run_details[i])) |
| 109 |
self.stream.write("\n") |
| 110 |
|
| 111 |
elif self.dots: |
| 112 |
self.stream.write('.') |
| 113 |
self.stream.flush()
|
| 114 |
|
| 115 |
def addError(self, test, err): |
| 116 |
super(BurninTestResult, self).addError(test, err) |
| 117 |
if self.showAll: |
| 118 |
self.stream.writeln("ERROR") |
| 119 |
if hasattr(test, 'result_dict'): |
| 120 |
run_details = test.result_dict |
| 121 |
|
| 122 |
self.stream.write("\n") |
| 123 |
for i in run_details: |
| 124 |
self.stream.write("%s : %s \n" % (i, run_details[i])) |
| 125 |
self.stream.write("\n") |
| 126 |
|
| 127 |
elif self.dots: |
| 128 |
self.stream.write('E') |
| 129 |
self.stream.flush()
|
| 130 |
|
| 131 |
def addFailure(self, test, err): |
| 132 |
super(BurninTestResult, self).addFailure(test, err) |
| 133 |
if self.showAll: |
| 134 |
self.stream.writeln("FAIL") |
| 135 |
if hasattr(test, 'result_dict'): |
| 136 |
run_details = test.result_dict |
| 137 |
|
| 138 |
self.stream.write("\n") |
| 139 |
for i in run_details: |
| 140 |
self.stream.write("%s : %s \n" % (i, run_details[i])) |
| 141 |
self.stream.write("\n") |
| 142 |
|
| 143 |
elif self.dots: |
| 144 |
self.stream.write('F') |
| 145 |
self.stream.flush()
|
| 146 |
|
| 147 |
|
| 148 |
# --------------------------------------------------------------------
|
| 149 |
# Format Results
|
| 150 |
class burninFormatter(logging.Formatter): |
| 151 |
err_fmt = red + "ERROR: %(msg)s" + normal
|
| 152 |
dbg_fmt = green + "* %(msg)s" + normal
|
| 153 |
info_fmt = "%(msg)s"
|
| 154 |
|
| 155 |
def __init__(self, fmt="%(levelno)s: %(msg)s"): |
| 156 |
logging.Formatter.__init__(self, fmt)
|
| 157 |
|
| 158 |
def format(self, record): |
| 159 |
format_orig = self._fmt
|
| 160 |
# Replace the original format with one customized by logging level
|
| 161 |
if record.levelno == 10: # DEBUG |
| 162 |
self._fmt = burninFormatter.dbg_fmt
|
| 163 |
elif record.levelno == 20: # INFO |
| 164 |
self._fmt = burninFormatter.info_fmt
|
| 165 |
elif record.levelno == 40: # ERROR |
| 166 |
self._fmt = burninFormatter.err_fmt
|
| 167 |
result = logging.Formatter.format(self, record)
|
| 168 |
self._fmt = format_orig
|
| 169 |
return result
|
| 170 |
|
| 171 |
log = logging.getLogger("burnin")
|
| 172 |
log.setLevel(logging.DEBUG) |
| 173 |
handler = logging.StreamHandler() |
| 174 |
handler.setFormatter(burninFormatter()) |
| 175 |
log.addHandler(handler) |
| 176 |
|
| 177 |
|
| 178 |
# --------------------------------------------------------------------
|
| 179 |
# UnauthorizedTestCase class
|
| 180 |
class UnauthorizedTestCase(unittest.TestCase): |
| 181 |
"""Test unauthorized access"""
|
| 182 |
@classmethod
|
| 183 |
def setUpClass(cls): |
| 184 |
cls.result_dict = dict()
|
| 185 |
|
| 186 |
def test_unauthorized_access(self): |
| 187 |
"""Test access without a valid token fails"""
|
| 188 |
log.info("Authentication test")
|
| 189 |
falseToken = '12345'
|
| 190 |
c = ComputeClient(API, falseToken) |
| 191 |
|
| 192 |
with self.assertRaises(ClientError) as cm: |
| 193 |
c.list_servers() |
| 194 |
self.assertEqual(cm.exception.status, 401) |
| 195 |
|
| 196 |
|
| 197 |
# --------------------------------------------------------------------
|
| 198 |
# ImagesTestCase class
|
| 199 |
class ImagesTestCase(unittest.TestCase): |
| 200 |
"""Test image lists for consistency"""
|
| 201 |
@classmethod
|
| 202 |
def setUpClass(cls): |
| 203 |
"""Initialize kamaki, get (detailed) list of images"""
|
| 204 |
log.info("Getting simple and detailed list of images")
|
| 205 |
cls.client = ComputeClient(API, TOKEN) |
| 206 |
cls.plankton = ImageClient(PLANKTON, TOKEN) |
| 207 |
cls.images = cls.plankton.list_public() |
| 208 |
cls.dimages = cls.plankton.list_public(detail=True)
|
| 209 |
cls.result_dict = dict()
|
| 210 |
|
| 211 |
def test_001_list_images(self): |
| 212 |
"""Test image list actually returns images"""
|
| 213 |
self.assertGreater(len(self.images), 0) |
| 214 |
|
| 215 |
def test_002_list_images_detailed(self): |
| 216 |
"""Test detailed image list is the same length as list"""
|
| 217 |
self.assertEqual(len(self.dimages), len(self.images)) |
| 218 |
|
| 219 |
def test_003_same_image_names(self): |
| 220 |
"""Test detailed and simple image list contain same names"""
|
| 221 |
names = sorted(map(lambda x: x["name"], self.images)) |
| 222 |
dnames = sorted(map(lambda x: x["name"], self.dimages)) |
| 223 |
self.assertEqual(names, dnames)
|
| 224 |
|
| 225 |
def test_004_unique_image_names(self): |
| 226 |
"""Test system images have unique names"""
|
| 227 |
sys_images = filter(lambda x: x['owner'] == PLANKTON_USER, |
| 228 |
self.dimages)
|
| 229 |
names = sorted(map(lambda x: x["name"], sys_images)) |
| 230 |
self.assertEqual(sorted(list(set(names))), names) |
| 231 |
|
| 232 |
def test_005_image_metadata(self): |
| 233 |
"""Test every image has specific metadata defined"""
|
| 234 |
keys = frozenset(["osfamily", "root_partition"]) |
| 235 |
details = self.client.list_images(detail=True) |
| 236 |
for i in details: |
| 237 |
self.assertTrue(keys.issubset(i["metadata"]["values"].keys())) |
| 238 |
|
| 239 |
|
| 240 |
# --------------------------------------------------------------------
|
| 241 |
# FlavorsTestCase class
|
| 242 |
class FlavorsTestCase(unittest.TestCase): |
| 243 |
"""Test flavor lists for consistency"""
|
| 244 |
@classmethod
|
| 245 |
def setUpClass(cls): |
| 246 |
"""Initialize kamaki, get (detailed) list of flavors"""
|
| 247 |
log.info("Getting simple and detailed list of flavors")
|
| 248 |
cls.client = ComputeClient(API, TOKEN) |
| 249 |
cls.flavors = cls.client.list_flavors() |
| 250 |
cls.dflavors = cls.client.list_flavors(detail=True)
|
| 251 |
cls.result_dict = dict()
|
| 252 |
|
| 253 |
def test_001_list_flavors(self): |
| 254 |
"""Test flavor list actually returns flavors"""
|
| 255 |
self.assertGreater(len(self.flavors), 0) |
| 256 |
|
| 257 |
def test_002_list_flavors_detailed(self): |
| 258 |
"""Test detailed flavor list is the same length as list"""
|
| 259 |
self.assertEquals(len(self.dflavors), len(self.flavors)) |
| 260 |
|
| 261 |
def test_003_same_flavor_names(self): |
| 262 |
"""Test detailed and simple flavor list contain same names"""
|
| 263 |
names = sorted(map(lambda x: x["name"], self.flavors)) |
| 264 |
dnames = sorted(map(lambda x: x["name"], self.dflavors)) |
| 265 |
self.assertEqual(names, dnames)
|
| 266 |
|
| 267 |
def test_004_unique_flavor_names(self): |
| 268 |
"""Test flavors have unique names"""
|
| 269 |
names = sorted(map(lambda x: x["name"], self.flavors)) |
| 270 |
self.assertEqual(sorted(list(set(names))), names) |
| 271 |
|
| 272 |
def test_005_well_formed_flavor_names(self): |
| 273 |
"""Test flavors have names of the form CxxRyyDzz
|
| 274 |
Where xx is vCPU count, yy is RAM in MiB, zz is Disk in GiB
|
| 275 |
"""
|
| 276 |
for f in self.dflavors: |
| 277 |
self.assertEqual("C%dR%dD%d" % (f["cpu"], f["ram"], f["disk"]), |
| 278 |
f["name"],
|
| 279 |
"Flavor %s does not match its specs." % f["name"]) |
| 280 |
|
| 281 |
|
| 282 |
# --------------------------------------------------------------------
|
| 283 |
# ServersTestCase class
|
| 284 |
class ServersTestCase(unittest.TestCase): |
| 285 |
"""Test server lists for consistency"""
|
| 286 |
@classmethod
|
| 287 |
def setUpClass(cls): |
| 288 |
"""Initialize kamaki, get (detailed) list of servers"""
|
| 289 |
log.info("Getting simple and detailed list of servers")
|
| 290 |
|
| 291 |
cls.client = ComputeClient(API, TOKEN) |
| 292 |
cls.servers = cls.client.list_servers() |
| 293 |
cls.dservers = cls.client.list_servers(detail=True)
|
| 294 |
cls.result_dict = dict()
|
| 295 |
|
| 296 |
# def test_001_list_servers(self):
|
| 297 |
# """Test server list actually returns servers"""
|
| 298 |
# self.assertGreater(len(self.servers), 0)
|
| 299 |
|
| 300 |
def test_002_list_servers_detailed(self): |
| 301 |
"""Test detailed server list is the same length as list"""
|
| 302 |
self.assertEqual(len(self.dservers), len(self.servers)) |
| 303 |
|
| 304 |
def test_003_same_server_names(self): |
| 305 |
"""Test detailed and simple flavor list contain same names"""
|
| 306 |
names = sorted(map(lambda x: x["name"], self.servers)) |
| 307 |
dnames = sorted(map(lambda x: x["name"], self.dservers)) |
| 308 |
self.assertEqual(names, dnames)
|
| 309 |
|
| 310 |
|
| 311 |
# This class gets replicated into actual TestCases dynamically
|
| 312 |
class SpawnServerTestCase(unittest.TestCase): |
| 313 |
"""Test scenario for server of the specified image"""
|
| 314 |
|
| 315 |
@classmethod
|
| 316 |
def setUpClass(cls): |
| 317 |
"""Initialize a kamaki instance"""
|
| 318 |
log.info("Spawning server for image `%s'" % cls.imagename)
|
| 319 |
cls.client = ComputeClient(API, TOKEN) |
| 320 |
cls.cyclades = CycladesClient(API, TOKEN) |
| 321 |
cls.result_dict = dict()
|
| 322 |
|
| 323 |
def _get_ipv4(self, server): |
| 324 |
"""Get the public IPv4 of a server from the detailed server info"""
|
| 325 |
|
| 326 |
nics = server["attachments"]["values"] |
| 327 |
|
| 328 |
for nic in nics: |
| 329 |
net_id = nic["network_id"]
|
| 330 |
if self.cyclades.get_network_details(net_id)["public"]: |
| 331 |
public_addrs = nic["ipv4"]
|
| 332 |
|
| 333 |
self.assertTrue(public_addrs is not None) |
| 334 |
|
| 335 |
return public_addrs
|
| 336 |
|
| 337 |
def _get_ipv6(self, server): |
| 338 |
"""Get the public IPv6 of a server from the detailed server info"""
|
| 339 |
|
| 340 |
nics = server["attachments"]["values"] |
| 341 |
|
| 342 |
for nic in nics: |
| 343 |
net_id = nic["network_id"]
|
| 344 |
if self.cyclades.get_network_details(net_id)["public"]: |
| 345 |
public_addrs = nic["ipv6"]
|
| 346 |
|
| 347 |
self.assertTrue(public_addrs is not None) |
| 348 |
|
| 349 |
return public_addrs
|
| 350 |
|
| 351 |
def _connect_loginname(self, os_value): |
| 352 |
"""Return the login name for connections based on the server OS"""
|
| 353 |
if os_value in ("Ubuntu", "Kubuntu", "Fedora"): |
| 354 |
return "user" |
| 355 |
elif os_value in ("windows", "windows_alpha1"): |
| 356 |
return "Administrator" |
| 357 |
else:
|
| 358 |
return "root" |
| 359 |
|
| 360 |
def _verify_server_status(self, current_status, new_status): |
| 361 |
"""Verify a server has switched to a specified status"""
|
| 362 |
server = self.client.get_server_details(self.serverid) |
| 363 |
if server["status"] not in (current_status, new_status): |
| 364 |
return None # Do not raise exception, return so the test fails |
| 365 |
self.assertEquals(server["status"], new_status) |
| 366 |
|
| 367 |
def _get_connected_tcp_socket(self, family, host, port): |
| 368 |
"""Get a connected socket from the specified family to host:port"""
|
| 369 |
sock = None
|
| 370 |
for res in \ |
| 371 |
socket.getaddrinfo(host, port, family, socket.SOCK_STREAM, 0,
|
| 372 |
socket.AI_PASSIVE): |
| 373 |
af, socktype, proto, canonname, sa = res |
| 374 |
try:
|
| 375 |
sock = socket.socket(af, socktype, proto) |
| 376 |
except socket.error:
|
| 377 |
sock = None
|
| 378 |
continue
|
| 379 |
try:
|
| 380 |
sock.connect(sa) |
| 381 |
except socket.error:
|
| 382 |
sock.close() |
| 383 |
sock = None
|
| 384 |
continue
|
| 385 |
self.assertIsNotNone(sock)
|
| 386 |
return sock
|
| 387 |
|
| 388 |
def _ping_once(self, ipv6, ip): |
| 389 |
"""Test server responds to a single IPv4 or IPv6 ping"""
|
| 390 |
cmd = "ping%s -c 2 -w 3 %s" % ("6" if ipv6 else "", ip) |
| 391 |
ping = subprocess.Popen(cmd, shell=True,
|
| 392 |
stdout=subprocess.PIPE, stderr=subprocess.PIPE) |
| 393 |
(stdout, stderr) = ping.communicate() |
| 394 |
ret = ping.wait() |
| 395 |
self.assertEquals(ret, 0) |
| 396 |
|
| 397 |
def _get_hostname_over_ssh(self, hostip, username, password): |
| 398 |
ssh = paramiko.SSHClient() |
| 399 |
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) |
| 400 |
try:
|
| 401 |
ssh.connect(hostip, username=username, password=password) |
| 402 |
except socket.error:
|
| 403 |
raise AssertionError |
| 404 |
stdin, stdout, stderr = ssh.exec_command("hostname")
|
| 405 |
lines = stdout.readlines() |
| 406 |
self.assertEqual(len(lines), 1) |
| 407 |
return lines[0] |
| 408 |
|
| 409 |
def _try_until_timeout_expires(self, warn_timeout, fail_timeout, |
| 410 |
opmsg, callable, *args, **kwargs):
|
| 411 |
if warn_timeout == fail_timeout:
|
| 412 |
warn_timeout = fail_timeout + 1
|
| 413 |
warn_tmout = time.time() + warn_timeout |
| 414 |
fail_tmout = time.time() + fail_timeout |
| 415 |
while True: |
| 416 |
self.assertLess(time.time(), fail_tmout,
|
| 417 |
"operation `%s' timed out" % opmsg)
|
| 418 |
if time.time() > warn_tmout:
|
| 419 |
log.warning("Server %d: `%s' operation `%s' not done yet",
|
| 420 |
self.serverid, self.servername, opmsg) |
| 421 |
try:
|
| 422 |
log.info("%s... " % opmsg)
|
| 423 |
return callable(*args, **kwargs) |
| 424 |
except AssertionError: |
| 425 |
pass
|
| 426 |
time.sleep(self.query_interval)
|
| 427 |
|
| 428 |
def _insist_on_tcp_connection(self, family, host, port): |
| 429 |
familystr = {socket.AF_INET: "IPv4", socket.AF_INET6: "IPv6",
|
| 430 |
socket.AF_UNSPEC: "Unspecified-IPv4/6"}
|
| 431 |
msg = "connect over %s to %s:%s" % \
|
| 432 |
(familystr.get(family, "Unknown"), host, port)
|
| 433 |
sock = self._try_until_timeout_expires(
|
| 434 |
self.action_timeout, self.action_timeout, |
| 435 |
msg, self._get_connected_tcp_socket,
|
| 436 |
family, host, port) |
| 437 |
return sock
|
| 438 |
|
| 439 |
def _insist_on_status_transition(self, current_status, new_status, |
| 440 |
fail_timeout, warn_timeout=None):
|
| 441 |
msg = "Server %d: `%s', waiting for %s -> %s" % \
|
| 442 |
(self.serverid, self.servername, current_status, new_status) |
| 443 |
if warn_timeout is None: |
| 444 |
warn_timeout = fail_timeout |
| 445 |
self._try_until_timeout_expires(warn_timeout, fail_timeout,
|
| 446 |
msg, self._verify_server_status,
|
| 447 |
current_status, new_status) |
| 448 |
# Ensure the status is actually the expected one
|
| 449 |
server = self.client.get_server_details(self.serverid) |
| 450 |
self.assertEquals(server["status"], new_status) |
| 451 |
|
| 452 |
def _insist_on_ssh_hostname(self, hostip, username, password): |
| 453 |
msg = "SSH to %s, as %s/%s" % (hostip, username, password)
|
| 454 |
hostname = self._try_until_timeout_expires(
|
| 455 |
self.action_timeout, self.action_timeout, |
| 456 |
msg, self._get_hostname_over_ssh,
|
| 457 |
hostip, username, password) |
| 458 |
|
| 459 |
# The hostname must be of the form 'prefix-id'
|
| 460 |
self.assertTrue(hostname.endswith("-%d\n" % self.serverid)) |
| 461 |
|
| 462 |
def _check_file_through_ssh(self, hostip, username, password, |
| 463 |
remotepath, content): |
| 464 |
msg = "Trying file injection through SSH to %s, as %s/%s" % \
|
| 465 |
(hostip, username, password) |
| 466 |
log.info(msg) |
| 467 |
try:
|
| 468 |
ssh = paramiko.SSHClient() |
| 469 |
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) |
| 470 |
ssh.connect(hostip, username=username, password=password) |
| 471 |
except socket.error:
|
| 472 |
raise AssertionError |
| 473 |
|
| 474 |
transport = paramiko.Transport((hostip, 22))
|
| 475 |
transport.connect(username=username, password=password) |
| 476 |
|
| 477 |
localpath = '/tmp/' + SNF_TEST_PREFIX + 'injection' |
| 478 |
sftp = paramiko.SFTPClient.from_transport(transport) |
| 479 |
sftp.get(remotepath, localpath) |
| 480 |
sftp.close() |
| 481 |
transport.close() |
| 482 |
|
| 483 |
f = open(localpath)
|
| 484 |
remote_content = b64encode(f.read()) |
| 485 |
|
| 486 |
# Check if files are the same
|
| 487 |
return (remote_content == content)
|
| 488 |
|
| 489 |
def _skipIf(self, condition, msg): |
| 490 |
if condition:
|
| 491 |
self.skipTest(msg)
|
| 492 |
|
| 493 |
def test_001_submit_create_server(self): |
| 494 |
"""Test submit create server request"""
|
| 495 |
|
| 496 |
log.info("Submit new server request")
|
| 497 |
server = self.client.create_server(self.servername, self.flavorid, |
| 498 |
self.imageid, self.personality) |
| 499 |
|
| 500 |
log.info("Server id: " + str(server["id"])) |
| 501 |
log.info("Server password: " + server["adminPass"]) |
| 502 |
self.assertEqual(server["name"], self.servername) |
| 503 |
self.assertEqual(server["flavorRef"], self.flavorid) |
| 504 |
self.assertEqual(server["imageRef"], self.imageid) |
| 505 |
self.assertEqual(server["status"], "BUILD") |
| 506 |
|
| 507 |
# Update class attributes to reflect data on building server
|
| 508 |
cls = type(self) |
| 509 |
cls.serverid = server["id"]
|
| 510 |
cls.username = None
|
| 511 |
cls.passwd = server["adminPass"]
|
| 512 |
|
| 513 |
self.result_dict["Server ID"] = str(server["id"]) |
| 514 |
self.result_dict["Password"] = str(server["adminPass"]) |
| 515 |
|
| 516 |
def test_002a_server_is_building_in_list(self): |
| 517 |
"""Test server is in BUILD state, in server list"""
|
| 518 |
log.info("Server in BUILD state in server list")
|
| 519 |
|
| 520 |
self.result_dict.clear()
|
| 521 |
|
| 522 |
servers = self.client.list_servers(detail=True) |
| 523 |
servers = filter(lambda x: x["name"] == self.servername, servers) |
| 524 |
|
| 525 |
server = servers[0]
|
| 526 |
self.assertEqual(server["name"], self.servername) |
| 527 |
self.assertEqual(server["flavorRef"], self.flavorid) |
| 528 |
self.assertEqual(server["imageRef"], self.imageid) |
| 529 |
self.assertEqual(server["status"], "BUILD") |
| 530 |
|
| 531 |
def test_002b_server_is_building_in_details(self): |
| 532 |
"""Test server is in BUILD state, in details"""
|
| 533 |
|
| 534 |
log.info("Server in BUILD state in details")
|
| 535 |
|
| 536 |
server = self.client.get_server_details(self.serverid) |
| 537 |
self.assertEqual(server["name"], self.servername) |
| 538 |
self.assertEqual(server["flavorRef"], self.flavorid) |
| 539 |
self.assertEqual(server["imageRef"], self.imageid) |
| 540 |
self.assertEqual(server["status"], "BUILD") |
| 541 |
|
| 542 |
def test_002c_set_server_metadata(self): |
| 543 |
|
| 544 |
log.info("Creating server metadata")
|
| 545 |
|
| 546 |
image = self.client.get_image_details(self.imageid) |
| 547 |
os_value = image["metadata"]["values"]["os"] |
| 548 |
users = image["metadata"]["values"].get("users", None) |
| 549 |
self.client.update_server_metadata(self.serverid, OS=os_value) |
| 550 |
|
| 551 |
userlist = users.split() |
| 552 |
|
| 553 |
# Determine the username to use for future connections
|
| 554 |
# to this host
|
| 555 |
cls = type(self) |
| 556 |
|
| 557 |
if "root" in userlist: |
| 558 |
cls.username = "root"
|
| 559 |
elif users is None: |
| 560 |
cls.username = self._connect_loginname(os_value)
|
| 561 |
else:
|
| 562 |
cls.username = choice(userlist) |
| 563 |
|
| 564 |
self.assertIsNotNone(cls.username)
|
| 565 |
|
| 566 |
def test_002d_verify_server_metadata(self): |
| 567 |
"""Test server metadata keys are set based on image metadata"""
|
| 568 |
|
| 569 |
log.info("Verifying image metadata")
|
| 570 |
|
| 571 |
servermeta = self.client.get_server_metadata(self.serverid) |
| 572 |
imagemeta = self.client.get_image_metadata(self.imageid) |
| 573 |
|
| 574 |
self.assertEqual(servermeta["OS"], imagemeta["os"]) |
| 575 |
|
| 576 |
def test_003_server_becomes_active(self): |
| 577 |
"""Test server becomes ACTIVE"""
|
| 578 |
|
| 579 |
log.info("Waiting for server to become ACTIVE")
|
| 580 |
|
| 581 |
self._insist_on_status_transition(
|
| 582 |
"BUILD", "ACTIVE", self.build_fail, self.build_warning) |
| 583 |
|
| 584 |
def test_003a_get_server_oob_console(self): |
| 585 |
"""Test getting OOB server console over VNC
|
| 586 |
|
| 587 |
Implementation of RFB protocol follows
|
| 588 |
http://www.realvnc.com/docs/rfbproto.pdf.
|
| 589 |
|
| 590 |
"""
|
| 591 |
console = self.cyclades.get_server_console(self.serverid) |
| 592 |
self.assertEquals(console['type'], "vnc") |
| 593 |
sock = self._insist_on_tcp_connection(
|
| 594 |
socket.AF_INET, console["host"], console["port"]) |
| 595 |
|
| 596 |
# Step 1. ProtocolVersion message (par. 6.1.1)
|
| 597 |
version = sock.recv(1024)
|
| 598 |
self.assertEquals(version, 'RFB 003.008\n') |
| 599 |
sock.send(version) |
| 600 |
|
| 601 |
# Step 2. Security (par 6.1.2): Only VNC Authentication supported
|
| 602 |
sec = sock.recv(1024)
|
| 603 |
self.assertEquals(list(sec), ['\x01', '\x02']) |
| 604 |
|
| 605 |
# Step 3. Request VNC Authentication (par 6.1.2)
|
| 606 |
sock.send('\x02')
|
| 607 |
|
| 608 |
# Step 4. Receive Challenge (par 6.2.2)
|
| 609 |
challenge = sock.recv(1024)
|
| 610 |
self.assertEquals(len(challenge), 16) |
| 611 |
|
| 612 |
# Step 5. DES-Encrypt challenge, use password as key (par 6.2.2)
|
| 613 |
response = d3des_generate_response( |
| 614 |
(console["password"] + '\0' * 8)[:8], challenge) |
| 615 |
sock.send(response) |
| 616 |
|
| 617 |
# Step 6. SecurityResult (par 6.1.3)
|
| 618 |
result = sock.recv(4)
|
| 619 |
self.assertEquals(list(result), ['\x00', '\x00', '\x00', '\x00']) |
| 620 |
sock.close() |
| 621 |
|
| 622 |
def test_004_server_has_ipv4(self): |
| 623 |
"""Test active server has a valid IPv4 address"""
|
| 624 |
|
| 625 |
log.info("Validate server's IPv4")
|
| 626 |
|
| 627 |
server = self.client.get_server_details(self.serverid) |
| 628 |
ipv4 = self._get_ipv4(server)
|
| 629 |
|
| 630 |
self.result_dict.clear()
|
| 631 |
self.result_dict["IPv4"] = str(ipv4) |
| 632 |
|
| 633 |
self.assertEquals(IP(ipv4).version(), 4) |
| 634 |
|
| 635 |
def test_005_server_has_ipv6(self): |
| 636 |
"""Test active server has a valid IPv6 address"""
|
| 637 |
self._skipIf(NO_IPV6, "--no-ipv6 flag enabled") |
| 638 |
|
| 639 |
log.info("Validate server's IPv6")
|
| 640 |
|
| 641 |
server = self.client.get_server_details(self.serverid) |
| 642 |
ipv6 = self._get_ipv6(server)
|
| 643 |
|
| 644 |
self.result_dict.clear()
|
| 645 |
self.result_dict["IPv6"] = str(ipv6) |
| 646 |
|
| 647 |
self.assertEquals(IP(ipv6).version(), 6) |
| 648 |
|
| 649 |
def test_006_server_responds_to_ping_IPv4(self): |
| 650 |
"""Test server responds to ping on IPv4 address"""
|
| 651 |
|
| 652 |
log.info("Testing if server responds to pings in IPv4")
|
| 653 |
self.result_dict.clear()
|
| 654 |
|
| 655 |
server = self.client.get_server_details(self.serverid) |
| 656 |
ip = self._get_ipv4(server)
|
| 657 |
self._try_until_timeout_expires(self.action_timeout, |
| 658 |
self.action_timeout,
|
| 659 |
"PING IPv4 to %s" % ip,
|
| 660 |
self._ping_once,
|
| 661 |
False, ip)
|
| 662 |
|
| 663 |
def test_007_server_responds_to_ping_IPv6(self): |
| 664 |
"""Test server responds to ping on IPv6 address"""
|
| 665 |
self._skipIf(NO_IPV6, "--no-ipv6 flag enabled") |
| 666 |
log.info("Testing if server responds to pings in IPv6")
|
| 667 |
|
| 668 |
server = self.client.get_server_details(self.serverid) |
| 669 |
ip = self._get_ipv6(server)
|
| 670 |
self._try_until_timeout_expires(self.action_timeout, |
| 671 |
self.action_timeout,
|
| 672 |
"PING IPv6 to %s" % ip,
|
| 673 |
self._ping_once,
|
| 674 |
True, ip)
|
| 675 |
|
| 676 |
def test_008_submit_shutdown_request(self): |
| 677 |
"""Test submit request to shutdown server"""
|
| 678 |
|
| 679 |
log.info("Shutting down server")
|
| 680 |
|
| 681 |
self.cyclades.shutdown_server(self.serverid) |
| 682 |
|
| 683 |
def test_009_server_becomes_stopped(self): |
| 684 |
"""Test server becomes STOPPED"""
|
| 685 |
|
| 686 |
log.info("Waiting until server becomes STOPPED")
|
| 687 |
self._insist_on_status_transition(
|
| 688 |
"ACTIVE", "STOPPED", self.action_timeout, self.action_timeout) |
| 689 |
|
| 690 |
def test_010_submit_start_request(self): |
| 691 |
"""Test submit start server request"""
|
| 692 |
|
| 693 |
log.info("Starting server")
|
| 694 |
|
| 695 |
self.cyclades.start_server(self.serverid) |
| 696 |
|
| 697 |
def test_011_server_becomes_active(self): |
| 698 |
"""Test server becomes ACTIVE again"""
|
| 699 |
|
| 700 |
log.info("Waiting until server becomes ACTIVE")
|
| 701 |
self._insist_on_status_transition(
|
| 702 |
"STOPPED", "ACTIVE", self.action_timeout, self.action_timeout) |
| 703 |
|
| 704 |
def test_011a_server_responds_to_ping_IPv4(self): |
| 705 |
"""Test server OS is actually up and running again"""
|
| 706 |
|
| 707 |
log.info("Testing if server is actually up and running")
|
| 708 |
|
| 709 |
self.test_006_server_responds_to_ping_IPv4()
|
| 710 |
|
| 711 |
def test_012_ssh_to_server_IPv4(self): |
| 712 |
"""Test SSH to server public IPv4 works, verify hostname"""
|
| 713 |
|
| 714 |
self._skipIf(self.is_windows, "only valid for Linux servers") |
| 715 |
server = self.client.get_server_details(self.serverid) |
| 716 |
self._insist_on_ssh_hostname(self._get_ipv4(server), |
| 717 |
self.username, self.passwd) |
| 718 |
|
| 719 |
def test_013_ssh_to_server_IPv6(self): |
| 720 |
"""Test SSH to server public IPv6 works, verify hostname"""
|
| 721 |
self._skipIf(self.is_windows, "only valid for Linux servers") |
| 722 |
self._skipIf(NO_IPV6, "--no-ipv6 flag enabled") |
| 723 |
|
| 724 |
server = self.client.get_server_details(self.serverid) |
| 725 |
self._insist_on_ssh_hostname(self._get_ipv6(server), |
| 726 |
self.username, self.passwd) |
| 727 |
|
| 728 |
def test_014_rdp_to_server_IPv4(self): |
| 729 |
"Test RDP connection to server public IPv4 works""" |
| 730 |
self._skipIf(not self.is_windows, "only valid for Windows servers") |
| 731 |
server = self.client.get_server_details(self.serverid) |
| 732 |
ipv4 = self._get_ipv4(server)
|
| 733 |
sock = self._insist_on_tcp_connection(socket.AF_INET, ipv4, 3389) |
| 734 |
|
| 735 |
# No actual RDP processing done. We assume the RDP server is there
|
| 736 |
# if the connection to the RDP port is successful.
|
| 737 |
# FIXME: Use rdesktop, analyze exit code? see manpage [costasd]
|
| 738 |
sock.close() |
| 739 |
|
| 740 |
def test_015_rdp_to_server_IPv6(self): |
| 741 |
"Test RDP connection to server public IPv6 works""" |
| 742 |
self._skipIf(not self.is_windows, "only valid for Windows servers") |
| 743 |
self._skipIf(NO_IPV6, "--no-ipv6 flag enabled") |
| 744 |
|
| 745 |
server = self.client.get_server_details(self.serverid) |
| 746 |
ipv6 = self._get_ipv6(server)
|
| 747 |
sock = self._get_tcp_connection(socket.AF_INET6, ipv6, 3389) |
| 748 |
|
| 749 |
# No actual RDP processing done. We assume the RDP server is there
|
| 750 |
# if the connection to the RDP port is successful.
|
| 751 |
sock.close() |
| 752 |
|
| 753 |
def test_016_personality_is_enforced(self): |
| 754 |
"""Test file injection for personality enforcement"""
|
| 755 |
self._skipIf(self.is_windows, "only implemented for Linux servers") |
| 756 |
self._skipIf(self.personality is None, "No personality file selected") |
| 757 |
|
| 758 |
log.info("Trying to inject file for personality enforcement")
|
| 759 |
|
| 760 |
server = self.client.get_server_details(self.serverid) |
| 761 |
|
| 762 |
for inj_file in self.personality: |
| 763 |
equal_files = self._check_file_through_ssh(self._get_ipv4(server), |
| 764 |
inj_file['owner'],
|
| 765 |
self.passwd,
|
| 766 |
inj_file['path'],
|
| 767 |
inj_file['contents'])
|
| 768 |
self.assertTrue(equal_files)
|
| 769 |
|
| 770 |
def test_017_submit_delete_request(self): |
| 771 |
"""Test submit request to delete server"""
|
| 772 |
|
| 773 |
log.info("Deleting server")
|
| 774 |
|
| 775 |
self.client.delete_server(self.serverid) |
| 776 |
|
| 777 |
def test_018_server_becomes_deleted(self): |
| 778 |
"""Test server becomes DELETED"""
|
| 779 |
|
| 780 |
log.info("Testing if server becomes DELETED")
|
| 781 |
|
| 782 |
self._insist_on_status_transition(
|
| 783 |
"ACTIVE", "DELETED", self.action_timeout, self.action_timeout) |
| 784 |
|
| 785 |
def test_019_server_no_longer_in_server_list(self): |
| 786 |
"""Test server is no longer in server list"""
|
| 787 |
|
| 788 |
log.info("Test if server is no longer listed")
|
| 789 |
|
| 790 |
servers = self.client.list_servers()
|
| 791 |
self.assertNotIn(self.serverid, [s["id"] for s in servers]) |
| 792 |
|
| 793 |
|
| 794 |
class NetworkTestCase(unittest.TestCase): |
| 795 |
""" Testing networking in cyclades """
|
| 796 |
|
| 797 |
@classmethod
|
| 798 |
def setUpClass(cls): |
| 799 |
"Initialize kamaki, get list of current networks"
|
| 800 |
|
| 801 |
cls.client = CycladesClient(API, TOKEN) |
| 802 |
cls.compute = ComputeClient(API, TOKEN) |
| 803 |
|
| 804 |
cls.servername = "%s%s for %s" % (SNF_TEST_PREFIX,
|
| 805 |
TEST_RUN_ID, |
| 806 |
cls.imagename) |
| 807 |
|
| 808 |
#Dictionary initialization for the vms credentials
|
| 809 |
cls.serverid = dict()
|
| 810 |
cls.username = dict()
|
| 811 |
cls.password = dict()
|
| 812 |
cls.is_windows = cls.imagename.lower().find("windows") >= 0 |
| 813 |
|
| 814 |
cls.result_dict = dict()
|
| 815 |
|
| 816 |
def _skipIf(self, condition, msg): |
| 817 |
if condition:
|
| 818 |
self.skipTest(msg)
|
| 819 |
|
| 820 |
def _get_ipv4(self, server): |
| 821 |
"""Get the public IPv4 of a server from the detailed server info"""
|
| 822 |
|
| 823 |
nics = server["attachments"]["values"] |
| 824 |
|
| 825 |
for nic in nics: |
| 826 |
net_id = nic["network_id"]
|
| 827 |
if self.client.get_network_details(net_id)["public"]: |
| 828 |
public_addrs = nic["ipv4"]
|
| 829 |
|
| 830 |
self.assertTrue(public_addrs is not None) |
| 831 |
|
| 832 |
return public_addrs
|
| 833 |
|
| 834 |
def _connect_loginname(self, os_value): |
| 835 |
"""Return the login name for connections based on the server OS"""
|
| 836 |
if os_value in ("Ubuntu", "Kubuntu", "Fedora"): |
| 837 |
return "user" |
| 838 |
elif os_value in ("windows", "windows_alpha1"): |
| 839 |
return "Administrator" |
| 840 |
else:
|
| 841 |
return "root" |
| 842 |
|
| 843 |
def _ping_once(self, ip): |
| 844 |
|
| 845 |
"""Test server responds to a single IPv4 or IPv6 ping"""
|
| 846 |
cmd = "ping -c 2 -w 3 %s" % (ip)
|
| 847 |
ping = subprocess.Popen(cmd, shell=True,
|
| 848 |
stdout=subprocess.PIPE, stderr=subprocess.PIPE) |
| 849 |
(stdout, stderr) = ping.communicate() |
| 850 |
ret = ping.wait() |
| 851 |
|
| 852 |
return (ret == 0) |
| 853 |
|
| 854 |
def test_00001a_submit_create_server_A(self): |
| 855 |
"""Test submit create server request"""
|
| 856 |
|
| 857 |
log.info("Creating test server A")
|
| 858 |
|
| 859 |
serverA = self.client.create_server(self.servername, self.flavorid, |
| 860 |
self.imageid, personality=None) |
| 861 |
|
| 862 |
self.assertEqual(serverA["name"], self.servername) |
| 863 |
self.assertEqual(serverA["flavorRef"], self.flavorid) |
| 864 |
self.assertEqual(serverA["imageRef"], self.imageid) |
| 865 |
self.assertEqual(serverA["status"], "BUILD") |
| 866 |
|
| 867 |
# Update class attributes to reflect data on building server
|
| 868 |
self.serverid['A'] = serverA["id"] |
| 869 |
self.username['A'] = None |
| 870 |
self.password['A'] = serverA["adminPass"] |
| 871 |
|
| 872 |
log.info("Server A id:" + str(serverA["id"])) |
| 873 |
log.info("Server password " + (self.password['A'])) |
| 874 |
|
| 875 |
self.result_dict["Server A ID"] = str(serverA["id"]) |
| 876 |
self.result_dict["Server A password"] = serverA["adminPass"] |
| 877 |
|
| 878 |
def test_00001b_serverA_becomes_active(self): |
| 879 |
"""Test server becomes ACTIVE"""
|
| 880 |
|
| 881 |
log.info("Waiting until test server A becomes ACTIVE")
|
| 882 |
self.result_dict.clear()
|
| 883 |
|
| 884 |
fail_tmout = time.time() + self.action_timeout
|
| 885 |
while True: |
| 886 |
d = self.client.get_server_details(self.serverid['A']) |
| 887 |
status = d['status']
|
| 888 |
if status == 'ACTIVE': |
| 889 |
active = True
|
| 890 |
break
|
| 891 |
elif time.time() > fail_tmout:
|
| 892 |
self.assertLess(time.time(), fail_tmout)
|
| 893 |
else:
|
| 894 |
time.sleep(self.query_interval)
|
| 895 |
|
| 896 |
self.assertTrue(active)
|
| 897 |
|
| 898 |
def test_00002a_submit_create_server_B(self): |
| 899 |
"""Test submit create server request"""
|
| 900 |
|
| 901 |
log.info("Creating test server B")
|
| 902 |
|
| 903 |
serverB = self.client.create_server(self.servername, self.flavorid, |
| 904 |
self.imageid, personality=None) |
| 905 |
|
| 906 |
self.assertEqual(serverB["name"], self.servername) |
| 907 |
self.assertEqual(serverB["flavorRef"], self.flavorid) |
| 908 |
self.assertEqual(serverB["imageRef"], self.imageid) |
| 909 |
self.assertEqual(serverB["status"], "BUILD") |
| 910 |
|
| 911 |
# Update class attributes to reflect data on building server
|
| 912 |
self.serverid['B'] = serverB["id"] |
| 913 |
self.username['B'] = None |
| 914 |
self.password['B'] = serverB["adminPass"] |
| 915 |
|
| 916 |
log.info("Server B id: " + str(serverB["id"])) |
| 917 |
log.info("Password " + (self.password['B'])) |
| 918 |
|
| 919 |
self.result_dict.clear()
|
| 920 |
self.result_dict["Server B ID"] = str(serverB["id"]) |
| 921 |
self.result_dict["Server B password"] = serverB["adminPass"] |
| 922 |
|
| 923 |
def test_00002b_serverB_becomes_active(self): |
| 924 |
"""Test server becomes ACTIVE"""
|
| 925 |
|
| 926 |
log.info("Waiting until test server B becomes ACTIVE")
|
| 927 |
self.result_dict.clear()
|
| 928 |
|
| 929 |
fail_tmout = time.time() + self.action_timeout
|
| 930 |
while True: |
| 931 |
d = self.client.get_server_details(self.serverid['B']) |
| 932 |
status = d['status']
|
| 933 |
if status == 'ACTIVE': |
| 934 |
active = True
|
| 935 |
break
|
| 936 |
elif time.time() > fail_tmout:
|
| 937 |
self.assertLess(time.time(), fail_tmout)
|
| 938 |
else:
|
| 939 |
time.sleep(self.query_interval)
|
| 940 |
|
| 941 |
self.assertTrue(active)
|
| 942 |
|
| 943 |
def test_001_create_network(self): |
| 944 |
"""Test submit create network request"""
|
| 945 |
|
| 946 |
log.info("Submit new network request")
|
| 947 |
self.result_dict.clear()
|
| 948 |
|
| 949 |
name = SNF_TEST_PREFIX + TEST_RUN_ID |
| 950 |
#previous_num = len(self.client.list_networks())
|
| 951 |
network = self.client.create_network(name, cidr='10.0.0.1/28') |
| 952 |
|
| 953 |
#Test if right name is assigned
|
| 954 |
self.assertEqual(network['name'], name) |
| 955 |
|
| 956 |
# Update class attributes
|
| 957 |
cls = type(self) |
| 958 |
cls.networkid = network['id']
|
| 959 |
#networks = self.client.list_networks()
|
| 960 |
|
| 961 |
fail_tmout = time.time() + self.action_timeout
|
| 962 |
|
| 963 |
#Test if new network is created
|
| 964 |
while True: |
| 965 |
d = self.client.get_network_details(network['id']) |
| 966 |
if d['status'] == 'ACTIVE': |
| 967 |
connected = True
|
| 968 |
break
|
| 969 |
elif time.time() > fail_tmout:
|
| 970 |
self.assertLess(time.time(), fail_tmout)
|
| 971 |
else:
|
| 972 |
log.info("Waiting for network to become ACTIVE")
|
| 973 |
time.sleep(self.query_interval)
|
| 974 |
|
| 975 |
self.assertTrue(connected)
|
| 976 |
|
| 977 |
self.result_dict["Private network ID"] = str(network['id']) |
| 978 |
|
| 979 |
def test_002_connect_to_network(self): |
| 980 |
"""Test connect VMs to network"""
|
| 981 |
|
| 982 |
log.info("Connect VMs to private network")
|
| 983 |
self.result_dict.clear()
|
| 984 |
|
| 985 |
self.client.connect_server(self.serverid['A'], self.networkid) |
| 986 |
self.client.connect_server(self.serverid['B'], self.networkid) |
| 987 |
|
| 988 |
#Insist on connecting until action timeout
|
| 989 |
fail_tmout = time.time() + self.action_timeout
|
| 990 |
|
| 991 |
while True: |
| 992 |
|
| 993 |
netsA = [x['network_id']
|
| 994 |
for x in self.client.get_server_details( |
| 995 |
self.serverid['A'])['attachments']['values']] |
| 996 |
netsB = [x['network_id']
|
| 997 |
for x in self.client.get_server_details( |
| 998 |
self.serverid['B'])['attachments']['values']] |
| 999 |
|
| 1000 |
if (self.networkid in netsA) and (self.networkid in netsB): |
| 1001 |
conn_exists = True
|
| 1002 |
break
|
| 1003 |
elif time.time() > fail_tmout:
|
| 1004 |
self.assertLess(time.time(), fail_tmout)
|
| 1005 |
else:
|
| 1006 |
time.sleep(self.query_interval)
|
| 1007 |
|
| 1008 |
#Adding private IPs to class attributes
|
| 1009 |
cls = type(self) |
| 1010 |
cls.priv_ip = dict()
|
| 1011 |
|
| 1012 |
nicsA = self.client.get_server_details(
|
| 1013 |
self.serverid['A'])['attachments']['values'] |
| 1014 |
nicsB = self.client.get_server_details(
|
| 1015 |
self.serverid['B'])['attachments']['values'] |
| 1016 |
|
| 1017 |
if conn_exists:
|
| 1018 |
for nic in nicsA: |
| 1019 |
if nic["network_id"] == self.networkid: |
| 1020 |
cls.priv_ip["A"] = nic["ipv4"] |
| 1021 |
|
| 1022 |
for nic in nicsB: |
| 1023 |
if nic["network_id"] == self.networkid: |
| 1024 |
cls.priv_ip["B"] = nic["ipv4"] |
| 1025 |
|
| 1026 |
self.assertTrue(conn_exists)
|
| 1027 |
|
| 1028 |
def test_002a_reboot(self): |
| 1029 |
"""Rebooting server A"""
|
| 1030 |
|
| 1031 |
log.info("Rebooting server A")
|
| 1032 |
|
| 1033 |
self.client.shutdown_server(self.serverid['A']) |
| 1034 |
|
| 1035 |
fail_tmout = time.time() + self.action_timeout
|
| 1036 |
while True: |
| 1037 |
d = self.client.get_server_details(self.serverid['A']) |
| 1038 |
status = d['status']
|
| 1039 |
if status == 'STOPPED': |
| 1040 |
break
|
| 1041 |
elif time.time() > fail_tmout:
|
| 1042 |
self.assertLess(time.time(), fail_tmout)
|
| 1043 |
else:
|
| 1044 |
time.sleep(self.query_interval)
|
| 1045 |
|
| 1046 |
self.client.start_server(self.serverid['A']) |
| 1047 |
|
| 1048 |
while True: |
| 1049 |
d = self.client.get_server_details(self.serverid['A']) |
| 1050 |
status = d['status']
|
| 1051 |
if status == 'ACTIVE': |
| 1052 |
active = True
|
| 1053 |
break
|
| 1054 |
elif time.time() > fail_tmout:
|
| 1055 |
self.assertLess(time.time(), fail_tmout)
|
| 1056 |
else:
|
| 1057 |
time.sleep(self.query_interval)
|
| 1058 |
|
| 1059 |
self.assertTrue(active)
|
| 1060 |
|
| 1061 |
def test_002b_ping_server_A(self): |
| 1062 |
"Test if server A responds to IPv4 pings"
|
| 1063 |
|
| 1064 |
log.info("Testing if server A responds to IPv4 pings ")
|
| 1065 |
self.result_dict.clear()
|
| 1066 |
|
| 1067 |
server = self.client.get_server_details(self.serverid['A']) |
| 1068 |
ip = self._get_ipv4(server)
|
| 1069 |
|
| 1070 |
fail_tmout = time.time() + self.action_timeout
|
| 1071 |
|
| 1072 |
s = False
|
| 1073 |
|
| 1074 |
self.result_dict["Server A public IP"] = str(ip) |
| 1075 |
|
| 1076 |
while True: |
| 1077 |
|
| 1078 |
if self._ping_once(ip): |
| 1079 |
s = True
|
| 1080 |
break
|
| 1081 |
|
| 1082 |
elif time.time() > fail_tmout:
|
| 1083 |
self.assertLess(time.time(), fail_tmout)
|
| 1084 |
|
| 1085 |
else:
|
| 1086 |
time.sleep(self.query_interval)
|
| 1087 |
|
| 1088 |
self.assertTrue(s)
|
| 1089 |
|
| 1090 |
def test_002c_reboot(self): |
| 1091 |
"""Reboot server B"""
|
| 1092 |
|
| 1093 |
log.info("Rebooting server B")
|
| 1094 |
self.result_dict.clear()
|
| 1095 |
|
| 1096 |
self.client.shutdown_server(self.serverid['B']) |
| 1097 |
|
| 1098 |
fail_tmout = time.time() + self.action_timeout
|
| 1099 |
while True: |
| 1100 |
d = self.client.get_server_details(self.serverid['B']) |
| 1101 |
status = d['status']
|
| 1102 |
if status == 'STOPPED': |
| 1103 |
break
|
| 1104 |
elif time.time() > fail_tmout:
|
| 1105 |
self.assertLess(time.time(), fail_tmout)
|
| 1106 |
else:
|
| 1107 |
time.sleep(self.query_interval)
|
| 1108 |
|
| 1109 |
self.client.start_server(self.serverid['B']) |
| 1110 |
|
| 1111 |
while True: |
| 1112 |
d = self.client.get_server_details(self.serverid['B']) |
| 1113 |
status = d['status']
|
| 1114 |
if status == 'ACTIVE': |
| 1115 |
active = True
|
| 1116 |
break
|
| 1117 |
elif time.time() > fail_tmout:
|
| 1118 |
self.assertLess(time.time(), fail_tmout)
|
| 1119 |
else:
|
| 1120 |
time.sleep(self.query_interval)
|
| 1121 |
|
| 1122 |
self.assertTrue(active)
|
| 1123 |
|
| 1124 |
def test_002d_ping_server_B(self): |
| 1125 |
"""Test if server B responds to IPv4 pings"""
|
| 1126 |
|
| 1127 |
log.info("Testing if server B responds to IPv4 pings")
|
| 1128 |
self.result_dict.clear()
|
| 1129 |
|
| 1130 |
server = self.client.get_server_details(self.serverid['B']) |
| 1131 |
ip = self._get_ipv4(server)
|
| 1132 |
|
| 1133 |
fail_tmout = time.time() + self.action_timeout
|
| 1134 |
|
| 1135 |
s = False
|
| 1136 |
|
| 1137 |
self.result_dict["Server B public IP"] = str(ip) |
| 1138 |
|
| 1139 |
while True: |
| 1140 |
if self._ping_once(ip): |
| 1141 |
s = True
|
| 1142 |
break
|
| 1143 |
|
| 1144 |
elif time.time() > fail_tmout:
|
| 1145 |
self.assertLess(time.time(), fail_tmout)
|
| 1146 |
|
| 1147 |
else:
|
| 1148 |
time.sleep(self.query_interval)
|
| 1149 |
|
| 1150 |
self.assertTrue(s)
|
| 1151 |
|
| 1152 |
def test_003a_setup_interface_A(self): |
| 1153 |
"""Set up eth1 for server A"""
|
| 1154 |
|
| 1155 |
self._skipIf(self.is_windows, "only valid for Linux servers") |
| 1156 |
|
| 1157 |
log.info("Setting up interface eth1 for server A")
|
| 1158 |
self.result_dict.clear()
|
| 1159 |
|
| 1160 |
server = self.client.get_server_details(self.serverid['A']) |
| 1161 |
image = self.client.get_image_details(self.imageid) |
| 1162 |
os_value = image['metadata']['values']['os'] |
| 1163 |
|
| 1164 |
users = image["metadata"]["values"].get("users", None) |
| 1165 |
userlist = users.split() |
| 1166 |
|
| 1167 |
if "root" in userlist: |
| 1168 |
loginname = "root"
|
| 1169 |
elif users is None: |
| 1170 |
loginname = self._connect_loginname(os_value)
|
| 1171 |
else:
|
| 1172 |
loginname = choice(userlist) |
| 1173 |
|
| 1174 |
hostip = self._get_ipv4(server)
|
| 1175 |
myPass = self.password['A'] |
| 1176 |
|
| 1177 |
log.info("SSH in server A as %s/%s" % (loginname, myPass))
|
| 1178 |
|
| 1179 |
res = False
|
| 1180 |
|
| 1181 |
if loginname != "root": |
| 1182 |
with fabric.settings(
|
| 1183 |
fabric.hide('warnings', 'running'), |
| 1184 |
warn_only=True,
|
| 1185 |
host_string=hostip, |
| 1186 |
user=loginname, password=myPass): |
| 1187 |
conf_res = fabric.sudo('ifconfig eth1 %s' % self.priv_ip["A"]) |
| 1188 |
if len(conf_res) == 0: |
| 1189 |
res = True
|
| 1190 |
|
| 1191 |
else:
|
| 1192 |
with fabric.settings(
|
| 1193 |
fabric.hide('warnings', 'running'), |
| 1194 |
warn_only=True,
|
| 1195 |
host_string=hostip, |
| 1196 |
user=loginname, password=myPass): |
| 1197 |
conf_res = fabric.run('ifconfig eth1 %s' % self.priv_ip["A"]) |
| 1198 |
if len(conf_res) == 0: |
| 1199 |
res = True
|
| 1200 |
|
| 1201 |
self.assertTrue(res)
|
| 1202 |
|
| 1203 |
def test_003b_setup_interface_B(self): |
| 1204 |
"""Setup eth1 for server B"""
|
| 1205 |
|
| 1206 |
self._skipIf(self.is_windows, "only valid for Linux servers") |
| 1207 |
|
| 1208 |
log.info("Setting up interface eth1 for server B")
|
| 1209 |
|
| 1210 |
server = self.client.get_server_details(self.serverid['B']) |
| 1211 |
image = self.client.get_image_details(self.imageid) |
| 1212 |
os_value = image['metadata']['values']['os'] |
| 1213 |
|
| 1214 |
users = image["metadata"]["values"].get("users", None) |
| 1215 |
userlist = users.split() |
| 1216 |
|
| 1217 |
if "root" in userlist: |
| 1218 |
loginname = "root"
|
| 1219 |
elif users is None: |
| 1220 |
loginname = self._connect_loginname(os_value)
|
| 1221 |
else:
|
| 1222 |
loginname = choice(userlist) |
| 1223 |
|
| 1224 |
hostip = self._get_ipv4(server)
|
| 1225 |
myPass = self.password['B'] |
| 1226 |
|
| 1227 |
log.info("SSH in server B as %s/%s" % (loginname, myPass))
|
| 1228 |
|
| 1229 |
res = False
|
| 1230 |
|
| 1231 |
if loginname != "root": |
| 1232 |
with fabric.settings(
|
| 1233 |
fabric.hide('warnings', 'running'), |
| 1234 |
warn_only=True,
|
| 1235 |
host_string=hostip, |
| 1236 |
user=loginname, password=myPass): |
| 1237 |
conf_res = fabric.sudo('ifconfig eth1 %s' % self.priv_ip["B"]) |
| 1238 |
if len(conf_res) == 0: |
| 1239 |
res = True
|
| 1240 |
|
| 1241 |
else:
|
| 1242 |
with fabric.settings(
|
| 1243 |
fabric.hide('warnings', 'running'), |
| 1244 |
warn_only=True,
|
| 1245 |
host_string=hostip, |
| 1246 |
user=loginname, password=myPass): |
| 1247 |
conf_res = fabric.run('ifconfig eth1 %s' % self.priv_ip["B"]) |
| 1248 |
if len(conf_res) == 0: |
| 1249 |
res = True
|
| 1250 |
|
| 1251 |
self.assertTrue(res)
|
| 1252 |
|
| 1253 |
def test_003c_test_connection_exists(self): |
| 1254 |
"""Ping server B from server A to test if connection exists"""
|
| 1255 |
|
| 1256 |
self._skipIf(self.is_windows, "only valid for Linux servers") |
| 1257 |
|
| 1258 |
log.info("Testing if server A is actually connected to server B")
|
| 1259 |
|
| 1260 |
server = self.client.get_server_details(self.serverid['A']) |
| 1261 |
image = self.client.get_image_details(self.imageid) |
| 1262 |
os_value = image['metadata']['values']['os'] |
| 1263 |
hostip = self._get_ipv4(server)
|
| 1264 |
|
| 1265 |
users = image["metadata"]["values"].get("users", None) |
| 1266 |
userlist = users.split() |
| 1267 |
|
| 1268 |
if "root" in userlist: |
| 1269 |
loginname = "root"
|
| 1270 |
elif users is None: |
| 1271 |
loginname = self._connect_loginname(os_value)
|
| 1272 |
else:
|
| 1273 |
loginname = choice(userlist) |
| 1274 |
|
| 1275 |
myPass = self.password['A'] |
| 1276 |
|
| 1277 |
try:
|
| 1278 |
ssh = paramiko.SSHClient() |
| 1279 |
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) |
| 1280 |
ssh.connect(hostip, username=loginname, password=myPass) |
| 1281 |
except socket.error:
|
| 1282 |
raise AssertionError |
| 1283 |
|
| 1284 |
cmd = "if ping -c 2 -w 3 %s >/dev/null; \
|
| 1285 |
then echo \'True\'; fi;" % self.priv_ip["B"] |
| 1286 |
stdin, stdout, stderr = ssh.exec_command(cmd) |
| 1287 |
lines = stdout.readlines() |
| 1288 |
|
| 1289 |
exists = False
|
| 1290 |
|
| 1291 |
if 'True\n' in lines: |
| 1292 |
exists = True
|
| 1293 |
|
| 1294 |
self.assertTrue(exists)
|
| 1295 |
|
| 1296 |
def test_004_disconnect_from_network(self): |
| 1297 |
"Disconnecting server A and B from network"
|
| 1298 |
|
| 1299 |
log.info("Disconnecting servers from private network")
|
| 1300 |
|
| 1301 |
prev_state = self.client.get_network_details(self.networkid) |
| 1302 |
prev_nics = prev_state['attachments']['values'] |
| 1303 |
#prev_conn = len(prev_nics)
|
| 1304 |
|
| 1305 |
nicsA = [x['id']
|
| 1306 |
for x in self.client.get_server_details( |
| 1307 |
self.serverid['A'])['attachments']['values']] |
| 1308 |
nicsB = [x['id']
|
| 1309 |
for x in self.client.get_server_details( |
| 1310 |
self.serverid['B'])['attachments']['values']] |
| 1311 |
|
| 1312 |
for nic in prev_nics: |
| 1313 |
if nic in nicsA: |
| 1314 |
self.client.disconnect_server(self.serverid['A'], nic) |
| 1315 |
if nic in nicsB: |
| 1316 |
self.client.disconnect_server(self.serverid['B'], nic) |
| 1317 |
|
| 1318 |
#Insist on deleting until action timeout
|
| 1319 |
fail_tmout = time.time() + self.action_timeout
|
| 1320 |
|
| 1321 |
while True: |
| 1322 |
netsA = [x['network_id']
|
| 1323 |
for x in self.client.get_server_details( |
| 1324 |
self.serverid['A'])['attachments']['values']] |
| 1325 |
netsB = [x['network_id']
|
| 1326 |
for x in self.client.get_server_details( |
| 1327 |
self.serverid['B'])['attachments']['values']] |
| 1328 |
|
| 1329 |
#connected = (self.client.get_network_details(self.networkid))
|
| 1330 |
#connections = connected['attachments']['values']
|
| 1331 |
if (self.networkid not in netsA) and (self.networkid not in netsB): |
| 1332 |
conn_exists = False
|
| 1333 |
break
|
| 1334 |
elif time.time() > fail_tmout:
|
| 1335 |
self.assertLess(time.time(), fail_tmout)
|
| 1336 |
else:
|
| 1337 |
time.sleep(self.query_interval)
|
| 1338 |
|
| 1339 |
self.assertFalse(conn_exists)
|
| 1340 |
|
| 1341 |
def test_005_destroy_network(self): |
| 1342 |
"""Test submit delete network request"""
|
| 1343 |
|
| 1344 |
log.info("Submitting delete network request")
|
| 1345 |
|
| 1346 |
self.client.delete_network(self.networkid) |
| 1347 |
|
| 1348 |
fail_tmout = time.time() + self.action_timeout
|
| 1349 |
|
| 1350 |
while True: |
| 1351 |
|
| 1352 |
curr_net = [] |
| 1353 |
networks = self.client.list_networks()
|
| 1354 |
|
| 1355 |
for net in networks: |
| 1356 |
curr_net.append(net['id'])
|
| 1357 |
|
| 1358 |
if self.networkid not in curr_net: |
| 1359 |
self.assertTrue(self.networkid not in curr_net) |
| 1360 |
break
|
| 1361 |
|
| 1362 |
elif time.time() > fail_tmout:
|
| 1363 |
self.assertLess(time.time(), fail_tmout)
|
| 1364 |
|
| 1365 |
else:
|
| 1366 |
time.sleep(self.query_interval)
|
| 1367 |
|
| 1368 |
def test_006_cleanup_servers(self): |
| 1369 |
"""Cleanup servers created for this test"""
|
| 1370 |
|
| 1371 |
log.info("Delete servers created for this test")
|
| 1372 |
|
| 1373 |
self.compute.delete_server(self.serverid['A']) |
| 1374 |
self.compute.delete_server(self.serverid['B']) |
| 1375 |
|
| 1376 |
fail_tmout = time.time() + self.action_timeout
|
| 1377 |
|
| 1378 |
#Ensure server gets deleted
|
| 1379 |
status = dict()
|
| 1380 |
|
| 1381 |
while True: |
| 1382 |
details = self.compute.get_server_details(self.serverid['A']) |
| 1383 |
status['A'] = details['status'] |
| 1384 |
details = self.compute.get_server_details(self.serverid['B']) |
| 1385 |
status['B'] = details['status'] |
| 1386 |
if (status['A'] == 'DELETED') and (status['B'] == 'DELETED'): |
| 1387 |
deleted = True
|
| 1388 |
break
|
| 1389 |
elif time.time() > fail_tmout:
|
| 1390 |
self.assertLess(time.time(), fail_tmout)
|
| 1391 |
else:
|
| 1392 |
time.sleep(self.query_interval)
|
| 1393 |
|
| 1394 |
self.assertTrue(deleted)
|
| 1395 |
|
| 1396 |
|
| 1397 |
class TestRunnerProcess(Process): |
| 1398 |
"""A distinct process used to execute part of the tests in parallel"""
|
| 1399 |
def __init__(self, **kw): |
| 1400 |
Process.__init__(self, **kw)
|
| 1401 |
kwargs = kw["kwargs"]
|
| 1402 |
self.testq = kwargs["testq"] |
| 1403 |
self.worker_folder = kwargs["worker_folder"] |
| 1404 |
|
| 1405 |
def run(self): |
| 1406 |
# Make sure this test runner process dies with the parent
|
| 1407 |
# and is not left behind.
|
| 1408 |
#
|
| 1409 |
# WARNING: This uses the prctl(2) call and is
|
| 1410 |
# Linux-specific.
|
| 1411 |
|
| 1412 |
prctl.set_pdeathsig(signal.SIGHUP) |
| 1413 |
|
| 1414 |
multi = logging.getLogger("multiprocess")
|
| 1415 |
|
| 1416 |
while True: |
| 1417 |
multi.debug("I am process %d, GETting from queue is %s" %
|
| 1418 |
(os.getpid(), self.testq))
|
| 1419 |
msg = self.testq.get()
|
| 1420 |
|
| 1421 |
multi.debug("Dequeued msg: %s" % msg)
|
| 1422 |
|
| 1423 |
if msg == "TEST_RUNNER_TERMINATE": |
| 1424 |
raise SystemExit |
| 1425 |
|
| 1426 |
elif issubclass(msg, unittest.TestCase): |
| 1427 |
# Assemble a TestSuite, and run it
|
| 1428 |
|
| 1429 |
log_file = os.path.join(self.worker_folder, 'details_' + |
| 1430 |
(msg.__name__) + "_" +
|
| 1431 |
TEST_RUN_ID + '.log')
|
| 1432 |
|
| 1433 |
fail_file = os.path.join(self.worker_folder, 'failed_' + |
| 1434 |
(msg.__name__) + "_" +
|
| 1435 |
TEST_RUN_ID + '.log')
|
| 1436 |
error_file = os.path.join(self.worker_folder, 'error_' + |
| 1437 |
(msg.__name__) + "_" +
|
| 1438 |
TEST_RUN_ID + '.log')
|
| 1439 |
|
| 1440 |
f = open(log_file, 'w') |
| 1441 |
fail = open(fail_file, 'w') |
| 1442 |
error = open(error_file, 'w') |
| 1443 |
|
| 1444 |
log.info(yellow + '* Starting testcase: %s' % msg + normal)
|
| 1445 |
|
| 1446 |
runner = unittest.TextTestRunner( |
| 1447 |
f, verbosity=2, failfast=True, |
| 1448 |
resultclass=BurninTestResult) |
| 1449 |
suite = unittest.TestLoader().loadTestsFromTestCase(msg) |
| 1450 |
result = runner.run(suite) |
| 1451 |
|
| 1452 |
for res in result.errors: |
| 1453 |
log.error("snf-burnin encountered an error in "
|
| 1454 |
"testcase: %s" % msg)
|
| 1455 |
log.error("See log for details")
|
| 1456 |
error.write(str(res[0]) + '\n') |
| 1457 |
error.write(str(res[0].shortDescription()) + '\n') |
| 1458 |
error.write('\n')
|
| 1459 |
|
| 1460 |
for res in result.failures: |
| 1461 |
log.error("snf-burnin failed in testcase: %s" % msg)
|
| 1462 |
log.error("See log for details")
|
| 1463 |
fail.write(str(res[0]) + '\n') |
| 1464 |
fail.write(str(res[0].shortDescription()) + '\n') |
| 1465 |
fail.write('\n')
|
| 1466 |
if not NOFAILFAST: |
| 1467 |
sys.exit() |
| 1468 |
|
| 1469 |
if (len(result.failures) == 0) and (len(result.errors) == 0): |
| 1470 |
log.debug("Passed testcase: %s" % msg)
|
| 1471 |
|
| 1472 |
f.close() |
| 1473 |
fail.close() |
| 1474 |
error.close() |
| 1475 |
|
| 1476 |
else:
|
| 1477 |
raise Exception("Cannot handle msg: %s" % msg) |
| 1478 |
|
| 1479 |
|
| 1480 |
def _run_cases_in_series(cases, image_folder): |
| 1481 |
"""Run instances of TestCase in series"""
|
| 1482 |
|
| 1483 |
for case in cases: |
| 1484 |
|
| 1485 |
test = case.__name__ |
| 1486 |
|
| 1487 |
log.info(yellow + '* Starting testcase: %s' % test + normal)
|
| 1488 |
log_file = os.path.join(image_folder, 'details_' +
|
| 1489 |
(case.__name__) + "_" +
|
| 1490 |
TEST_RUN_ID + '.log')
|
| 1491 |
fail_file = os.path.join(image_folder, 'failed_' +
|
| 1492 |
(case.__name__) + "_" +
|
| 1493 |
TEST_RUN_ID + '.log')
|
| 1494 |
error_file = os.path.join(image_folder, 'error_' +
|
| 1495 |
(case.__name__) + "_" +
|
| 1496 |
TEST_RUN_ID + '.log')
|
| 1497 |
|
| 1498 |
f = open(log_file, "w") |
| 1499 |
fail = open(fail_file, "w") |
| 1500 |
error = open(error_file, "w") |
| 1501 |
|
| 1502 |
suite = unittest.TestLoader().loadTestsFromTestCase(case) |
| 1503 |
runner = unittest.TextTestRunner( |
| 1504 |
f, verbosity=2, failfast=True, |
| 1505 |
resultclass=BurninTestResult) |
| 1506 |
result = runner.run(suite) |
| 1507 |
|
| 1508 |
for res in result.errors: |
| 1509 |
log.error("snf-burnin encountered an error in "
|
| 1510 |
"testcase: %s" % test)
|
| 1511 |
log.error("See log for details")
|
| 1512 |
error.write(str(res[0]) + '\n') |
| 1513 |
error.write(str(res[0].shortDescription()) + '\n') |
| 1514 |
error.write('\n')
|
| 1515 |
|
| 1516 |
for res in result.failures: |
| 1517 |
log.error("snf-burnin failed in testcase: %s" % test)
|
| 1518 |
log.error("See log for details")
|
| 1519 |
fail.write(str(res[0]) + '\n') |
| 1520 |
fail.write(str(res[0].shortDescription()) + '\n') |
| 1521 |
fail.write('\n')
|
| 1522 |
if not NOFAILFAST: |
| 1523 |
sys.exit() |
| 1524 |
|
| 1525 |
if (len(result.failures) == 0) and (len(result.errors) == 0): |
| 1526 |
log.debug("Passed testcase: %s" % test)
|
| 1527 |
|
| 1528 |
|
| 1529 |
def _run_cases_in_parallel(cases, fanout, image_folder): |
| 1530 |
"""Run instances of TestCase in parallel, in a number of distinct processes
|
| 1531 |
|
| 1532 |
The cases iterable specifies the TestCases to be executed in parallel,
|
| 1533 |
by test runners running in distinct processes.
|
| 1534 |
The fanout parameter specifies the number of processes to spawn,
|
| 1535 |
and defaults to 1.
|
| 1536 |
The runner argument specifies the test runner class to use inside each
|
| 1537 |
runner process.
|
| 1538 |
|
| 1539 |
"""
|
| 1540 |
|
| 1541 |
multi = logging.getLogger("multiprocess")
|
| 1542 |
handler = logging.StreamHandler() |
| 1543 |
multi.addHandler(handler) |
| 1544 |
|
| 1545 |
if VERBOSE:
|
| 1546 |
multi.setLevel(logging.DEBUG) |
| 1547 |
else:
|
| 1548 |
multi.setLevel(logging.INFO) |
| 1549 |
|
| 1550 |
testq = [] |
| 1551 |
worker_folder = [] |
| 1552 |
runners = [] |
| 1553 |
|
| 1554 |
for i in xrange(0, fanout): |
| 1555 |
testq.append(Queue()) |
| 1556 |
worker_folder.append(os.path.join(image_folder, 'process'+str(i))) |
| 1557 |
os.mkdir(worker_folder[i]) |
| 1558 |
|
| 1559 |
for i in xrange(0, fanout): |
| 1560 |
kwargs = dict(testq=testq[i], worker_folder=worker_folder[i])
|
| 1561 |
runners.append(TestRunnerProcess(kwargs=kwargs)) |
| 1562 |
|
| 1563 |
multi.debug("Spawning %d test runner processes" % len(runners)) |
| 1564 |
|
| 1565 |
for p in runners: |
| 1566 |
p.start() |
| 1567 |
|
| 1568 |
# Enqueue test cases
|
| 1569 |
for i in xrange(0, fanout): |
| 1570 |
map(testq[i].put, cases)
|
| 1571 |
testq[i].put("TEST_RUNNER_TERMINATE")
|
| 1572 |
|
| 1573 |
multi.debug("Spawned %d test runners, PIDs are %s" %
|
| 1574 |
(len(runners), [p.pid for p in runners])) |
| 1575 |
|
| 1576 |
multi.debug("Joining %d processes" % len(runners)) |
| 1577 |
|
| 1578 |
for p in runners: |
| 1579 |
p.join() |
| 1580 |
|
| 1581 |
multi.debug("Done joining %d processes" % len(runners)) |
| 1582 |
|
| 1583 |
|
| 1584 |
def _spawn_server_test_case(**kwargs): |
| 1585 |
"""Construct a new unit test case class from SpawnServerTestCase"""
|
| 1586 |
|
| 1587 |
name = "SpawnServerTestCase_%s" % kwargs["imageid"] |
| 1588 |
cls = type(name, (SpawnServerTestCase,), kwargs)
|
| 1589 |
|
| 1590 |
# Patch extra parameters into test names by manipulating method docstrings
|
| 1591 |
for (mname, m) in \ |
| 1592 |
inspect.getmembers(cls, lambda x: inspect.ismethod(x)):
|
| 1593 |
if hasattr(m, __doc__): |
| 1594 |
m.__func__.__doc__ = "[%s] %s" % (cls.imagename, m.__doc__)
|
| 1595 |
|
| 1596 |
# Make sure the class can be pickled, by listing it among
|
| 1597 |
# the attributes of __main__. A PicklingError is raised otherwise.
|
| 1598 |
|
| 1599 |
thismodule = sys.modules[__name__] |
| 1600 |
setattr(thismodule, name, cls)
|
| 1601 |
return cls
|
| 1602 |
|
| 1603 |
|
| 1604 |
def _spawn_network_test_case(**kwargs): |
| 1605 |
"""Construct a new unit test case class from NetworkTestCase"""
|
| 1606 |
|
| 1607 |
name = "NetworkTestCase" + TEST_RUN_ID
|
| 1608 |
cls = type(name, (NetworkTestCase,), kwargs)
|
| 1609 |
|
| 1610 |
# Make sure the class can be pickled, by listing it among
|
| 1611 |
# the attributes of __main__. A PicklingError is raised otherwise.
|
| 1612 |
|
| 1613 |
thismodule = sys.modules[__name__] |
| 1614 |
setattr(thismodule, name, cls)
|
| 1615 |
return cls
|
| 1616 |
|
| 1617 |
|
| 1618 |
# --------------------------------------------------------------------
|
| 1619 |
# Clean up servers/networks functions
|
| 1620 |
def cleanup_servers(timeout, query_interval, delete_stale=False): |
| 1621 |
|
| 1622 |
c = ComputeClient(API, TOKEN) |
| 1623 |
|
| 1624 |
servers = c.list_servers() |
| 1625 |
stale = [s for s in servers if s["name"].startswith(SNF_TEST_PREFIX)] |
| 1626 |
|
| 1627 |
if len(stale) == 0: |
| 1628 |
return
|
| 1629 |
|
| 1630 |
# Show staled servers
|
| 1631 |
print >>sys.stderr, yellow + \
|
| 1632 |
"Found these stale servers from previous runs:" + \
|
| 1633 |
normal |
| 1634 |
print >>sys.stderr, " " + \ |
| 1635 |
"\n ".join(["%d: %s" % (s["id"], s["name"]) for s in stale]) |
| 1636 |
|
| 1637 |
# Delete staled servers
|
| 1638 |
if delete_stale:
|
| 1639 |
print >> sys.stderr, "Deleting %d stale servers:" % len(stale) |
| 1640 |
fail_tmout = time.time() + timeout |
| 1641 |
for s in stale: |
| 1642 |
c.delete_server(s["id"])
|
| 1643 |
# Wait for all servers to be deleted
|
| 1644 |
while True: |
| 1645 |
servers = c.list_servers() |
| 1646 |
stale = [s for s in servers |
| 1647 |
if s["name"].startswith(SNF_TEST_PREFIX)] |
| 1648 |
if len(stale) == 0: |
| 1649 |
print >> sys.stderr, green + " ...done" + normal |
| 1650 |
break
|
| 1651 |
elif time.time() > fail_tmout:
|
| 1652 |
print >> sys.stderr, red + \
|
| 1653 |
"Not all stale servers deleted. Action timed out." + \
|
| 1654 |
normal |
| 1655 |
sys.exit(1)
|
| 1656 |
else:
|
| 1657 |
time.sleep(query_interval) |
| 1658 |
else:
|
| 1659 |
print >> sys.stderr, "Use --delete-stale to delete them." |
| 1660 |
|
| 1661 |
|
| 1662 |
def cleanup_networks(action_timeout, query_interval, delete_stale=False): |
| 1663 |
|
| 1664 |
c = CycladesClient(API, TOKEN) |
| 1665 |
|
| 1666 |
networks = c.list_networks() |
| 1667 |
stale = [n for n in networks if n["name"].startswith(SNF_TEST_PREFIX)] |
| 1668 |
|
| 1669 |
if len(stale) == 0: |
| 1670 |
return
|
| 1671 |
|
| 1672 |
# Show staled networks
|
| 1673 |
print >> sys.stderr, yellow + \
|
| 1674 |
"Found these stale networks from previous runs:" + \
|
| 1675 |
normal |
| 1676 |
print " " + \ |
| 1677 |
"\n ".join(["%s: %s" % (str(n["id"]), n["name"]) for n in stale]) |
| 1678 |
|
| 1679 |
# Delete staled networks
|
| 1680 |
if delete_stale:
|
| 1681 |
print >> sys.stderr, "Deleting %d stale networks:" % len(stale) |
| 1682 |
fail_tmout = time.time() + action_timeout |
| 1683 |
for n in stale: |
| 1684 |
c.delete_network(n["id"])
|
| 1685 |
# Wait for all networks to be deleted
|
| 1686 |
while True: |
| 1687 |
networks = c.list_networks() |
| 1688 |
stale = [n for n in networks |
| 1689 |
if n["name"].startswith(SNF_TEST_PREFIX)] |
| 1690 |
if len(stale) == 0: |
| 1691 |
print >> sys.stderr, green + " ...done" + normal |
| 1692 |
break
|
| 1693 |
elif time.time() > fail_tmout:
|
| 1694 |
print >> sys.stderr, red + \
|
| 1695 |
"Not all stale networks deleted. Action timed out." + \
|
| 1696 |
normal |
| 1697 |
sys.exit(1)
|
| 1698 |
else:
|
| 1699 |
time.sleep(query_interval) |
| 1700 |
else:
|
| 1701 |
print >> sys.stderr, "Use --delete-stale to delete them." |
| 1702 |
|
| 1703 |
|
| 1704 |
# --------------------------------------------------------------------
|
| 1705 |
# Parse arguments functions
|
| 1706 |
def parse_comma(option, opt, value, parser): |
| 1707 |
tests = set(['all', 'auth', 'images', 'flavors', |
| 1708 |
'servers', 'server_spawn', 'network_spawn']) |
| 1709 |
parse_input = value.split(',')
|
| 1710 |
|
| 1711 |
if not (set(parse_input)).issubset(tests): |
| 1712 |
raise OptionValueError("The selected set of tests is invalid") |
| 1713 |
|
| 1714 |
setattr(parser.values, option.dest, value.split(',')) |
| 1715 |
|
| 1716 |
|
| 1717 |
def parse_arguments(args): |
| 1718 |
|
| 1719 |
kw = {}
|
| 1720 |
kw["usage"] = "%prog [options]" |
| 1721 |
kw["description"] = \
|
| 1722 |
"%prog runs a number of test scenarios on a " \
|
| 1723 |
"Synnefo deployment."
|
| 1724 |
|
| 1725 |
parser = OptionParser(**kw) |
| 1726 |
parser.disable_interspersed_args() |
| 1727 |
|
| 1728 |
parser.add_option("--api",
|
| 1729 |
action="store", type="string", dest="api", |
| 1730 |
help="The API URI to use to reach the Synnefo API",
|
| 1731 |
default=DEFAULT_API) |
| 1732 |
parser.add_option("--plankton",
|
| 1733 |
action="store", type="string", dest="plankton", |
| 1734 |
help="The API URI to use to reach the Plankton API",
|
| 1735 |
default=DEFAULT_PLANKTON) |
| 1736 |
parser.add_option("--plankton-user",
|
| 1737 |
action="store", type="string", dest="plankton_user", |
| 1738 |
help="Owner of system images",
|
| 1739 |
default=DEFAULT_PLANKTON_USER) |
| 1740 |
parser.add_option("--token",
|
| 1741 |
action="store", type="string", dest="token", |
| 1742 |
help="The token to use for authentication to the API")
|
| 1743 |
parser.add_option("--nofailfast",
|
| 1744 |
action="store_true", dest="nofailfast", |
| 1745 |
help="Do not fail immediately if one of the tests "
|
| 1746 |
"fails (EXPERIMENTAL)",
|
| 1747 |
default=False)
|
| 1748 |
parser.add_option("--no-ipv6",
|
| 1749 |
action="store_true", dest="no_ipv6", |
| 1750 |
help="Disables ipv6 related tests",
|
| 1751 |
default=False)
|
| 1752 |
parser.add_option("--action-timeout",
|
| 1753 |
action="store", type="int", dest="action_timeout", |
| 1754 |
metavar="TIMEOUT",
|
| 1755 |
help="Wait SECONDS seconds for a server action to "
|
| 1756 |
"complete, then the test is considered failed",
|
| 1757 |
default=100)
|
| 1758 |
parser.add_option("--build-warning",
|
| 1759 |
action="store", type="int", dest="build_warning", |
| 1760 |
metavar="TIMEOUT",
|
| 1761 |
help="Warn if TIMEOUT seconds have passed and a "
|
| 1762 |
"build operation is still pending",
|
| 1763 |
default=600)
|
| 1764 |
parser.add_option("--build-fail",
|
| 1765 |
action="store", type="int", dest="build_fail", |
| 1766 |
metavar="BUILD_TIMEOUT",
|
| 1767 |
help="Fail the test if TIMEOUT seconds have passed "
|
| 1768 |
"and a build operation is still incomplete",
|
| 1769 |
default=900)
|
| 1770 |
parser.add_option("--query-interval",
|
| 1771 |
action="store", type="int", dest="query_interval", |
| 1772 |
metavar="INTERVAL",
|
| 1773 |
help="Query server status when requests are pending "
|
| 1774 |
"every INTERVAL seconds",
|
| 1775 |
default=3)
|
| 1776 |
parser.add_option("--fanout",
|
| 1777 |
action="store", type="int", dest="fanout", |
| 1778 |
metavar="COUNT",
|
| 1779 |
help="Spawn up to COUNT child processes to execute "
|
| 1780 |
"in parallel, essentially have up to COUNT "
|
| 1781 |
"server build requests outstanding (EXPERIMENTAL)",
|
| 1782 |
default=1)
|
| 1783 |
parser.add_option("--force-flavor",
|
| 1784 |
action="store", type="int", dest="force_flavorid", |
| 1785 |
metavar="FLAVOR ID",
|
| 1786 |
help="Force all server creations to use the specified "
|
| 1787 |
"FLAVOR ID instead of a randomly chosen one, "
|
| 1788 |
"useful if disk space is scarce",
|
| 1789 |
default=None)
|
| 1790 |
parser.add_option("--image-id",
|
| 1791 |
action="store", type="string", dest="force_imageid", |
| 1792 |
metavar="IMAGE ID",
|
| 1793 |
help="Test the specified image id, use 'all' to test "
|
| 1794 |
"all available images (mandatory argument)",
|
| 1795 |
default=None)
|
| 1796 |
parser.add_option("--show-stale",
|
| 1797 |
action="store_true", dest="show_stale", |
| 1798 |
help="Show stale servers from previous runs, whose "
|
| 1799 |
"name starts with `%s'" % SNF_TEST_PREFIX,
|
| 1800 |
default=False)
|
| 1801 |
parser.add_option("--delete-stale",
|
| 1802 |
action="store_true", dest="delete_stale", |
| 1803 |
help="Delete stale servers from previous runs, whose "
|
| 1804 |
"name starts with `%s'" % SNF_TEST_PREFIX,
|
| 1805 |
default=False)
|
| 1806 |
parser.add_option("--force-personality",
|
| 1807 |
action="store", type="string", dest="personality_path", |
| 1808 |
help="Force a personality file injection.\
|
| 1809 |
File path required. ",
|
| 1810 |
default=None)
|
| 1811 |
parser.add_option("--log-folder",
|
| 1812 |
action="store", type="string", dest="log_folder", |
| 1813 |
help="Define the absolute path where the output \
|
| 1814 |
log is stored. ",
|
| 1815 |
default="/var/log/burnin/")
|
| 1816 |
parser.add_option("--verbose", "-V", |
| 1817 |
action="store_true", dest="verbose", |
| 1818 |
help="Print detailed output about multiple "
|
| 1819 |
"processes spawning",
|
| 1820 |
default=False)
|
| 1821 |
parser.add_option("--set-tests",
|
| 1822 |
action="callback",
|
| 1823 |
dest="tests",
|
| 1824 |
type="string",
|
| 1825 |
help='Set comma seperated tests for this run. \
|
| 1826 |
Available tests: auth, images, flavors, \
|
| 1827 |
servers, server_spawn, \
|
| 1828 |
network_spawn. \
|
| 1829 |
Default = all',
|
| 1830 |
default='all',
|
| 1831 |
callback=parse_comma) |
| 1832 |
|
| 1833 |
(opts, args) = parser.parse_args(args) |
| 1834 |
|
| 1835 |
# -----------------------
|
| 1836 |
# Verify arguments
|
| 1837 |
|
| 1838 |
# `delete_stale' implies `show_stale'
|
| 1839 |
if opts.delete_stale:
|
| 1840 |
opts.show_stale = True
|
| 1841 |
|
| 1842 |
# `image-id' is mandatory
|
| 1843 |
if not opts.show_stale: |
| 1844 |
if not opts.force_imageid: |
| 1845 |
print >>sys.stderr, red + \
|
| 1846 |
"The --image-id argument is mandatory.\n" + \
|
| 1847 |
normal |
| 1848 |
parser.print_help() |
| 1849 |
sys.exit(1)
|
| 1850 |
if opts.force_imageid != 'all': |
| 1851 |
try:
|
| 1852 |
opts.force_imageid = str(opts.force_imageid)
|
| 1853 |
except ValueError: |
| 1854 |
print >>sys.stderr, red + \
|
| 1855 |
"Invalid value specified for" + \
|
| 1856 |
"--image-id. Use a valid id, or `all'." + \
|
| 1857 |
normal |
| 1858 |
sys.exit(1)
|
| 1859 |
|
| 1860 |
# `token' is mandatory
|
| 1861 |
if not opts.token: |
| 1862 |
print >>sys.stderr, red + \
|
| 1863 |
"The --token argument is mandatory.\n" + \
|
| 1864 |
normal |
| 1865 |
parser.print_help() |
| 1866 |
sys.exit(1)
|
| 1867 |
|
| 1868 |
return (opts, args)
|
| 1869 |
|
| 1870 |
|
| 1871 |
# --------------------------------------------------------------------
|
| 1872 |
# Burnin main function
|
| 1873 |
def main(): |
| 1874 |
"""Assemble test cases into a test suite, and run it
|
| 1875 |
|
| 1876 |
IMPORTANT: Tests have dependencies and have to be run in the specified
|
| 1877 |
order inside a single test case. They communicate through attributes of the
|
| 1878 |
corresponding TestCase class (shared fixtures). Distinct subclasses of
|
| 1879 |
TestCase MAY SHARE NO DATA, since they are run in parallel, in distinct
|
| 1880 |
test runner processes.
|
| 1881 |
|
| 1882 |
"""
|
| 1883 |
|
| 1884 |
# Parse arguments using `optparse'
|
| 1885 |
(opts, args) = parse_arguments(sys.argv[1:])
|
| 1886 |
|
| 1887 |
# Some global variables
|
| 1888 |
global API, TOKEN, PLANKTON, PLANKTON_USER, NO_IPV6, VERBOSE, NOFAILFAST
|
| 1889 |
API = opts.api |
| 1890 |
TOKEN = opts.token |
| 1891 |
PLANKTON = opts.plankton |
| 1892 |
PLANKTON_USER = opts.plankton_user |
| 1893 |
NO_IPV6 = opts.no_ipv6 |
| 1894 |
VERBOSE = opts.verbose |
| 1895 |
NOFAILFAST = opts.nofailfast |
| 1896 |
|
| 1897 |
# If `show_stale', cleanup stale servers
|
| 1898 |
# from previous runs and exit
|
| 1899 |
if opts.show_stale:
|
| 1900 |
# We must clean the servers first
|
| 1901 |
cleanup_servers(opts.action_timeout, opts.query_interval, |
| 1902 |
delete_stale=opts.delete_stale) |
| 1903 |
cleanup_networks(opts.action_timeout, opts.query_interval, |
| 1904 |
delete_stale=opts.delete_stale) |
| 1905 |
return 0 |
| 1906 |
|
| 1907 |
# Initialize a kamaki instance, get flavors, images
|
| 1908 |
c = ComputeClient(API, TOKEN) |
| 1909 |
DIMAGES = c.list_images(detail=True)
|
| 1910 |
DFLAVORS = c.list_flavors(detail=True)
|
| 1911 |
|
| 1912 |
# FIXME: logging, log, LOG PID, TEST_RUN_ID, arguments
|
| 1913 |
# Run them: FIXME: In parallel, FAILEARLY, catchbreak?
|
| 1914 |
#unittest.main(verbosity=2, catchbreak=True)
|
| 1915 |
|
| 1916 |
# Get a list of images we are going to test
|
| 1917 |
if opts.force_imageid == 'all': |
| 1918 |
test_images = DIMAGES |
| 1919 |
else:
|
| 1920 |
test_images = filter(lambda x: x["id"] == opts.force_imageid, DIMAGES) |
| 1921 |
|
| 1922 |
# Create output (logging) folder
|
| 1923 |
if not os.path.exists(opts.log_folder): |
| 1924 |
os.mkdir(opts.log_folder) |
| 1925 |
test_folder = os.path.join(opts.log_folder, TEST_RUN_ID) |
| 1926 |
os.mkdir(test_folder) |
| 1927 |
|
| 1928 |
for image in test_images: |
| 1929 |
imageid = str(image["id"]) |
| 1930 |
imagename = image["name"]
|
| 1931 |
# Choose a flavor (given from user or random)
|
| 1932 |
if opts.force_flavorid:
|
| 1933 |
flavorid = opts.force_flavorid |
| 1934 |
else:
|
| 1935 |
flavorid = choice([f["id"] for f in DFLAVORS if f["disk"] >= 20]) |
| 1936 |
# Personality dictionary for file injection test
|
| 1937 |
if opts.personality_path is not None: |
| 1938 |
f = open(opts.personality_path)
|
| 1939 |
content = b64encode(f.read()) |
| 1940 |
personality = [] |
| 1941 |
st = os.stat(opts.personality_path) |
| 1942 |
personality.append({
|
| 1943 |
'path': '/root/test_inj_file', |
| 1944 |
'owner': 'root', |
| 1945 |
'group': 'root', |
| 1946 |
'mode': 0x7777 & st.st_mode, |
| 1947 |
'contents': content})
|
| 1948 |
else:
|
| 1949 |
personality = None
|
| 1950 |
# Give a name to our test servers
|
| 1951 |
servername = "%s%s for %s" % (SNF_TEST_PREFIX, TEST_RUN_ID, imagename)
|
| 1952 |
is_windows = imagename.lower().find("windows") >= 0 |
| 1953 |
|
| 1954 |
# Create Server TestCases
|
| 1955 |
ServerTestCase = _spawn_server_test_case( |
| 1956 |
imageid=imageid, |
| 1957 |
flavorid=flavorid, |
| 1958 |
imagename=imagename, |
| 1959 |
personality=personality, |
| 1960 |
servername=servername, |
| 1961 |
is_windows=is_windows, |
| 1962 |
action_timeout=opts.action_timeout, |
| 1963 |
build_warning=opts.build_warning, |
| 1964 |
build_fail=opts.build_fail, |
| 1965 |
query_interval=opts.query_interval) |
| 1966 |
# Create Network TestCases
|
| 1967 |
NetworkTestCase = _spawn_network_test_case( |
| 1968 |
action_timeout=opts.action_timeout, |
| 1969 |
imageid=imageid, |
| 1970 |
flavorid=flavorid, |
| 1971 |
imagename=imagename, |
| 1972 |
query_interval=opts.query_interval) |
| 1973 |
|
| 1974 |
# Choose the tests we are going to run
|
| 1975 |
test_dict = {'auth': UnauthorizedTestCase,
|
| 1976 |
'images': ImagesTestCase,
|
| 1977 |
'flavors': FlavorsTestCase,
|
| 1978 |
'servers': ServersTestCase,
|
| 1979 |
'server_spawn': ServerTestCase,
|
| 1980 |
'network_spawn': NetworkTestCase}
|
| 1981 |
seq_cases = [] |
| 1982 |
if 'all' in opts.tests: |
| 1983 |
seq_cases = [UnauthorizedTestCase, ImagesTestCase, FlavorsTestCase, |
| 1984 |
ServersTestCase, ServerTestCase, NetworkTestCase] |
| 1985 |
else:
|
| 1986 |
for test in opts.tests: |
| 1987 |
seq_cases.append(test_dict[test]) |
| 1988 |
|
| 1989 |
# Folder for each image
|
| 1990 |
image_folder = os.path.join(test_folder, imageid) |
| 1991 |
os.mkdir(image_folder) |
| 1992 |
|
| 1993 |
# Run each test
|
| 1994 |
if opts.fanout > 1: |
| 1995 |
_run_cases_in_parallel(seq_cases, opts.fanout, image_folder) |
| 1996 |
else:
|
| 1997 |
_run_cases_in_series(seq_cases, image_folder) |
| 1998 |
|
| 1999 |
|
| 2000 |
# --------------------------------------------------------------------
|
| 2001 |
# Call main
|
| 2002 |
if __name__ == "__main__": |
| 2003 |
sys.exit(main()) |