Statistics
| Branch: | Tag: | Revision:

root / snf-cyclades-app / synnefo / tools / burnin.py @ 74193008

History | View | Annotate | Download (33.9 kB)

1 5a140b23 Vangelis Koukis
#!/usr/bin/env python
2 5a140b23 Vangelis Koukis
3 5a140b23 Vangelis Koukis
# Copyright 2011 GRNET S.A. All rights reserved.
4 5a140b23 Vangelis Koukis
#
5 5a140b23 Vangelis Koukis
# Redistribution and use in source and binary forms, with or
6 5a140b23 Vangelis Koukis
# without modification, are permitted provided that the following
7 5a140b23 Vangelis Koukis
# conditions are met:
8 5a140b23 Vangelis Koukis
#
9 5a140b23 Vangelis Koukis
#   1. Redistributions of source code must retain the above
10 5a140b23 Vangelis Koukis
#      copyright notice, this list of conditions and the following
11 5a140b23 Vangelis Koukis
#      disclaimer.
12 5a140b23 Vangelis Koukis
#
13 5a140b23 Vangelis Koukis
#   2. Redistributions in binary form must reproduce the above
14 5a140b23 Vangelis Koukis
#      copyright notice, this list of conditions and the following
15 5a140b23 Vangelis Koukis
#      disclaimer in the documentation and/or other materials
16 5a140b23 Vangelis Koukis
#      provided with the distribution.
17 5a140b23 Vangelis Koukis
#
18 5a140b23 Vangelis Koukis
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
19 5a140b23 Vangelis Koukis
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
20 5a140b23 Vangelis Koukis
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
21 5a140b23 Vangelis Koukis
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
22 5a140b23 Vangelis Koukis
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 5a140b23 Vangelis Koukis
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 5a140b23 Vangelis Koukis
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
25 5a140b23 Vangelis Koukis
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
26 5a140b23 Vangelis Koukis
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27 5a140b23 Vangelis Koukis
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
28 5a140b23 Vangelis Koukis
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 5a140b23 Vangelis Koukis
# POSSIBILITY OF SUCH DAMAGE.
30 5a140b23 Vangelis Koukis
#
31 5a140b23 Vangelis Koukis
# The views and conclusions contained in the software and
32 5a140b23 Vangelis Koukis
# documentation are those of the authors and should not be
33 5a140b23 Vangelis Koukis
# interpreted as representing official policies, either expressed
34 5a140b23 Vangelis Koukis
# or implied, of GRNET S.A.
35 5a140b23 Vangelis Koukis
36 5a140b23 Vangelis Koukis
"""Perform integration testing on a running Synnefo deployment"""
37 5a140b23 Vangelis Koukis
38 21bbbc9b Vangelis Koukis
import __main__
39 5a140b23 Vangelis Koukis
import datetime
40 5a140b23 Vangelis Koukis
import inspect
41 5a140b23 Vangelis Koukis
import logging
42 5a140b23 Vangelis Koukis
import os
43 5a140b23 Vangelis Koukis
import paramiko
44 21bbbc9b Vangelis Koukis
import prctl
45 5a140b23 Vangelis Koukis
import subprocess
46 21bbbc9b Vangelis Koukis
import signal
47 5a140b23 Vangelis Koukis
import socket
48 bc14ba88 Vangelis Koukis
import struct
49 5a140b23 Vangelis Koukis
import sys
50 5a140b23 Vangelis Koukis
import time
51 5a140b23 Vangelis Koukis
52 5a140b23 Vangelis Koukis
from IPy import IP
53 21bbbc9b Vangelis Koukis
from multiprocessing import Process, Queue
54 bc14ba88 Vangelis Koukis
from random import choice
55 21bbbc9b Vangelis Koukis
56 1c636ad6 John Giannelos
from kamaki.clients import ClientError, ComputeClient, CycladesClient
57 1c636ad6 John Giannelos
from kamaki.config import Config
58 1c636ad6 John Giannelos
59 bc14ba88 Vangelis Koukis
from vncauthproxy.d3des import generate_response as d3des_generate_response
60 5a140b23 Vangelis Koukis
61 5a140b23 Vangelis Koukis
# Use backported unittest functionality if Python < 2.7
62 5a140b23 Vangelis Koukis
try:
63 5a140b23 Vangelis Koukis
    import unittest2 as unittest
64 5a140b23 Vangelis Koukis
except ImportError:
65 bc14ba88 Vangelis Koukis
    if sys.version_info < (2, 7):
66 bc14ba88 Vangelis Koukis
        raise Exception("The unittest2 package is required for Python < 2.7")
67 5a140b23 Vangelis Koukis
    import unittest
68 5a140b23 Vangelis Koukis
69 5a140b23 Vangelis Koukis
70 21bbbc9b Vangelis Koukis
API = None
71 21bbbc9b Vangelis Koukis
TOKEN = None
72 38d247df Kostas Papadimitriou
DEFAULT_API = "http://127.0.0.1:8000/api/v1.1"
73 38d247df Kostas Papadimitriou
74 5a140b23 Vangelis Koukis
# A unique id identifying this test run
75 21bbbc9b Vangelis Koukis
TEST_RUN_ID = datetime.datetime.strftime(datetime.datetime.now(),
76 21bbbc9b Vangelis Koukis
                                         "%Y%m%d%H%M%S")
77 21bbbc9b Vangelis Koukis
SNF_TEST_PREFIX = "snf-test-"
78 5a140b23 Vangelis Koukis
79 5a140b23 Vangelis Koukis
# Setup logging (FIXME - verigak)
80 5a140b23 Vangelis Koukis
logging.basicConfig(format="%(message)s")
81 00f87624 Vangelis Koukis
log = logging.getLogger("burnin")
82 5a140b23 Vangelis Koukis
log.setLevel(logging.INFO)
83 5a140b23 Vangelis Koukis
84 5a140b23 Vangelis Koukis
85 5a140b23 Vangelis Koukis
class UnauthorizedTestCase(unittest.TestCase):
86 5a140b23 Vangelis Koukis
    def test_unauthorized_access(self):
87 5a140b23 Vangelis Koukis
        """Test access without a valid token fails"""
88 1c636ad6 John Giannelos
        falseToken = '12345'
89 1c636ad6 John Giannelos
        conf = Config()
90 1c636ad6 John Giannelos
        conf.set('compute_token', falseToken)
91 1c636ad6 John Giannelos
92 5a140b23 Vangelis Koukis
        with self.assertRaises(ClientError) as cm:
93 5a140b23 Vangelis Koukis
            c.list_servers()
94 5a140b23 Vangelis Koukis
        self.assertEqual(cm.exception.status, 401)
95 5a140b23 Vangelis Koukis
96 5a140b23 Vangelis Koukis
97 5a140b23 Vangelis Koukis
class ImagesTestCase(unittest.TestCase):
98 5a140b23 Vangelis Koukis
    """Test image lists for consistency"""
99 5a140b23 Vangelis Koukis
    @classmethod
100 5a140b23 Vangelis Koukis
    def setUpClass(cls):
101 5a140b23 Vangelis Koukis
        """Initialize kamaki, get (detailed) list of images"""
102 5a140b23 Vangelis Koukis
        log.info("Getting simple and detailed list of images")
103 1c636ad6 John Giannelos
104 1c636ad6 John Giannelos
        conf = Config()
105 1c636ad6 John Giannelos
        conf.set('compute_token', TOKEN)
106 1c636ad6 John Giannelos
        cls.client = ComputeClient(conf)
107 5a140b23 Vangelis Koukis
        cls.images = cls.client.list_images()
108 5a140b23 Vangelis Koukis
        cls.dimages = cls.client.list_images(detail=True)
109 5a140b23 Vangelis Koukis
110 5a140b23 Vangelis Koukis
    def test_001_list_images(self):
111 5a140b23 Vangelis Koukis
        """Test image list actually returns images"""
112 5a140b23 Vangelis Koukis
        self.assertGreater(len(self.images), 0)
113 5a140b23 Vangelis Koukis
114 5a140b23 Vangelis Koukis
    def test_002_list_images_detailed(self):
115 5a140b23 Vangelis Koukis
        """Test detailed image list is the same length as list"""
116 5a140b23 Vangelis Koukis
        self.assertEqual(len(self.dimages), len(self.images))
117 5a140b23 Vangelis Koukis
118 5a140b23 Vangelis Koukis
    def test_003_same_image_names(self):
119 5a140b23 Vangelis Koukis
        """Test detailed and simple image list contain same names"""
120 5a140b23 Vangelis Koukis
        names = sorted(map(lambda x: x["name"], self.images))
121 5a140b23 Vangelis Koukis
        dnames = sorted(map(lambda x: x["name"], self.dimages))
122 5a140b23 Vangelis Koukis
        self.assertEqual(names, dnames)
123 5a140b23 Vangelis Koukis
124 5a140b23 Vangelis Koukis
    def test_004_unique_image_names(self):
125 5a140b23 Vangelis Koukis
        """Test images have unique names"""
126 5a140b23 Vangelis Koukis
        names = sorted(map(lambda x: x["name"], self.images))
127 5a140b23 Vangelis Koukis
        self.assertEqual(sorted(list(set(names))), names)
128 5a140b23 Vangelis Koukis
129 5a140b23 Vangelis Koukis
    def test_005_image_metadata(self):
130 5a140b23 Vangelis Koukis
        """Test every image has specific metadata defined"""
131 1c636ad6 John Giannelos
        keys = frozenset(["os", "description", "size"])
132 5a140b23 Vangelis Koukis
        for i in self.dimages:
133 5a140b23 Vangelis Koukis
            self.assertTrue(keys.issubset(i["metadata"]["values"].keys()))
134 5a140b23 Vangelis Koukis
135 5a140b23 Vangelis Koukis
136 5a140b23 Vangelis Koukis
class FlavorsTestCase(unittest.TestCase):
137 5a140b23 Vangelis Koukis
    """Test flavor lists for consistency"""
138 5a140b23 Vangelis Koukis
    @classmethod
139 5a140b23 Vangelis Koukis
    def setUpClass(cls):
140 5a140b23 Vangelis Koukis
        """Initialize kamaki, get (detailed) list of flavors"""
141 5a140b23 Vangelis Koukis
        log.info("Getting simple and detailed list of flavors")
142 1c636ad6 John Giannelos
143 1c636ad6 John Giannelos
        conf = Config()
144 1c636ad6 John Giannelos
        conf.set('compute_token', TOKEN)
145 1c636ad6 John Giannelos
        cls.client = ComputeClient(conf)
146 5a140b23 Vangelis Koukis
        cls.flavors = cls.client.list_flavors()
147 5a140b23 Vangelis Koukis
        cls.dflavors = cls.client.list_flavors(detail=True)
148 5a140b23 Vangelis Koukis
149 5a140b23 Vangelis Koukis
    def test_001_list_flavors(self):
150 5a140b23 Vangelis Koukis
        """Test flavor list actually returns flavors"""
151 5a140b23 Vangelis Koukis
        self.assertGreater(len(self.flavors), 0)
152 5a140b23 Vangelis Koukis
153 5a140b23 Vangelis Koukis
    def test_002_list_flavors_detailed(self):
154 5a140b23 Vangelis Koukis
        """Test detailed flavor list is the same length as list"""
155 5a140b23 Vangelis Koukis
        self.assertEquals(len(self.dflavors), len(self.flavors))
156 5a140b23 Vangelis Koukis
157 5a140b23 Vangelis Koukis
    def test_003_same_flavor_names(self):
158 5a140b23 Vangelis Koukis
        """Test detailed and simple flavor list contain same names"""
159 5a140b23 Vangelis Koukis
        names = sorted(map(lambda x: x["name"], self.flavors))
160 5a140b23 Vangelis Koukis
        dnames = sorted(map(lambda x: x["name"], self.dflavors))
161 5a140b23 Vangelis Koukis
        self.assertEqual(names, dnames)
162 5a140b23 Vangelis Koukis
163 5a140b23 Vangelis Koukis
    def test_004_unique_flavor_names(self):
164 5a140b23 Vangelis Koukis
        """Test flavors have unique names"""
165 5a140b23 Vangelis Koukis
        names = sorted(map(lambda x: x["name"], self.flavors))
166 5a140b23 Vangelis Koukis
        self.assertEqual(sorted(list(set(names))), names)
167 5a140b23 Vangelis Koukis
168 5a140b23 Vangelis Koukis
    def test_005_well_formed_flavor_names(self):
169 5a140b23 Vangelis Koukis
        """Test flavors have names of the form CxxRyyDzz
170 5a140b23 Vangelis Koukis

171 5a140b23 Vangelis Koukis
        Where xx is vCPU count, yy is RAM in MiB, zz is Disk in GiB
172 5a140b23 Vangelis Koukis

173 5a140b23 Vangelis Koukis
        """
174 5a140b23 Vangelis Koukis
        for f in self.dflavors:
175 5a140b23 Vangelis Koukis
            self.assertEqual("C%dR%dD%d" % (f["cpu"], f["ram"], f["disk"]),
176 5a140b23 Vangelis Koukis
                             f["name"],
177 5a140b23 Vangelis Koukis
                             "Flavor %s does not match its specs." % f["name"])
178 5a140b23 Vangelis Koukis
179 5a140b23 Vangelis Koukis
180 5a140b23 Vangelis Koukis
class ServersTestCase(unittest.TestCase):
181 5a140b23 Vangelis Koukis
    """Test server lists for consistency"""
182 5a140b23 Vangelis Koukis
    @classmethod
183 5a140b23 Vangelis Koukis
    def setUpClass(cls):
184 5a140b23 Vangelis Koukis
        """Initialize kamaki, get (detailed) list of servers"""
185 5a140b23 Vangelis Koukis
        log.info("Getting simple and detailed list of servers")
186 1c636ad6 John Giannelos
187 1c636ad6 John Giannelos
        conf = Config()
188 1c636ad6 John Giannelos
        conf.set('compute_token', TOKEN)
189 1c636ad6 John Giannelos
        cls.client = ComputeClient(conf)
190 5a140b23 Vangelis Koukis
        cls.servers = cls.client.list_servers()
191 5a140b23 Vangelis Koukis
        cls.dservers = cls.client.list_servers(detail=True)
192 5a140b23 Vangelis Koukis
193 5a140b23 Vangelis Koukis
    def test_001_list_servers(self):
194 5a140b23 Vangelis Koukis
        """Test server list actually returns servers"""
195 5a140b23 Vangelis Koukis
        self.assertGreater(len(self.servers), 0)
196 5a140b23 Vangelis Koukis
197 5a140b23 Vangelis Koukis
    def test_002_list_servers_detailed(self):
198 5a140b23 Vangelis Koukis
        """Test detailed server list is the same length as list"""
199 5a140b23 Vangelis Koukis
        self.assertEqual(len(self.dservers), len(self.servers))
200 5a140b23 Vangelis Koukis
201 5a140b23 Vangelis Koukis
    def test_003_same_server_names(self):
202 5a140b23 Vangelis Koukis
        """Test detailed and simple flavor list contain same names"""
203 5a140b23 Vangelis Koukis
        names = sorted(map(lambda x: x["name"], self.servers))
204 5a140b23 Vangelis Koukis
        dnames = sorted(map(lambda x: x["name"], self.dservers))
205 5a140b23 Vangelis Koukis
        self.assertEqual(names, dnames)
206 5a140b23 Vangelis Koukis
207 5a140b23 Vangelis Koukis
208 5a140b23 Vangelis Koukis
# This class gets replicated into actual TestCases dynamically
209 5a140b23 Vangelis Koukis
class SpawnServerTestCase(unittest.TestCase):
210 5a140b23 Vangelis Koukis
    """Test scenario for server of the specified image"""
211 5a140b23 Vangelis Koukis
212 5a140b23 Vangelis Koukis
    @classmethod
213 5a140b23 Vangelis Koukis
    def setUpClass(cls):
214 5a140b23 Vangelis Koukis
        """Initialize a kamaki instance"""
215 bc14ba88 Vangelis Koukis
        log.info("Spawning server for image `%s'", cls.imagename)
216 1c636ad6 John Giannelos
217 1c636ad6 John Giannelos
        conf = Config()
218 1c636ad6 John Giannelos
        conf.set('compute_token', TOKEN)
219 1c636ad6 John Giannelos
        cls.client = ComputeClient(conf)
220 74193008 John Giannelos
        cls.cyclades = CycladesClient(conf)
221 5a140b23 Vangelis Koukis
222 5a140b23 Vangelis Koukis
    def _get_ipv4(self, server):
223 bc14ba88 Vangelis Koukis
        """Get the public IPv4 of a server from the detailed server info"""
224 1c636ad6 John Giannelos
225 1c636ad6 John Giannelos
        details = 
226 5a140b23 Vangelis Koukis
        public_addrs = filter(lambda x: x["id"] == "public",
227 5a140b23 Vangelis Koukis
                              server["addresses"]["values"])
228 5a140b23 Vangelis Koukis
        self.assertEqual(len(public_addrs), 1)
229 5a140b23 Vangelis Koukis
        ipv4_addrs = filter(lambda x: x["version"] == 4,
230 5a140b23 Vangelis Koukis
                            public_addrs[0]["values"])
231 5a140b23 Vangelis Koukis
        self.assertEqual(len(ipv4_addrs), 1)
232 5a140b23 Vangelis Koukis
        return ipv4_addrs[0]["addr"]
233 5a140b23 Vangelis Koukis
234 5a140b23 Vangelis Koukis
    def _get_ipv6(self, server):
235 bc14ba88 Vangelis Koukis
        """Get the public IPv6 of a server from the detailed server info"""
236 5a140b23 Vangelis Koukis
        public_addrs = filter(lambda x: x["id"] == "public",
237 5a140b23 Vangelis Koukis
                              server["addresses"]["values"])
238 5a140b23 Vangelis Koukis
        self.assertEqual(len(public_addrs), 1)
239 5a140b23 Vangelis Koukis
        ipv6_addrs = filter(lambda x: x["version"] == 6,
240 5a140b23 Vangelis Koukis
                            public_addrs[0]["values"])
241 5a140b23 Vangelis Koukis
        self.assertEqual(len(ipv6_addrs), 1)
242 5a140b23 Vangelis Koukis
        return ipv6_addrs[0]["addr"]
243 5a140b23 Vangelis Koukis
244 bc14ba88 Vangelis Koukis
    def _connect_loginname(self, os):
245 bc14ba88 Vangelis Koukis
        """Return the login name for connections based on the server OS"""
246 1c636ad6 John Giannelos
        if os in ("Ubuntu", "Kubuntu", "Fedora"):
247 21bbbc9b Vangelis Koukis
            return "user"
248 1c636ad6 John Giannelos
        elif os in ("windows", "windows_alpha1"):
249 21bbbc9b Vangelis Koukis
            return "Administrator"
250 bc14ba88 Vangelis Koukis
        else:
251 21bbbc9b Vangelis Koukis
            return "root"
252 bc14ba88 Vangelis Koukis
253 bc14ba88 Vangelis Koukis
    def _verify_server_status(self, current_status, new_status):
254 bc14ba88 Vangelis Koukis
        """Verify a server has switched to a specified status"""
255 bc14ba88 Vangelis Koukis
        server = self.client.get_server_details(self.serverid)
256 21bbbc9b Vangelis Koukis
        if server["status"] not in (current_status, new_status):
257 21bbbc9b Vangelis Koukis
            return None  # Do not raise exception, return so the test fails
258 bc14ba88 Vangelis Koukis
        self.assertEquals(server["status"], new_status)
259 bc14ba88 Vangelis Koukis
260 bc14ba88 Vangelis Koukis
    def _get_connected_tcp_socket(self, family, host, port):
261 bc14ba88 Vangelis Koukis
        """Get a connected socket from the specified family to host:port"""
262 bc14ba88 Vangelis Koukis
        sock = None
263 bc14ba88 Vangelis Koukis
        for res in \
264 bc14ba88 Vangelis Koukis
            socket.getaddrinfo(host, port, family, socket.SOCK_STREAM, 0,
265 bc14ba88 Vangelis Koukis
                               socket.AI_PASSIVE):
266 bc14ba88 Vangelis Koukis
            af, socktype, proto, canonname, sa = res
267 bc14ba88 Vangelis Koukis
            try:
268 bc14ba88 Vangelis Koukis
                sock = socket.socket(af, socktype, proto)
269 bc14ba88 Vangelis Koukis
            except socket.error as msg:
270 bc14ba88 Vangelis Koukis
                sock = None
271 bc14ba88 Vangelis Koukis
                continue
272 bc14ba88 Vangelis Koukis
            try:
273 bc14ba88 Vangelis Koukis
                sock.connect(sa)
274 bc14ba88 Vangelis Koukis
            except socket.error as msg:
275 bc14ba88 Vangelis Koukis
                sock.close()
276 bc14ba88 Vangelis Koukis
                sock = None
277 bc14ba88 Vangelis Koukis
                continue
278 bc14ba88 Vangelis Koukis
        self.assertIsNotNone(sock)
279 bc14ba88 Vangelis Koukis
        return sock
280 bc14ba88 Vangelis Koukis
281 bc14ba88 Vangelis Koukis
    def _ping_once(self, ipv6, ip):
282 bc14ba88 Vangelis Koukis
        """Test server responds to a single IPv4 or IPv6 ping"""
283 bc14ba88 Vangelis Koukis
        cmd = "ping%s -c 2 -w 3 %s" % ("6" if ipv6 else "", ip)
284 bc14ba88 Vangelis Koukis
        ping = subprocess.Popen(cmd, shell=True,
285 bc14ba88 Vangelis Koukis
                                stdout=subprocess.PIPE, stderr=subprocess.PIPE)
286 bc14ba88 Vangelis Koukis
        (stdout, stderr) = ping.communicate()
287 bc14ba88 Vangelis Koukis
        ret = ping.wait()
288 bc14ba88 Vangelis Koukis
        self.assertEquals(ret, 0)
289 5a140b23 Vangelis Koukis
290 bc14ba88 Vangelis Koukis
    def _get_hostname_over_ssh(self, hostip, username, password):
291 bc14ba88 Vangelis Koukis
        ssh = paramiko.SSHClient()
292 bc14ba88 Vangelis Koukis
        ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
293 bc14ba88 Vangelis Koukis
        try:
294 bc14ba88 Vangelis Koukis
            ssh.connect(hostip, username=username, password=password)
295 bc14ba88 Vangelis Koukis
        except socket.error:
296 bc14ba88 Vangelis Koukis
            raise AssertionError
297 bc14ba88 Vangelis Koukis
        stdin, stdout, stderr = ssh.exec_command("hostname")
298 bc14ba88 Vangelis Koukis
        lines = stdout.readlines()
299 bc14ba88 Vangelis Koukis
        self.assertEqual(len(lines), 1)
300 4fdd25ab Vangelis Koukis
        return lines[0]
301 bc14ba88 Vangelis Koukis
302 bc14ba88 Vangelis Koukis
    def _try_until_timeout_expires(self, warn_timeout, fail_timeout,
303 bc14ba88 Vangelis Koukis
                                   opmsg, callable, *args, **kwargs):
304 bc14ba88 Vangelis Koukis
        if warn_timeout == fail_timeout:
305 5a140b23 Vangelis Koukis
            warn_timeout = fail_timeout + 1
306 5a140b23 Vangelis Koukis
        warn_tmout = time.time() + warn_timeout
307 5a140b23 Vangelis Koukis
        fail_tmout = time.time() + fail_timeout
308 5a140b23 Vangelis Koukis
        while True:
309 4fdd25ab Vangelis Koukis
            self.assertLess(time.time(), fail_tmout,
310 21bbbc9b Vangelis Koukis
                            "operation `%s' timed out" % opmsg)
311 5a140b23 Vangelis Koukis
            if time.time() > warn_tmout:
312 4fdd25ab Vangelis Koukis
                log.warning("Server %d: `%s' operation `%s' not done yet",
313 4fdd25ab Vangelis Koukis
                            self.serverid, self.servername, opmsg)
314 bc14ba88 Vangelis Koukis
            try:
315 4fdd25ab Vangelis Koukis
                log.info("%s... " % opmsg)
316 bc14ba88 Vangelis Koukis
                return callable(*args, **kwargs)
317 bc14ba88 Vangelis Koukis
            except AssertionError:
318 bc14ba88 Vangelis Koukis
                pass
319 5a140b23 Vangelis Koukis
            time.sleep(self.query_interval)
320 5a140b23 Vangelis Koukis
321 bc14ba88 Vangelis Koukis
    def _insist_on_tcp_connection(self, family, host, port):
322 21bbbc9b Vangelis Koukis
        familystr = {socket.AF_INET: "IPv4", socket.AF_INET6: "IPv6",
323 21bbbc9b Vangelis Koukis
                     socket.AF_UNSPEC: "Unspecified-IPv4/6"}
324 bc14ba88 Vangelis Koukis
        msg = "connect over %s to %s:%s" % \
325 bc14ba88 Vangelis Koukis
              (familystr.get(family, "Unknown"), host, port)
326 bc14ba88 Vangelis Koukis
        sock = self._try_until_timeout_expires(
327 bc14ba88 Vangelis Koukis
                self.action_timeout, self.action_timeout,
328 bc14ba88 Vangelis Koukis
                msg, self._get_connected_tcp_socket,
329 bc14ba88 Vangelis Koukis
                family, host, port)
330 bc14ba88 Vangelis Koukis
        return sock
331 bc14ba88 Vangelis Koukis
332 bc14ba88 Vangelis Koukis
    def _insist_on_status_transition(self, current_status, new_status,
333 bc14ba88 Vangelis Koukis
                                    fail_timeout, warn_timeout=None):
334 4fdd25ab Vangelis Koukis
        msg = "Server %d: `%s', waiting for %s -> %s" % \
335 4fdd25ab Vangelis Koukis
              (self.serverid, self.servername, current_status, new_status)
336 bc14ba88 Vangelis Koukis
        if warn_timeout is None:
337 bc14ba88 Vangelis Koukis
            warn_timeout = fail_timeout
338 bc14ba88 Vangelis Koukis
        self._try_until_timeout_expires(warn_timeout, fail_timeout,
339 bc14ba88 Vangelis Koukis
                                        msg, self._verify_server_status,
340 bc14ba88 Vangelis Koukis
                                        current_status, new_status)
341 21bbbc9b Vangelis Koukis
        # Ensure the status is actually the expected one
342 21bbbc9b Vangelis Koukis
        server = self.client.get_server_details(self.serverid)
343 21bbbc9b Vangelis Koukis
        self.assertEquals(server["status"], new_status)
344 bc14ba88 Vangelis Koukis
345 bc14ba88 Vangelis Koukis
    def _insist_on_ssh_hostname(self, hostip, username, password):
346 4fdd25ab Vangelis Koukis
        msg = "SSH to %s, as %s/%s" % (hostip, username, password)
347 bc14ba88 Vangelis Koukis
        hostname = self._try_until_timeout_expires(
348 bc14ba88 Vangelis Koukis
                self.action_timeout, self.action_timeout,
349 bc14ba88 Vangelis Koukis
                msg, self._get_hostname_over_ssh,
350 bc14ba88 Vangelis Koukis
                hostip, username, password)
351 bc14ba88 Vangelis Koukis
352 bc14ba88 Vangelis Koukis
        # The hostname must be of the form 'prefix-id'
353 bc14ba88 Vangelis Koukis
        self.assertTrue(hostname.endswith("-%d\n" % self.serverid))
354 5a140b23 Vangelis Koukis
355 5a140b23 Vangelis Koukis
    def _skipIf(self, condition, msg):
356 5a140b23 Vangelis Koukis
        if condition:
357 5a140b23 Vangelis Koukis
            self.skipTest(msg)
358 5a140b23 Vangelis Koukis
359 5a140b23 Vangelis Koukis
    def test_001_submit_create_server(self):
360 5a140b23 Vangelis Koukis
        """Test submit create server request"""
361 5a140b23 Vangelis Koukis
        server = self.client.create_server(self.servername, self.flavorid,
362 5a140b23 Vangelis Koukis
                                           self.imageid, self.personality)
363 5a140b23 Vangelis Koukis
        self.assertEqual(server["name"], self.servername)
364 5a140b23 Vangelis Koukis
        self.assertEqual(server["flavorRef"], self.flavorid)
365 5a140b23 Vangelis Koukis
        self.assertEqual(server["imageRef"], self.imageid)
366 5a140b23 Vangelis Koukis
        self.assertEqual(server["status"], "BUILD")
367 5a140b23 Vangelis Koukis
368 5a140b23 Vangelis Koukis
        # Update class attributes to reflect data on building server
369 5a140b23 Vangelis Koukis
        cls = type(self)
370 5a140b23 Vangelis Koukis
        cls.serverid = server["id"]
371 bc14ba88 Vangelis Koukis
        cls.username = None
372 5a140b23 Vangelis Koukis
        cls.passwd = server["adminPass"]
373 5a140b23 Vangelis Koukis
374 5a140b23 Vangelis Koukis
    def test_002a_server_is_building_in_list(self):
375 5a140b23 Vangelis Koukis
        """Test server is in BUILD state, in server list"""
376 5a140b23 Vangelis Koukis
        servers = self.client.list_servers(detail=True)
377 5a140b23 Vangelis Koukis
        servers = filter(lambda x: x["name"] == self.servername, servers)
378 5a140b23 Vangelis Koukis
        self.assertEqual(len(servers), 1)
379 5a140b23 Vangelis Koukis
        server = servers[0]
380 5a140b23 Vangelis Koukis
        self.assertEqual(server["name"], self.servername)
381 5a140b23 Vangelis Koukis
        self.assertEqual(server["flavorRef"], self.flavorid)
382 5a140b23 Vangelis Koukis
        self.assertEqual(server["imageRef"], self.imageid)
383 5a140b23 Vangelis Koukis
        self.assertEqual(server["status"], "BUILD")
384 5a140b23 Vangelis Koukis
385 5a140b23 Vangelis Koukis
    def test_002b_server_is_building_in_details(self):
386 5a140b23 Vangelis Koukis
        """Test server is in BUILD state, in details"""
387 5a140b23 Vangelis Koukis
        server = self.client.get_server_details(self.serverid)
388 5a140b23 Vangelis Koukis
        self.assertEqual(server["name"], self.servername)
389 5a140b23 Vangelis Koukis
        self.assertEqual(server["flavorRef"], self.flavorid)
390 5a140b23 Vangelis Koukis
        self.assertEqual(server["imageRef"], self.imageid)
391 5a140b23 Vangelis Koukis
        self.assertEqual(server["status"], "BUILD")
392 5a140b23 Vangelis Koukis
393 5a140b23 Vangelis Koukis
    def test_002c_set_server_metadata(self):
394 5a140b23 Vangelis Koukis
        image = self.client.get_image_details(self.imageid)
395 74193008 John Giannelos
        os = image["metadata"]["values"]["os"]
396 74193008 John Giannelos
        loginname = image["metadata"]["values"].get("users", None)
397 bc14ba88 Vangelis Koukis
        self.client.update_server_metadata(self.serverid, OS=os)
398 bc14ba88 Vangelis Koukis
399 bc14ba88 Vangelis Koukis
        # Determine the username to use for future connections
400 bc14ba88 Vangelis Koukis
        # to this host
401 bc14ba88 Vangelis Koukis
        cls = type(self)
402 bc14ba88 Vangelis Koukis
        cls.username = loginname
403 bc14ba88 Vangelis Koukis
        if not cls.username:
404 bc14ba88 Vangelis Koukis
            cls.username = self._connect_loginname(os)
405 bc14ba88 Vangelis Koukis
        self.assertIsNotNone(cls.username)
406 5a140b23 Vangelis Koukis
407 5a140b23 Vangelis Koukis
    def test_002d_verify_server_metadata(self):
408 5a140b23 Vangelis Koukis
        """Test server metadata keys are set based on image metadata"""
409 5a140b23 Vangelis Koukis
        servermeta = self.client.get_server_metadata(self.serverid)
410 5a140b23 Vangelis Koukis
        imagemeta = self.client.get_image_metadata(self.imageid)
411 5a140b23 Vangelis Koukis
        self.assertEqual(servermeta["OS"], imagemeta["OS"])
412 5a140b23 Vangelis Koukis
413 5a140b23 Vangelis Koukis
    def test_003_server_becomes_active(self):
414 5a140b23 Vangelis Koukis
        """Test server becomes ACTIVE"""
415 bc14ba88 Vangelis Koukis
        self._insist_on_status_transition("BUILD", "ACTIVE",
416 5a140b23 Vangelis Koukis
                                         self.build_fail, self.build_warning)
417 5a140b23 Vangelis Koukis
418 5a140b23 Vangelis Koukis
    def test_003a_get_server_oob_console(self):
419 bc14ba88 Vangelis Koukis
        """Test getting OOB server console over VNC
420 bc14ba88 Vangelis Koukis

421 bc14ba88 Vangelis Koukis
        Implementation of RFB protocol follows
422 bc14ba88 Vangelis Koukis
        http://www.realvnc.com/docs/rfbproto.pdf.
423 bc14ba88 Vangelis Koukis

424 bc14ba88 Vangelis Koukis
        """
425 74193008 John Giannelos
        
426 74193008 John Giannelos
        console = self.cyclades.get_server_console(self.serverid)
427 5a140b23 Vangelis Koukis
        self.assertEquals(console['type'], "vnc")
428 bc14ba88 Vangelis Koukis
        sock = self._insist_on_tcp_connection(socket.AF_UNSPEC,
429 5a140b23 Vangelis Koukis
                                        console["host"], console["port"])
430 bc14ba88 Vangelis Koukis
431 bc14ba88 Vangelis Koukis
        # Step 1. ProtocolVersion message (par. 6.1.1)
432 5a140b23 Vangelis Koukis
        version = sock.recv(1024)
433 bc14ba88 Vangelis Koukis
        self.assertEquals(version, 'RFB 003.008\n')
434 bc14ba88 Vangelis Koukis
        sock.send(version)
435 bc14ba88 Vangelis Koukis
436 bc14ba88 Vangelis Koukis
        # Step 2. Security (par 6.1.2): Only VNC Authentication supported
437 bc14ba88 Vangelis Koukis
        sec = sock.recv(1024)
438 bc14ba88 Vangelis Koukis
        self.assertEquals(list(sec), ['\x01', '\x02'])
439 bc14ba88 Vangelis Koukis
440 bc14ba88 Vangelis Koukis
        # Step 3. Request VNC Authentication (par 6.1.2)
441 bc14ba88 Vangelis Koukis
        sock.send('\x02')
442 bc14ba88 Vangelis Koukis
443 bc14ba88 Vangelis Koukis
        # Step 4. Receive Challenge (par 6.2.2)
444 bc14ba88 Vangelis Koukis
        challenge = sock.recv(1024)
445 bc14ba88 Vangelis Koukis
        self.assertEquals(len(challenge), 16)
446 bc14ba88 Vangelis Koukis
447 bc14ba88 Vangelis Koukis
        # Step 5. DES-Encrypt challenge, use password as key (par 6.2.2)
448 bc14ba88 Vangelis Koukis
        response = d3des_generate_response(
449 bc14ba88 Vangelis Koukis
            (console["password"] + '\0' * 8)[:8], challenge)
450 bc14ba88 Vangelis Koukis
        sock.send(response)
451 bc14ba88 Vangelis Koukis
452 bc14ba88 Vangelis Koukis
        # Step 6. SecurityResult (par 6.1.3)
453 bc14ba88 Vangelis Koukis
        result = sock.recv(4)
454 bc14ba88 Vangelis Koukis
        self.assertEquals(list(result), ['\x00', '\x00', '\x00', '\x00'])
455 5a140b23 Vangelis Koukis
        sock.close()
456 5a140b23 Vangelis Koukis
457 5a140b23 Vangelis Koukis
    def test_004_server_has_ipv4(self):
458 5a140b23 Vangelis Koukis
        """Test active server has a valid IPv4 address"""
459 5a140b23 Vangelis Koukis
        server = self.client.get_server_details(self.serverid)
460 5a140b23 Vangelis Koukis
        ipv4 = self._get_ipv4(server)
461 5a140b23 Vangelis Koukis
        self.assertEquals(IP(ipv4).version(), 4)
462 5a140b23 Vangelis Koukis
463 5a140b23 Vangelis Koukis
    def test_005_server_has_ipv6(self):
464 5a140b23 Vangelis Koukis
        """Test active server has a valid IPv6 address"""
465 5a140b23 Vangelis Koukis
        server = self.client.get_server_details(self.serverid)
466 5a140b23 Vangelis Koukis
        ipv6 = self._get_ipv6(server)
467 5a140b23 Vangelis Koukis
        self.assertEquals(IP(ipv6).version(), 6)
468 5a140b23 Vangelis Koukis
469 5a140b23 Vangelis Koukis
    def test_006_server_responds_to_ping_IPv4(self):
470 5a140b23 Vangelis Koukis
        """Test server responds to ping on IPv4 address"""
471 5a140b23 Vangelis Koukis
        server = self.client.get_server_details(self.serverid)
472 bc14ba88 Vangelis Koukis
        ip = self._get_ipv4(server)
473 bc14ba88 Vangelis Koukis
        self._try_until_timeout_expires(self.action_timeout,
474 bc14ba88 Vangelis Koukis
                                        self.action_timeout,
475 4fdd25ab Vangelis Koukis
                                        "PING IPv4 to %s" % ip,
476 4fdd25ab Vangelis Koukis
                                        self._ping_once,
477 bc14ba88 Vangelis Koukis
                                        False, ip)
478 5a140b23 Vangelis Koukis
479 5a140b23 Vangelis Koukis
    def test_007_server_responds_to_ping_IPv6(self):
480 5a140b23 Vangelis Koukis
        """Test server responds to ping on IPv6 address"""
481 5a140b23 Vangelis Koukis
        server = self.client.get_server_details(self.serverid)
482 bc14ba88 Vangelis Koukis
        ip = self._get_ipv6(server)
483 bc14ba88 Vangelis Koukis
        self._try_until_timeout_expires(self.action_timeout,
484 bc14ba88 Vangelis Koukis
                                        self.action_timeout,
485 4fdd25ab Vangelis Koukis
                                        "PING IPv6 to %s" % ip,
486 4fdd25ab Vangelis Koukis
                                        self._ping_once,
487 bc14ba88 Vangelis Koukis
                                        True, ip)
488 5a140b23 Vangelis Koukis
489 5a140b23 Vangelis Koukis
    def test_008_submit_shutdown_request(self):
490 5a140b23 Vangelis Koukis
        """Test submit request to shutdown server"""
491 74193008 John Giannelos
        self.cyclades.shutdown_server(self.serverid)
492 5a140b23 Vangelis Koukis
493 5a140b23 Vangelis Koukis
    def test_009_server_becomes_stopped(self):
494 5a140b23 Vangelis Koukis
        """Test server becomes STOPPED"""
495 bc14ba88 Vangelis Koukis
        self._insist_on_status_transition("ACTIVE", "STOPPED",
496 bc14ba88 Vangelis Koukis
                                         self.action_timeout,
497 5a140b23 Vangelis Koukis
                                         self.action_timeout)
498 5a140b23 Vangelis Koukis
499 5a140b23 Vangelis Koukis
    def test_010_submit_start_request(self):
500 5a140b23 Vangelis Koukis
        """Test submit start server request"""
501 74193008 John Giannelos
        self.cyclades.start_server(self.serverid)
502 5a140b23 Vangelis Koukis
503 5a140b23 Vangelis Koukis
    def test_011_server_becomes_active(self):
504 5a140b23 Vangelis Koukis
        """Test server becomes ACTIVE again"""
505 bc14ba88 Vangelis Koukis
        self._insist_on_status_transition("STOPPED", "ACTIVE",
506 bc14ba88 Vangelis Koukis
                                         self.action_timeout,
507 5a140b23 Vangelis Koukis
                                         self.action_timeout)
508 5a140b23 Vangelis Koukis
509 5a140b23 Vangelis Koukis
    def test_011a_server_responds_to_ping_IPv4(self):
510 5a140b23 Vangelis Koukis
        """Test server OS is actually up and running again"""
511 5a140b23 Vangelis Koukis
        self.test_006_server_responds_to_ping_IPv4()
512 5a140b23 Vangelis Koukis
513 5a140b23 Vangelis Koukis
    def test_012_ssh_to_server_IPv4(self):
514 5a140b23 Vangelis Koukis
        """Test SSH to server public IPv4 works, verify hostname"""
515 bc14ba88 Vangelis Koukis
        self._skipIf(self.is_windows, "only valid for Linux servers")
516 5a140b23 Vangelis Koukis
        server = self.client.get_server_details(self.serverid)
517 bc14ba88 Vangelis Koukis
        self._insist_on_ssh_hostname(self._get_ipv4(server),
518 bc14ba88 Vangelis Koukis
                                     self.username, self.passwd)
519 5a140b23 Vangelis Koukis
520 5a140b23 Vangelis Koukis
    def test_013_ssh_to_server_IPv6(self):
521 5a140b23 Vangelis Koukis
        """Test SSH to server public IPv6 works, verify hostname"""
522 bc14ba88 Vangelis Koukis
        self._skipIf(self.is_windows, "only valid for Linux servers")
523 5a140b23 Vangelis Koukis
        server = self.client.get_server_details(self.serverid)
524 bc14ba88 Vangelis Koukis
        self._insist_on_ssh_hostname(self._get_ipv6(server),
525 bc14ba88 Vangelis Koukis
                                     self.username, self.passwd)
526 5a140b23 Vangelis Koukis
527 5a140b23 Vangelis Koukis
    def test_014_rdp_to_server_IPv4(self):
528 5a140b23 Vangelis Koukis
        "Test RDP connection to server public IPv4 works"""
529 bc14ba88 Vangelis Koukis
        self._skipIf(not self.is_windows, "only valid for Windows servers")
530 5a140b23 Vangelis Koukis
        server = self.client.get_server_details(self.serverid)
531 5a140b23 Vangelis Koukis
        ipv4 = self._get_ipv4(server)
532 bc14ba88 Vangelis Koukis
        sock = _insist_on_tcp_connection(socket.AF_INET, ipv4, 3389)
533 5a140b23 Vangelis Koukis
534 5a140b23 Vangelis Koukis
        # No actual RDP processing done. We assume the RDP server is there
535 5a140b23 Vangelis Koukis
        # if the connection to the RDP port is successful.
536 cb1fa17c Vangelis Koukis
        # FIXME: Use rdesktop, analyze exit code? see manpage [costasd]
537 5a140b23 Vangelis Koukis
        sock.close()
538 5a140b23 Vangelis Koukis
539 5a140b23 Vangelis Koukis
    def test_015_rdp_to_server_IPv6(self):
540 5a140b23 Vangelis Koukis
        "Test RDP connection to server public IPv6 works"""
541 bc14ba88 Vangelis Koukis
        self._skipIf(not self.is_windows, "only valid for Windows servers")
542 5a140b23 Vangelis Koukis
        server = self.client.get_server_details(self.serverid)
543 5a140b23 Vangelis Koukis
        ipv6 = self._get_ipv6(server)
544 5a140b23 Vangelis Koukis
        sock = _get_tcp_connection(socket.AF_INET6, ipv6, 3389)
545 5a140b23 Vangelis Koukis
546 5a140b23 Vangelis Koukis
        # No actual RDP processing done. We assume the RDP server is there
547 5a140b23 Vangelis Koukis
        # if the connection to the RDP port is successful.
548 5a140b23 Vangelis Koukis
        sock.close()
549 5a140b23 Vangelis Koukis
550 5a140b23 Vangelis Koukis
    def test_016_personality_is_enforced(self):
551 5a140b23 Vangelis Koukis
        """Test file injection for personality enforcement"""
552 bc14ba88 Vangelis Koukis
        self._skipIf(self.is_windows, "only implemented for Linux servers")
553 5a140b23 Vangelis Koukis
        self.assertTrue(False, "test not implemented, will fail")
554 5a140b23 Vangelis Koukis
555 4fdd25ab Vangelis Koukis
    def test_017_submit_delete_request(self):
556 4fdd25ab Vangelis Koukis
        """Test submit request to delete server"""
557 4fdd25ab Vangelis Koukis
        self.client.delete_server(self.serverid)
558 4fdd25ab Vangelis Koukis
559 4fdd25ab Vangelis Koukis
    def test_018_server_becomes_deleted(self):
560 4fdd25ab Vangelis Koukis
        """Test server becomes DELETED"""
561 4fdd25ab Vangelis Koukis
        self._insist_on_status_transition("ACTIVE", "DELETED",
562 4fdd25ab Vangelis Koukis
                                         self.action_timeout,
563 4fdd25ab Vangelis Koukis
                                         self.action_timeout)
564 4fdd25ab Vangelis Koukis
565 4fdd25ab Vangelis Koukis
    def test_019_server_no_longer_in_server_list(self):
566 4fdd25ab Vangelis Koukis
        """Test server is no longer in server list"""
567 4fdd25ab Vangelis Koukis
        servers = self.client.list_servers()
568 21bbbc9b Vangelis Koukis
        self.assertNotIn(self.serverid, [s["id"] for s in servers])
569 21bbbc9b Vangelis Koukis
570 21bbbc9b Vangelis Koukis
571 21bbbc9b Vangelis Koukis
class TestRunnerProcess(Process):
572 21bbbc9b Vangelis Koukis
    """A distinct process used to execute part of the tests in parallel"""
573 21bbbc9b Vangelis Koukis
    def __init__(self, **kw):
574 21bbbc9b Vangelis Koukis
        Process.__init__(self, **kw)
575 21bbbc9b Vangelis Koukis
        kwargs = kw["kwargs"]
576 21bbbc9b Vangelis Koukis
        self.testq = kwargs["testq"]
577 21bbbc9b Vangelis Koukis
        self.runner = kwargs["runner"]
578 21bbbc9b Vangelis Koukis
579 21bbbc9b Vangelis Koukis
    def run(self):
580 21bbbc9b Vangelis Koukis
        # Make sure this test runner process dies with the parent
581 21bbbc9b Vangelis Koukis
        # and is not left behind.
582 21bbbc9b Vangelis Koukis
        #
583 21bbbc9b Vangelis Koukis
        # WARNING: This uses the prctl(2) call and is
584 21bbbc9b Vangelis Koukis
        # Linux-specific.
585 21bbbc9b Vangelis Koukis
        prctl.set_pdeathsig(signal.SIGHUP)
586 21bbbc9b Vangelis Koukis
587 21bbbc9b Vangelis Koukis
        while True:
588 21bbbc9b Vangelis Koukis
            log.debug("I am process %d, GETting from queue is %s",
589 21bbbc9b Vangelis Koukis
                     os.getpid(), self.testq)
590 21bbbc9b Vangelis Koukis
            msg = self.testq.get()
591 21bbbc9b Vangelis Koukis
            log.debug("Dequeued msg: %s", msg)
592 21bbbc9b Vangelis Koukis
593 21bbbc9b Vangelis Koukis
            if msg == "TEST_RUNNER_TERMINATE":
594 21bbbc9b Vangelis Koukis
                raise SystemExit
595 21bbbc9b Vangelis Koukis
            elif issubclass(msg, unittest.TestCase):
596 21bbbc9b Vangelis Koukis
                # Assemble a TestSuite, and run it
597 21bbbc9b Vangelis Koukis
                suite = unittest.TestLoader().loadTestsFromTestCase(msg)
598 21bbbc9b Vangelis Koukis
                self.runner.run(suite)
599 21bbbc9b Vangelis Koukis
            else:
600 21bbbc9b Vangelis Koukis
                raise Exception("Cannot handle msg: %s" % msg)
601 21bbbc9b Vangelis Koukis
602 21bbbc9b Vangelis Koukis
603 21bbbc9b Vangelis Koukis
def _run_cases_in_parallel(cases, fanout=1, runner=None):
604 21bbbc9b Vangelis Koukis
    """Run instances of TestCase in parallel, in a number of distinct processes
605 21bbbc9b Vangelis Koukis

606 21bbbc9b Vangelis Koukis
    The cases iterable specifies the TestCases to be executed in parallel,
607 21bbbc9b Vangelis Koukis
    by test runners running in distinct processes.
608 21bbbc9b Vangelis Koukis
    The fanout parameter specifies the number of processes to spawn,
609 21bbbc9b Vangelis Koukis
    and defaults to 1.
610 21bbbc9b Vangelis Koukis
    The runner argument specifies the test runner class to use inside each
611 21bbbc9b Vangelis Koukis
    runner process.
612 21bbbc9b Vangelis Koukis

613 21bbbc9b Vangelis Koukis
    """
614 21bbbc9b Vangelis Koukis
    if runner is None:
615 00f87624 Vangelis Koukis
        runner = unittest.TextTestRunner(verbosity=2, failfast=True)
616 21bbbc9b Vangelis Koukis
617 21bbbc9b Vangelis Koukis
    # testq: The master process enqueues TestCase objects into this queue,
618 21bbbc9b Vangelis Koukis
    #        test runner processes pick them up for execution, in parallel.
619 21bbbc9b Vangelis Koukis
    testq = Queue()
620 21bbbc9b Vangelis Koukis
    runners = []
621 21bbbc9b Vangelis Koukis
    for i in xrange(0, fanout):
622 21bbbc9b Vangelis Koukis
        kwargs = dict(testq=testq, runner=runner)
623 21bbbc9b Vangelis Koukis
        runners.append(TestRunnerProcess(kwargs=kwargs))
624 21bbbc9b Vangelis Koukis
625 21bbbc9b Vangelis Koukis
    log.info("Spawning %d test runner processes", len(runners))
626 21bbbc9b Vangelis Koukis
    for p in runners:
627 21bbbc9b Vangelis Koukis
        p.start()
628 21bbbc9b Vangelis Koukis
    log.debug("Spawned %d test runners, PIDs are %s",
629 21bbbc9b Vangelis Koukis
              len(runners), [p.pid for p in runners])
630 21bbbc9b Vangelis Koukis
631 21bbbc9b Vangelis Koukis
    # Enqueue test cases
632 21bbbc9b Vangelis Koukis
    map(testq.put, cases)
633 21bbbc9b Vangelis Koukis
    map(testq.put, ["TEST_RUNNER_TERMINATE"] * len(runners))
634 21bbbc9b Vangelis Koukis
635 21bbbc9b Vangelis Koukis
    log.debug("Joining %d processes", len(runners))
636 21bbbc9b Vangelis Koukis
    for p in runners:
637 21bbbc9b Vangelis Koukis
        p.join()
638 21bbbc9b Vangelis Koukis
    log.debug("Done joining %d processes", len(runners))
639 4fdd25ab Vangelis Koukis
640 5a140b23 Vangelis Koukis
641 5a140b23 Vangelis Koukis
def _spawn_server_test_case(**kwargs):
642 5a140b23 Vangelis Koukis
    """Construct a new unit test case class from SpawnServerTestCase"""
643 5a140b23 Vangelis Koukis
644 21bbbc9b Vangelis Koukis
    name = "SpawnServerTestCase_%d" % kwargs["imageid"]
645 5a140b23 Vangelis Koukis
    cls = type(name, (SpawnServerTestCase,), kwargs)
646 5a140b23 Vangelis Koukis
647 5a140b23 Vangelis Koukis
    # Patch extra parameters into test names by manipulating method docstrings
648 5a140b23 Vangelis Koukis
    for (mname, m) in \
649 5a140b23 Vangelis Koukis
        inspect.getmembers(cls, lambda x: inspect.ismethod(x)):
650 5a140b23 Vangelis Koukis
            if hasattr(m, __doc__):
651 5a140b23 Vangelis Koukis
                m.__func__.__doc__ = "[%s] %s" % (imagename, m.__doc__)
652 e72bcf60 Vangelis Koukis
653 e72bcf60 Vangelis Koukis
    # Make sure the class can be pickled, by listing it among
654 e72bcf60 Vangelis Koukis
    # the attributes of __main__. A PicklingError is raised otherwise.
655 e72bcf60 Vangelis Koukis
    setattr(__main__, name, cls)
656 5a140b23 Vangelis Koukis
    return cls
657 5a140b23 Vangelis Koukis
658 5a140b23 Vangelis Koukis
659 5a140b23 Vangelis Koukis
def cleanup_servers(delete_stale=False):
660 74193008 John Giannelos
661 74193008 John Giannelos
    conf = Config()
662 74193008 John Giannelos
    conf.set('compute_token', TOKEN)
663 74193008 John Giannelos
    c = ComputeClient(conf)
664 74193008 John Giannelos
665 5a140b23 Vangelis Koukis
    servers = c.list_servers()
666 5a140b23 Vangelis Koukis
    stale = [s for s in servers if s["name"].startswith(SNF_TEST_PREFIX)]
667 5a140b23 Vangelis Koukis
668 4fdd25ab Vangelis Koukis
    if len(stale) == 0:
669 4fdd25ab Vangelis Koukis
        return
670 4fdd25ab Vangelis Koukis
671 5a140b23 Vangelis Koukis
    print >> sys.stderr, "Found these stale servers from previous runs:"
672 5a140b23 Vangelis Koukis
    print "    " + \
673 21bbbc9b Vangelis Koukis
          "\n    ".join(["%d: %s" % (s["id"], s["name"]) for s in stale])
674 5a140b23 Vangelis Koukis
675 5a140b23 Vangelis Koukis
    if delete_stale:
676 5a140b23 Vangelis Koukis
        print >> sys.stderr, "Deleting %d stale servers:" % len(stale)
677 5a140b23 Vangelis Koukis
        for server in stale:
678 21bbbc9b Vangelis Koukis
            c.delete_server(server["id"])
679 5a140b23 Vangelis Koukis
        print >> sys.stderr, "    ...done"
680 5a140b23 Vangelis Koukis
    else:
681 5a140b23 Vangelis Koukis
        print >> sys.stderr, "Use --delete-stale to delete them."
682 5a140b23 Vangelis Koukis
683 5a140b23 Vangelis Koukis
684 5a140b23 Vangelis Koukis
def parse_arguments(args):
685 5a140b23 Vangelis Koukis
    from optparse import OptionParser
686 5a140b23 Vangelis Koukis
687 5a140b23 Vangelis Koukis
    kw = {}
688 5a140b23 Vangelis Koukis
    kw["usage"] = "%prog [options]"
689 5a140b23 Vangelis Koukis
    kw["description"] = \
690 5a140b23 Vangelis Koukis
        "%prog runs a number of test scenarios on a " \
691 5a140b23 Vangelis Koukis
        "Synnefo deployment."
692 5a140b23 Vangelis Koukis
693 5a140b23 Vangelis Koukis
    parser = OptionParser(**kw)
694 5a140b23 Vangelis Koukis
    parser.disable_interspersed_args()
695 21bbbc9b Vangelis Koukis
    parser.add_option("--api",
696 21bbbc9b Vangelis Koukis
                      action="store", type="string", dest="api",
697 21bbbc9b Vangelis Koukis
                      help="The API URI to use to reach the Synnefo API",
698 21bbbc9b Vangelis Koukis
                      default=DEFAULT_API)
699 21bbbc9b Vangelis Koukis
    parser.add_option("--token",
700 21bbbc9b Vangelis Koukis
                      action="store", type="string", dest="token",
701 38d247df Kostas Papadimitriou
                      help="The token to use for authentication to the API")
702 00f87624 Vangelis Koukis
    parser.add_option("--nofailfast",
703 00f87624 Vangelis Koukis
                      action="store_true", dest="nofailfast",
704 00f87624 Vangelis Koukis
                      help="Do not fail immediately if one of the tests " \
705 00f87624 Vangelis Koukis
                           "fails (EXPERIMENTAL)",
706 bc14ba88 Vangelis Koukis
                      default=False)
707 5a140b23 Vangelis Koukis
    parser.add_option("--action-timeout",
708 5a140b23 Vangelis Koukis
                      action="store", type="int", dest="action_timeout",
709 5a140b23 Vangelis Koukis
                      metavar="TIMEOUT",
710 5a140b23 Vangelis Koukis
                      help="Wait SECONDS seconds for a server action to " \
711 5a140b23 Vangelis Koukis
                           "complete, then the test is considered failed",
712 5a140b23 Vangelis Koukis
                      default=20)
713 5a140b23 Vangelis Koukis
    parser.add_option("--build-warning",
714 5a140b23 Vangelis Koukis
                      action="store", type="int", dest="build_warning",
715 5a140b23 Vangelis Koukis
                      metavar="TIMEOUT",
716 5a140b23 Vangelis Koukis
                      help="Warn if TIMEOUT seconds have passed and a " \
717 5a140b23 Vangelis Koukis
                           "build operation is still pending",
718 5a140b23 Vangelis Koukis
                      default=600)
719 5a140b23 Vangelis Koukis
    parser.add_option("--build-fail",
720 5a140b23 Vangelis Koukis
                      action="store", type="int", dest="build_fail",
721 5a140b23 Vangelis Koukis
                      metavar="BUILD_TIMEOUT",
722 5a140b23 Vangelis Koukis
                      help="Fail the test if TIMEOUT seconds have passed " \
723 5a140b23 Vangelis Koukis
                           "and a build operation is still incomplete",
724 5a140b23 Vangelis Koukis
                      default=900)
725 5a140b23 Vangelis Koukis
    parser.add_option("--query-interval",
726 5a140b23 Vangelis Koukis
                      action="store", type="int", dest="query_interval",
727 5a140b23 Vangelis Koukis
                      metavar="INTERVAL",
728 5a140b23 Vangelis Koukis
                      help="Query server status when requests are pending " \
729 5a140b23 Vangelis Koukis
                           "every INTERVAL seconds",
730 5a140b23 Vangelis Koukis
                      default=3)
731 21bbbc9b Vangelis Koukis
    parser.add_option("--fanout",
732 21bbbc9b Vangelis Koukis
                      action="store", type="int", dest="fanout",
733 5a140b23 Vangelis Koukis
                      metavar="COUNT",
734 21bbbc9b Vangelis Koukis
                      help="Spawn up to COUNT child processes to execute " \
735 21bbbc9b Vangelis Koukis
                           "in parallel, essentially have up to COUNT " \
736 00f87624 Vangelis Koukis
                           "server build requests outstanding (EXPERIMENTAL)",
737 5a140b23 Vangelis Koukis
                      default=1)
738 5a140b23 Vangelis Koukis
    parser.add_option("--force-flavor",
739 5a140b23 Vangelis Koukis
                      action="store", type="int", dest="force_flavorid",
740 5a140b23 Vangelis Koukis
                      metavar="FLAVOR ID",
741 5a140b23 Vangelis Koukis
                      help="Force all server creations to use the specified "\
742 5a140b23 Vangelis Koukis
                           "FLAVOR ID instead of a randomly chosen one, " \
743 5a140b23 Vangelis Koukis
                           "useful if disk space is scarce",
744 00f87624 Vangelis Koukis
                      default=None)
745 7f62a0b5 Vangelis Koukis
    parser.add_option("--image-id",
746 7f62a0b5 Vangelis Koukis
                      action="store", type="string", dest="force_imageid",
747 00f87624 Vangelis Koukis
                      metavar="IMAGE ID",
748 7f62a0b5 Vangelis Koukis
                      help="Test the specified image id, use 'all' to test " \
749 7f62a0b5 Vangelis Koukis
                           "all available images (mandatory argument)",
750 00f87624 Vangelis Koukis
                      default=None)
751 5a140b23 Vangelis Koukis
    parser.add_option("--show-stale",
752 5a140b23 Vangelis Koukis
                      action="store_true", dest="show_stale",
753 5a140b23 Vangelis Koukis
                      help="Show stale servers from previous runs, whose "\
754 21bbbc9b Vangelis Koukis
                           "name starts with `%s'" % SNF_TEST_PREFIX,
755 5a140b23 Vangelis Koukis
                      default=False)
756 5a140b23 Vangelis Koukis
    parser.add_option("--delete-stale",
757 5a140b23 Vangelis Koukis
                      action="store_true", dest="delete_stale",
758 5a140b23 Vangelis Koukis
                      help="Delete stale servers from previous runs, whose "\
759 21bbbc9b Vangelis Koukis
                           "name starts with `%s'" % SNF_TEST_PREFIX,
760 5a140b23 Vangelis Koukis
                      default=False)
761 5a140b23 Vangelis Koukis
762 5a140b23 Vangelis Koukis
    # FIXME: Change the default for build-fanout to 10
763 5a140b23 Vangelis Koukis
    # FIXME: Allow the user to specify a specific set of Images to test
764 5a140b23 Vangelis Koukis
765 5a140b23 Vangelis Koukis
    (opts, args) = parser.parse_args(args)
766 5a140b23 Vangelis Koukis
767 5a140b23 Vangelis Koukis
    # Verify arguments
768 5a140b23 Vangelis Koukis
    if opts.delete_stale:
769 5a140b23 Vangelis Koukis
        opts.show_stale = True
770 5a140b23 Vangelis Koukis
771 7f62a0b5 Vangelis Koukis
    if not opts.show_stale:
772 7f62a0b5 Vangelis Koukis
        if not opts.force_imageid:
773 7f62a0b5 Vangelis Koukis
            print >>sys.stderr, "The --image-id argument is mandatory."
774 7f62a0b5 Vangelis Koukis
            parser.print_help()
775 7f62a0b5 Vangelis Koukis
            sys.exit(1)
776 7f62a0b5 Vangelis Koukis
777 7f62a0b5 Vangelis Koukis
        if opts.force_imageid != 'all':
778 7f62a0b5 Vangelis Koukis
            try:
779 7f62a0b5 Vangelis Koukis
                opts.force_imageid = int(opts.force_imageid)
780 7f62a0b5 Vangelis Koukis
            except ValueError:
781 7f62a0b5 Vangelis Koukis
                print >>sys.stderr, "Invalid value specified for --image-id." \
782 7f62a0b5 Vangelis Koukis
                                    "Use a numeric id, or `all'."
783 7f62a0b5 Vangelis Koukis
                sys.exit(1)
784 7f62a0b5 Vangelis Koukis
785 5a140b23 Vangelis Koukis
    return (opts, args)
786 5a140b23 Vangelis Koukis
787 5a140b23 Vangelis Koukis
788 5a140b23 Vangelis Koukis
def main():
789 5a140b23 Vangelis Koukis
    """Assemble test cases into a test suite, and run it
790 5a140b23 Vangelis Koukis

791 5a140b23 Vangelis Koukis
    IMPORTANT: Tests have dependencies and have to be run in the specified
792 5a140b23 Vangelis Koukis
    order inside a single test case. They communicate through attributes of the
793 21bbbc9b Vangelis Koukis
    corresponding TestCase class (shared fixtures). Distinct subclasses of
794 21bbbc9b Vangelis Koukis
    TestCase MAY SHARE NO DATA, since they are run in parallel, in distinct
795 21bbbc9b Vangelis Koukis
    test runner processes.
796 5a140b23 Vangelis Koukis

797 5a140b23 Vangelis Koukis
    """
798 5a140b23 Vangelis Koukis
    (opts, args) = parse_arguments(sys.argv[1:])
799 5a140b23 Vangelis Koukis
800 21bbbc9b Vangelis Koukis
    global API, TOKEN
801 21bbbc9b Vangelis Koukis
    API = opts.api
802 21bbbc9b Vangelis Koukis
    TOKEN = opts.token
803 21bbbc9b Vangelis Koukis
804 5a140b23 Vangelis Koukis
    # Cleanup stale servers from previous runs
805 5a140b23 Vangelis Koukis
    if opts.show_stale:
806 5a140b23 Vangelis Koukis
        cleanup_servers(delete_stale=opts.delete_stale)
807 5a140b23 Vangelis Koukis
        return 0
808 5a140b23 Vangelis Koukis
809 5a140b23 Vangelis Koukis
    # Initialize a kamaki instance, get flavors, images
810 74193008 John Giannelos
811 74193008 John Giannelos
    conf = Config()
812 74193008 John Giannelos
    conf.set('compute_token', TOKEN)
813 74193008 John Giannelos
    c = ComputeClient(conf)
814 74193008 John Giannelos
815 5a140b23 Vangelis Koukis
    DIMAGES = c.list_images(detail=True)
816 5a140b23 Vangelis Koukis
    DFLAVORS = c.list_flavors(detail=True)
817 5a140b23 Vangelis Koukis
818 21bbbc9b Vangelis Koukis
    # FIXME: logging, log, LOG PID, TEST_RUN_ID, arguments
819 21bbbc9b Vangelis Koukis
    # FIXME: Network testing? Create, destroy, connect, ping, disconnect VMs?
820 5a140b23 Vangelis Koukis
    # Run them: FIXME: In parallel, FAILEARLY, catchbreak?
821 5a140b23 Vangelis Koukis
    #unittest.main(verbosity=2, catchbreak=True)
822 5a140b23 Vangelis Koukis
823 00f87624 Vangelis Koukis
    runner = unittest.TextTestRunner(verbosity=2, failfast=not opts.nofailfast)
824 21bbbc9b Vangelis Koukis
    # The following cases run sequentially
825 21bbbc9b Vangelis Koukis
    seq_cases = [UnauthorizedTestCase, FlavorsTestCase, ImagesTestCase]
826 21bbbc9b Vangelis Koukis
    _run_cases_in_parallel(seq_cases, fanout=3, runner=runner)
827 21bbbc9b Vangelis Koukis
828 21bbbc9b Vangelis Koukis
    # The following cases run in parallel
829 21bbbc9b Vangelis Koukis
    par_cases = []
830 5a140b23 Vangelis Koukis
831 7f62a0b5 Vangelis Koukis
    if opts.force_imageid == 'all':
832 00f87624 Vangelis Koukis
        test_images = DIMAGES
833 7f62a0b5 Vangelis Koukis
    else:
834 7f62a0b5 Vangelis Koukis
        test_images = filter(lambda x: x["id"] == opts.force_imageid, DIMAGES)
835 00f87624 Vangelis Koukis
836 00f87624 Vangelis Koukis
    for image in test_images:
837 21bbbc9b Vangelis Koukis
        imageid = image["id"]
838 21bbbc9b Vangelis Koukis
        imagename = image["name"]
839 21bbbc9b Vangelis Koukis
        if opts.force_flavorid:
840 21bbbc9b Vangelis Koukis
            flavorid = opts.force_flavorid
841 21bbbc9b Vangelis Koukis
        else:
842 21bbbc9b Vangelis Koukis
            flavorid = choice([f["id"] for f in DFLAVORS if f["disk"] >= 20])
843 21bbbc9b Vangelis Koukis
        personality = None   # FIXME
844 21bbbc9b Vangelis Koukis
        servername = "%s%s for %s" % (SNF_TEST_PREFIX, TEST_RUN_ID, imagename)
845 21bbbc9b Vangelis Koukis
        is_windows = imagename.lower().find("windows") >= 0
846 21bbbc9b Vangelis Koukis
        case = _spawn_server_test_case(imageid=imageid, flavorid=flavorid,
847 21bbbc9b Vangelis Koukis
                                       imagename=imagename,
848 21bbbc9b Vangelis Koukis
                                       personality=personality,
849 21bbbc9b Vangelis Koukis
                                       servername=servername,
850 21bbbc9b Vangelis Koukis
                                       is_windows=is_windows,
851 21bbbc9b Vangelis Koukis
                                       action_timeout=opts.action_timeout,
852 21bbbc9b Vangelis Koukis
                                       build_warning=opts.build_warning,
853 21bbbc9b Vangelis Koukis
                                       build_fail=opts.build_fail,
854 21bbbc9b Vangelis Koukis
                                       query_interval=opts.query_interval)
855 21bbbc9b Vangelis Koukis
        par_cases.append(case)
856 21bbbc9b Vangelis Koukis
857 21bbbc9b Vangelis Koukis
    _run_cases_in_parallel(par_cases, fanout=opts.fanout, runner=runner)
858 5a140b23 Vangelis Koukis
859 5a140b23 Vangelis Koukis
if __name__ == "__main__":
860 5a140b23 Vangelis Koukis
    sys.exit(main())