Revision cd737122

/dev/null
1
#!/usr/bin/env python
2

  
3
# Copyright 2011 GRNET S.A. All rights reserved.
4
#
5
# Redistribution and use in source and binary forms, with or
6
# without modification, are permitted provided that the following
7
# conditions are met:
8
#
9
#   1. Redistributions of source code must retain the above
10
#      copyright notice, this list of conditions and the following
11
#      disclaimer.
12
#
13
#   2. Redistributions in binary form must reproduce the above
14
#      copyright notice, this list of conditions and the following
15
#      disclaimer in the documentation and/or other materials
16
#      provided with the distribution.
17
#
18
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
19
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
20
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
21
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
22
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
25
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
26
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
28
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29
# POSSIBILITY OF SUCH DAMAGE.
30
#
31
# The views and conclusions contained in the software and
32
# documentation are those of the authors and should not be
33
# interpreted as representing official policies, either expressed
34
# or implied, of GRNET S.A.
35

  
36
"""Perform integration testing on a running Synnefo deployment"""
37

  
38
import __main__
39
import datetime
40
import inspect
41
import logging
42
import os
43
import paramiko
44
import prctl
45
import subprocess
46
import signal
47
import socket
48
import struct
49
import sys
50
import time
51

  
52
from IPy import IP
53
from multiprocessing import Process, Queue
54
from random import choice
55

  
56
from kamaki.client import Client, ClientError
57
from vncauthproxy.d3des import generate_response as d3des_generate_response
58

  
59
# Use backported unittest functionality if Python < 2.7
60
try:
61
    import unittest2 as unittest
62
except ImportError:
63
    if sys.version_info < (2, 7):
64
        raise Exception("The unittest2 package is required for Python < 2.7")
65
    import unittest
66

  
67

  
68
API = None
69
TOKEN = None
70
DEFAULT_API = "http://dev67.dev.grnet.gr:8000/api/v1.1"
71
DEFAULT_TOKEN = "46e427d657b20defe352804f0eb6f8a2"
72
# A unique id identifying this test run
73
TEST_RUN_ID = datetime.datetime.strftime(datetime.datetime.now(),
74
                                         "%Y%m%d%H%M%S")
75
SNF_TEST_PREFIX = "snf-test-"
76

  
77
# Setup logging (FIXME - verigak)
78
logging.basicConfig(format="%(message)s")
79
log = logging.getLogger("burnin")
80
log.setLevel(logging.INFO)
81

  
82

  
83
class UnauthorizedTestCase(unittest.TestCase):
84
    def test_unauthorized_access(self):
85
        """Test access without a valid token fails"""
86
        c = Client(API, "123")
87
        with self.assertRaises(ClientError) as cm:
88
            c.list_servers()
89
        self.assertEqual(cm.exception.status, 401)
90

  
91

  
92
class ImagesTestCase(unittest.TestCase):
93
    """Test image lists for consistency"""
94
    @classmethod
95
    def setUpClass(cls):
96
        """Initialize kamaki, get (detailed) list of images"""
97
        log.info("Getting simple and detailed list of images")
98
        cls.client = Client(API, TOKEN)
99
        cls.images = cls.client.list_images()
100
        cls.dimages = cls.client.list_images(detail=True)
101

  
102
    def test_001_list_images(self):
103
        """Test image list actually returns images"""
104
        self.assertGreater(len(self.images), 0)
105

  
106
    def test_002_list_images_detailed(self):
107
        """Test detailed image list is the same length as list"""
108
        self.assertEqual(len(self.dimages), len(self.images))
109

  
110
    def test_003_same_image_names(self):
111
        """Test detailed and simple image list contain same names"""
112
        names = sorted(map(lambda x: x["name"], self.images))
113
        dnames = sorted(map(lambda x: x["name"], self.dimages))
114
        self.assertEqual(names, dnames)
115

  
116
    def test_004_unique_image_names(self):
117
        """Test images have unique names"""
118
        names = sorted(map(lambda x: x["name"], self.images))
119
        self.assertEqual(sorted(list(set(names))), names)
120

  
121
    def test_005_image_metadata(self):
122
        """Test every image has specific metadata defined"""
123
        keys = frozenset(["OS", "description", "size"])
124
        for i in self.dimages:
125
            self.assertTrue(keys.issubset(i["metadata"]["values"].keys()))
126

  
127

  
128
class FlavorsTestCase(unittest.TestCase):
129
    """Test flavor lists for consistency"""
130
    @classmethod
131
    def setUpClass(cls):
132
        """Initialize kamaki, get (detailed) list of flavors"""
133
        log.info("Getting simple and detailed list of flavors")
134
        cls.client = Client(API, TOKEN)
135
        cls.flavors = cls.client.list_flavors()
136
        cls.dflavors = cls.client.list_flavors(detail=True)
137

  
138
    def test_001_list_flavors(self):
139
        """Test flavor list actually returns flavors"""
140
        self.assertGreater(len(self.flavors), 0)
141

  
142
    def test_002_list_flavors_detailed(self):
143
        """Test detailed flavor list is the same length as list"""
144
        self.assertEquals(len(self.dflavors), len(self.flavors))
145

  
146
    def test_003_same_flavor_names(self):
147
        """Test detailed and simple flavor list contain same names"""
148
        names = sorted(map(lambda x: x["name"], self.flavors))
149
        dnames = sorted(map(lambda x: x["name"], self.dflavors))
150
        self.assertEqual(names, dnames)
151

  
152
    def test_004_unique_flavor_names(self):
153
        """Test flavors have unique names"""
154
        names = sorted(map(lambda x: x["name"], self.flavors))
155
        self.assertEqual(sorted(list(set(names))), names)
156

  
157
    def test_005_well_formed_flavor_names(self):
158
        """Test flavors have names of the form CxxRyyDzz
159

  
160
        Where xx is vCPU count, yy is RAM in MiB, zz is Disk in GiB
161

  
162
        """
163
        for f in self.dflavors:
164
            self.assertEqual("C%dR%dD%d" % (f["cpu"], f["ram"], f["disk"]),
165
                             f["name"],
166
                             "Flavor %s does not match its specs." % f["name"])
167

  
168

  
169
class ServersTestCase(unittest.TestCase):
170
    """Test server lists for consistency"""
171
    @classmethod
172
    def setUpClass(cls):
173
        """Initialize kamaki, get (detailed) list of servers"""
174
        log.info("Getting simple and detailed list of servers")
175
        cls.client = Client(API, TOKEN)
176
        cls.servers = cls.client.list_servers()
177
        cls.dservers = cls.client.list_servers(detail=True)
178

  
179
    def test_001_list_servers(self):
180
        """Test server list actually returns servers"""
181
        self.assertGreater(len(self.servers), 0)
182

  
183
    def test_002_list_servers_detailed(self):
184
        """Test detailed server list is the same length as list"""
185
        self.assertEqual(len(self.dservers), len(self.servers))
186

  
187
    def test_003_same_server_names(self):
188
        """Test detailed and simple flavor list contain same names"""
189
        names = sorted(map(lambda x: x["name"], self.servers))
190
        dnames = sorted(map(lambda x: x["name"], self.dservers))
191
        self.assertEqual(names, dnames)
192

  
193

  
194
# This class gets replicated into actual TestCases dynamically
195
class SpawnServerTestCase(unittest.TestCase):
196
    """Test scenario for server of the specified image"""
197

  
198
    @classmethod
199
    def setUpClass(cls):
200
        """Initialize a kamaki instance"""
201
        log.info("Spawning server for image `%s'", cls.imagename)
202
        cls.client = Client(API, TOKEN)
203

  
204
    def _get_ipv4(self, server):
205
        """Get the public IPv4 of a server from the detailed server info"""
206
        public_addrs = filter(lambda x: x["id"] == "public",
207
                              server["addresses"]["values"])
208
        self.assertEqual(len(public_addrs), 1)
209
        ipv4_addrs = filter(lambda x: x["version"] == 4,
210
                            public_addrs[0]["values"])
211
        self.assertEqual(len(ipv4_addrs), 1)
212
        return ipv4_addrs[0]["addr"]
213

  
214
    def _get_ipv6(self, server):
215
        """Get the public IPv6 of a server from the detailed server info"""
216
        public_addrs = filter(lambda x: x["id"] == "public",
217
                              server["addresses"]["values"])
218
        self.assertEqual(len(public_addrs), 1)
219
        ipv6_addrs = filter(lambda x: x["version"] == 6,
220
                            public_addrs[0]["values"])
221
        self.assertEqual(len(ipv6_addrs), 1)
222
        return ipv6_addrs[0]["addr"]
223

  
224
    def _connect_loginname(self, os):
225
        """Return the login name for connections based on the server OS"""
226
        if os in ("ubuntu", "kubuntu", "fedora"):
227
            return "user"
228
        elif os == "windows":
229
            return "Administrator"
230
        else:
231
            return "root"
232

  
233
    def _verify_server_status(self, current_status, new_status):
234
        """Verify a server has switched to a specified status"""
235
        server = self.client.get_server_details(self.serverid)
236
        if server["status"] not in (current_status, new_status):
237
            return None  # Do not raise exception, return so the test fails
238
        self.assertEquals(server["status"], new_status)
239

  
240
    def _get_connected_tcp_socket(self, family, host, port):
241
        """Get a connected socket from the specified family to host:port"""
242
        sock = None
243
        for res in \
244
            socket.getaddrinfo(host, port, family, socket.SOCK_STREAM, 0,
245
                               socket.AI_PASSIVE):
246
            af, socktype, proto, canonname, sa = res
247
            try:
248
                sock = socket.socket(af, socktype, proto)
249
            except socket.error as msg:
250
                sock = None
251
                continue
252
            try:
253
                sock.connect(sa)
254
            except socket.error as msg:
255
                sock.close()
256
                sock = None
257
                continue
258
        self.assertIsNotNone(sock)
259
        return sock
260

  
261
    def _ping_once(self, ipv6, ip):
262
        """Test server responds to a single IPv4 or IPv6 ping"""
263
        cmd = "ping%s -c 2 -w 3 %s" % ("6" if ipv6 else "", ip)
264
        ping = subprocess.Popen(cmd, shell=True,
265
                                stdout=subprocess.PIPE, stderr=subprocess.PIPE)
266
        (stdout, stderr) = ping.communicate()
267
        ret = ping.wait()
268
        self.assertEquals(ret, 0)
269

  
270
    def _get_hostname_over_ssh(self, hostip, username, password):
271
        ssh = paramiko.SSHClient()
272
        ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
273
        try:
274
            ssh.connect(hostip, username=username, password=password)
275
        except socket.error:
276
            raise AssertionError
277
        stdin, stdout, stderr = ssh.exec_command("hostname")
278
        lines = stdout.readlines()
279
        self.assertEqual(len(lines), 1)
280
        return lines[0]
281

  
282
    def _try_until_timeout_expires(self, warn_timeout, fail_timeout,
283
                                   opmsg, callable, *args, **kwargs):
284
        if warn_timeout == fail_timeout:
285
            warn_timeout = fail_timeout + 1
286
        warn_tmout = time.time() + warn_timeout
287
        fail_tmout = time.time() + fail_timeout
288
        while True:
289
            self.assertLess(time.time(), fail_tmout,
290
                            "operation `%s' timed out" % opmsg)
291
            if time.time() > warn_tmout:
292
                log.warning("Server %d: `%s' operation `%s' not done yet",
293
                            self.serverid, self.servername, opmsg)
294
            try:
295
                log.info("%s... " % opmsg)
296
                return callable(*args, **kwargs)
297
            except AssertionError:
298
                pass
299
            time.sleep(self.query_interval)
300

  
301
    def _insist_on_tcp_connection(self, family, host, port):
302
        familystr = {socket.AF_INET: "IPv4", socket.AF_INET6: "IPv6",
303
                     socket.AF_UNSPEC: "Unspecified-IPv4/6"}
304
        msg = "connect over %s to %s:%s" % \
305
              (familystr.get(family, "Unknown"), host, port)
306
        sock = self._try_until_timeout_expires(
307
                self.action_timeout, self.action_timeout,
308
                msg, self._get_connected_tcp_socket,
309
                family, host, port)
310
        return sock
311

  
312
    def _insist_on_status_transition(self, current_status, new_status,
313
                                    fail_timeout, warn_timeout=None):
314
        msg = "Server %d: `%s', waiting for %s -> %s" % \
315
              (self.serverid, self.servername, current_status, new_status)
316
        if warn_timeout is None:
317
            warn_timeout = fail_timeout
318
        self._try_until_timeout_expires(warn_timeout, fail_timeout,
319
                                        msg, self._verify_server_status,
320
                                        current_status, new_status)
321
        # Ensure the status is actually the expected one
322
        server = self.client.get_server_details(self.serverid)
323
        self.assertEquals(server["status"], new_status)
324

  
325
    def _insist_on_ssh_hostname(self, hostip, username, password):
326
        msg = "SSH to %s, as %s/%s" % (hostip, username, password)
327
        hostname = self._try_until_timeout_expires(
328
                self.action_timeout, self.action_timeout,
329
                msg, self._get_hostname_over_ssh,
330
                hostip, username, password)
331

  
332
        # The hostname must be of the form 'prefix-id'
333
        self.assertTrue(hostname.endswith("-%d\n" % self.serverid))
334

  
335
    def _skipIf(self, condition, msg):
336
        if condition:
337
            self.skipTest(msg)
338

  
339
    def test_001_submit_create_server(self):
340
        """Test submit create server request"""
341
        server = self.client.create_server(self.servername, self.flavorid,
342
                                           self.imageid, self.personality)
343
        self.assertEqual(server["name"], self.servername)
344
        self.assertEqual(server["flavorRef"], self.flavorid)
345
        self.assertEqual(server["imageRef"], self.imageid)
346
        self.assertEqual(server["status"], "BUILD")
347

  
348
        # Update class attributes to reflect data on building server
349
        cls = type(self)
350
        cls.serverid = server["id"]
351
        cls.username = None
352
        cls.passwd = server["adminPass"]
353

  
354
    def test_002a_server_is_building_in_list(self):
355
        """Test server is in BUILD state, in server list"""
356
        servers = self.client.list_servers(detail=True)
357
        servers = filter(lambda x: x["name"] == self.servername, servers)
358
        self.assertEqual(len(servers), 1)
359
        server = servers[0]
360
        self.assertEqual(server["name"], self.servername)
361
        self.assertEqual(server["flavorRef"], self.flavorid)
362
        self.assertEqual(server["imageRef"], self.imageid)
363
        self.assertEqual(server["status"], "BUILD")
364

  
365
    def test_002b_server_is_building_in_details(self):
366
        """Test server is in BUILD state, in details"""
367
        server = self.client.get_server_details(self.serverid)
368
        self.assertEqual(server["name"], self.servername)
369
        self.assertEqual(server["flavorRef"], self.flavorid)
370
        self.assertEqual(server["imageRef"], self.imageid)
371
        self.assertEqual(server["status"], "BUILD")
372

  
373
    def test_002c_set_server_metadata(self):
374
        image = self.client.get_image_details(self.imageid)
375
        os = image["metadata"]["values"]["OS"]
376
        loginname = image["metadata"]["values"].get("loginname", None)
377
        self.client.update_server_metadata(self.serverid, OS=os)
378

  
379
        # Determine the username to use for future connections
380
        # to this host
381
        cls = type(self)
382
        cls.username = loginname
383
        if not cls.username:
384
            cls.username = self._connect_loginname(os)
385
        self.assertIsNotNone(cls.username)
386

  
387
    def test_002d_verify_server_metadata(self):
388
        """Test server metadata keys are set based on image metadata"""
389
        servermeta = self.client.get_server_metadata(self.serverid)
390
        imagemeta = self.client.get_image_metadata(self.imageid)
391
        self.assertEqual(servermeta["OS"], imagemeta["OS"])
392

  
393
    def test_003_server_becomes_active(self):
394
        """Test server becomes ACTIVE"""
395
        self._insist_on_status_transition("BUILD", "ACTIVE",
396
                                         self.build_fail, self.build_warning)
397

  
398
    def test_003a_get_server_oob_console(self):
399
        """Test getting OOB server console over VNC
400

  
401
        Implementation of RFB protocol follows
402
        http://www.realvnc.com/docs/rfbproto.pdf.
403

  
404
        """
405
        console = self.client.get_server_console(self.serverid)
406
        self.assertEquals(console['type'], "vnc")
407
        sock = self._insist_on_tcp_connection(socket.AF_UNSPEC,
408
                                        console["host"], console["port"])
409

  
410
        # Step 1. ProtocolVersion message (par. 6.1.1)
411
        version = sock.recv(1024)
412
        self.assertEquals(version, 'RFB 003.008\n')
413
        sock.send(version)
414

  
415
        # Step 2. Security (par 6.1.2): Only VNC Authentication supported
416
        sec = sock.recv(1024)
417
        self.assertEquals(list(sec), ['\x01', '\x02'])
418

  
419
        # Step 3. Request VNC Authentication (par 6.1.2)
420
        sock.send('\x02')
421

  
422
        # Step 4. Receive Challenge (par 6.2.2)
423
        challenge = sock.recv(1024)
424
        self.assertEquals(len(challenge), 16)
425

  
426
        # Step 5. DES-Encrypt challenge, use password as key (par 6.2.2)
427
        response = d3des_generate_response(
428
            (console["password"] + '\0' * 8)[:8], challenge)
429
        sock.send(response)
430

  
431
        # Step 6. SecurityResult (par 6.1.3)
432
        result = sock.recv(4)
433
        self.assertEquals(list(result), ['\x00', '\x00', '\x00', '\x00'])
434
        sock.close()
435

  
436
    def test_004_server_has_ipv4(self):
437
        """Test active server has a valid IPv4 address"""
438
        server = self.client.get_server_details(self.serverid)
439
        ipv4 = self._get_ipv4(server)
440
        self.assertEquals(IP(ipv4).version(), 4)
441

  
442
    def test_005_server_has_ipv6(self):
443
        """Test active server has a valid IPv6 address"""
444
        server = self.client.get_server_details(self.serverid)
445
        ipv6 = self._get_ipv6(server)
446
        self.assertEquals(IP(ipv6).version(), 6)
447

  
448
    def test_006_server_responds_to_ping_IPv4(self):
449
        """Test server responds to ping on IPv4 address"""
450
        server = self.client.get_server_details(self.serverid)
451
        ip = self._get_ipv4(server)
452
        self._try_until_timeout_expires(self.action_timeout,
453
                                        self.action_timeout,
454
                                        "PING IPv4 to %s" % ip,
455
                                        self._ping_once,
456
                                        False, ip)
457

  
458
    def test_007_server_responds_to_ping_IPv6(self):
459
        """Test server responds to ping on IPv6 address"""
460
        server = self.client.get_server_details(self.serverid)
461
        ip = self._get_ipv6(server)
462
        self._try_until_timeout_expires(self.action_timeout,
463
                                        self.action_timeout,
464
                                        "PING IPv6 to %s" % ip,
465
                                        self._ping_once,
466
                                        True, ip)
467

  
468
    def test_008_submit_shutdown_request(self):
469
        """Test submit request to shutdown server"""
470
        self.client.shutdown_server(self.serverid)
471

  
472
    def test_009_server_becomes_stopped(self):
473
        """Test server becomes STOPPED"""
474
        self._insist_on_status_transition("ACTIVE", "STOPPED",
475
                                         self.action_timeout,
476
                                         self.action_timeout)
477

  
478
    def test_010_submit_start_request(self):
479
        """Test submit start server request"""
480
        self.client.start_server(self.serverid)
481

  
482
    def test_011_server_becomes_active(self):
483
        """Test server becomes ACTIVE again"""
484
        self._insist_on_status_transition("STOPPED", "ACTIVE",
485
                                         self.action_timeout,
486
                                         self.action_timeout)
487

  
488
    def test_011a_server_responds_to_ping_IPv4(self):
489
        """Test server OS is actually up and running again"""
490
        self.test_006_server_responds_to_ping_IPv4()
491

  
492
    def test_012_ssh_to_server_IPv4(self):
493
        """Test SSH to server public IPv4 works, verify hostname"""
494
        self._skipIf(self.is_windows, "only valid for Linux servers")
495
        server = self.client.get_server_details(self.serverid)
496
        self._insist_on_ssh_hostname(self._get_ipv4(server),
497
                                     self.username, self.passwd)
498

  
499
    def test_013_ssh_to_server_IPv6(self):
500
        """Test SSH to server public IPv6 works, verify hostname"""
501
        self._skipIf(self.is_windows, "only valid for Linux servers")
502
        server = self.client.get_server_details(self.serverid)
503
        self._insist_on_ssh_hostname(self._get_ipv6(server),
504
                                     self.username, self.passwd)
505

  
506
    def test_014_rdp_to_server_IPv4(self):
507
        "Test RDP connection to server public IPv4 works"""
508
        self._skipIf(not self.is_windows, "only valid for Windows servers")
509
        server = self.client.get_server_details(self.serverid)
510
        ipv4 = self._get_ipv4(server)
511
        sock = _insist_on_tcp_connection(socket.AF_INET, ipv4, 3389)
512

  
513
        # No actual RDP processing done. We assume the RDP server is there
514
        # if the connection to the RDP port is successful.
515
        # FIXME: Use rdesktop, analyze exit code? see manpage [costasd]
516
        sock.close()
517

  
518
    def test_015_rdp_to_server_IPv6(self):
519
        "Test RDP connection to server public IPv6 works"""
520
        self._skipIf(not self.is_windows, "only valid for Windows servers")
521
        server = self.client.get_server_details(self.serverid)
522
        ipv6 = self._get_ipv6(server)
523
        sock = _get_tcp_connection(socket.AF_INET6, ipv6, 3389)
524

  
525
        # No actual RDP processing done. We assume the RDP server is there
526
        # if the connection to the RDP port is successful.
527
        sock.close()
528

  
529
    def test_016_personality_is_enforced(self):
530
        """Test file injection for personality enforcement"""
531
        self._skipIf(self.is_windows, "only implemented for Linux servers")
532
        self.assertTrue(False, "test not implemented, will fail")
533

  
534
    def test_017_submit_delete_request(self):
535
        """Test submit request to delete server"""
536
        self.client.delete_server(self.serverid)
537

  
538
    def test_018_server_becomes_deleted(self):
539
        """Test server becomes DELETED"""
540
        self._insist_on_status_transition("ACTIVE", "DELETED",
541
                                         self.action_timeout,
542
                                         self.action_timeout)
543

  
544
    def test_019_server_no_longer_in_server_list(self):
545
        """Test server is no longer in server list"""
546
        servers = self.client.list_servers()
547
        self.assertNotIn(self.serverid, [s["id"] for s in servers])
548

  
549

  
550
class TestRunnerProcess(Process):
551
    """A distinct process used to execute part of the tests in parallel"""
552
    def __init__(self, **kw):
553
        Process.__init__(self, **kw)
554
        kwargs = kw["kwargs"]
555
        self.testq = kwargs["testq"]
556
        self.runner = kwargs["runner"]
557

  
558
    def run(self):
559
        # Make sure this test runner process dies with the parent
560
        # and is not left behind.
561
        #
562
        # WARNING: This uses the prctl(2) call and is
563
        # Linux-specific.
564
        prctl.set_pdeathsig(signal.SIGHUP)
565

  
566
        while True:
567
            log.debug("I am process %d, GETting from queue is %s",
568
                     os.getpid(), self.testq)
569
            msg = self.testq.get()
570
            log.debug("Dequeued msg: %s", msg)
571

  
572
            if msg == "TEST_RUNNER_TERMINATE":
573
                raise SystemExit
574
            elif issubclass(msg, unittest.TestCase):
575
                # Assemble a TestSuite, and run it
576
                suite = unittest.TestLoader().loadTestsFromTestCase(msg)
577
                self.runner.run(suite)
578
            else:
579
                raise Exception("Cannot handle msg: %s" % msg)
580

  
581

  
582
def _run_cases_in_parallel(cases, fanout=1, runner=None):
583
    """Run instances of TestCase in parallel, in a number of distinct processes
584

  
585
    The cases iterable specifies the TestCases to be executed in parallel,
586
    by test runners running in distinct processes.
587
    The fanout parameter specifies the number of processes to spawn,
588
    and defaults to 1.
589
    The runner argument specifies the test runner class to use inside each
590
    runner process.
591

  
592
    """
593
    if runner is None:
594
        runner = unittest.TextTestRunner(verbosity=2, failfast=True)
595

  
596
    # testq: The master process enqueues TestCase objects into this queue,
597
    #        test runner processes pick them up for execution, in parallel.
598
    testq = Queue()
599
    runners = []
600
    for i in xrange(0, fanout):
601
        kwargs = dict(testq=testq, runner=runner)
602
        runners.append(TestRunnerProcess(kwargs=kwargs))
603

  
604
    log.info("Spawning %d test runner processes", len(runners))
605
    for p in runners:
606
        p.start()
607
    log.debug("Spawned %d test runners, PIDs are %s",
608
              len(runners), [p.pid for p in runners])
609

  
610
    # Enqueue test cases
611
    map(testq.put, cases)
612
    map(testq.put, ["TEST_RUNNER_TERMINATE"] * len(runners))
613

  
614
    log.debug("Joining %d processes", len(runners))
615
    for p in runners:
616
        p.join()
617
    log.debug("Done joining %d processes", len(runners))
618

  
619

  
620
def _spawn_server_test_case(**kwargs):
621
    """Construct a new unit test case class from SpawnServerTestCase"""
622

  
623
    name = "SpawnServerTestCase_%d" % kwargs["imageid"]
624
    cls = type(name, (SpawnServerTestCase,), kwargs)
625

  
626
    # Patch extra parameters into test names by manipulating method docstrings
627
    for (mname, m) in \
628
        inspect.getmembers(cls, lambda x: inspect.ismethod(x)):
629
            if hasattr(m, __doc__):
630
                m.__func__.__doc__ = "[%s] %s" % (imagename, m.__doc__)
631

  
632
    # Make sure the class can be pickled, by listing it among
633
    # the attributes of __main__. A PicklingError is raised otherwise.
634
    setattr(__main__, name, cls)
635
    return cls
636

  
637

  
638
def cleanup_servers(delete_stale=False):
639
    c = Client(API, TOKEN)
640
    servers = c.list_servers()
641
    stale = [s for s in servers if s["name"].startswith(SNF_TEST_PREFIX)]
642

  
643
    if len(stale) == 0:
644
        return
645

  
646
    print >> sys.stderr, "Found these stale servers from previous runs:"
647
    print "    " + \
648
          "\n    ".join(["%d: %s" % (s["id"], s["name"]) for s in stale])
649

  
650
    if delete_stale:
651
        print >> sys.stderr, "Deleting %d stale servers:" % len(stale)
652
        for server in stale:
653
            c.delete_server(server["id"])
654
        print >> sys.stderr, "    ...done"
655
    else:
656
        print >> sys.stderr, "Use --delete-stale to delete them."
657

  
658

  
659
def parse_arguments(args):
660
    from optparse import OptionParser
661

  
662
    kw = {}
663
    kw["usage"] = "%prog [options]"
664
    kw["description"] = \
665
        "%prog runs a number of test scenarios on a " \
666
        "Synnefo deployment."
667

  
668
    parser = OptionParser(**kw)
669
    parser.disable_interspersed_args()
670
    parser.add_option("--api",
671
                      action="store", type="string", dest="api",
672
                      help="The API URI to use to reach the Synnefo API",
673
                      default=DEFAULT_API)
674
    parser.add_option("--token",
675
                      action="store", type="string", dest="token",
676
                      help="The token to use for authentication to the API",
677
                      default=DEFAULT_TOKEN)
678
    parser.add_option("--nofailfast",
679
                      action="store_true", dest="nofailfast",
680
                      help="Do not fail immediately if one of the tests " \
681
                           "fails (EXPERIMENTAL)",
682
                      default=False)
683
    parser.add_option("--action-timeout",
684
                      action="store", type="int", dest="action_timeout",
685
                      metavar="TIMEOUT",
686
                      help="Wait SECONDS seconds for a server action to " \
687
                           "complete, then the test is considered failed",
688
                      default=20)
689
    parser.add_option("--build-warning",
690
                      action="store", type="int", dest="build_warning",
691
                      metavar="TIMEOUT",
692
                      help="Warn if TIMEOUT seconds have passed and a " \
693
                           "build operation is still pending",
694
                      default=600)
695
    parser.add_option("--build-fail",
696
                      action="store", type="int", dest="build_fail",
697
                      metavar="BUILD_TIMEOUT",
698
                      help="Fail the test if TIMEOUT seconds have passed " \
699
                           "and a build operation is still incomplete",
700
                      default=900)
701
    parser.add_option("--query-interval",
702
                      action="store", type="int", dest="query_interval",
703
                      metavar="INTERVAL",
704
                      help="Query server status when requests are pending " \
705
                           "every INTERVAL seconds",
706
                      default=3)
707
    parser.add_option("--fanout",
708
                      action="store", type="int", dest="fanout",
709
                      metavar="COUNT",
710
                      help="Spawn up to COUNT child processes to execute " \
711
                           "in parallel, essentially have up to COUNT " \
712
                           "server build requests outstanding (EXPERIMENTAL)",
713
                      default=1)
714
    parser.add_option("--force-flavor",
715
                      action="store", type="int", dest="force_flavorid",
716
                      metavar="FLAVOR ID",
717
                      help="Force all server creations to use the specified "\
718
                           "FLAVOR ID instead of a randomly chosen one, " \
719
                           "useful if disk space is scarce",
720
                      default=None)
721
    parser.add_option("--image-id",
722
                      action="store", type="string", dest="force_imageid",
723
                      metavar="IMAGE ID",
724
                      help="Test the specified image id, use 'all' to test " \
725
                           "all available images (mandatory argument)",
726
                      default=None)
727
    parser.add_option("--show-stale",
728
                      action="store_true", dest="show_stale",
729
                      help="Show stale servers from previous runs, whose "\
730
                           "name starts with `%s'" % SNF_TEST_PREFIX,
731
                      default=False)
732
    parser.add_option("--delete-stale",
733
                      action="store_true", dest="delete_stale",
734
                      help="Delete stale servers from previous runs, whose "\
735
                           "name starts with `%s'" % SNF_TEST_PREFIX,
736
                      default=False)
737

  
738
    # FIXME: Change the default for build-fanout to 10
739
    # FIXME: Allow the user to specify a specific set of Images to test
740

  
741
    (opts, args) = parser.parse_args(args)
742

  
743
    # Verify arguments
744
    if opts.delete_stale:
745
        opts.show_stale = True
746

  
747
    if not opts.show_stale:
748
        if not opts.force_imageid:
749
            print >>sys.stderr, "The --image-id argument is mandatory."
750
            parser.print_help()
751
            sys.exit(1)
752

  
753
        if opts.force_imageid != 'all':
754
            try:
755
                opts.force_imageid = int(opts.force_imageid)
756
            except ValueError:
757
                print >>sys.stderr, "Invalid value specified for --image-id." \
758
                                    "Use a numeric id, or `all'."
759
                sys.exit(1)
760

  
761
    return (opts, args)
762

  
763

  
764
def main():
765
    """Assemble test cases into a test suite, and run it
766

  
767
    IMPORTANT: Tests have dependencies and have to be run in the specified
768
    order inside a single test case. They communicate through attributes of the
769
    corresponding TestCase class (shared fixtures). Distinct subclasses of
770
    TestCase MAY SHARE NO DATA, since they are run in parallel, in distinct
771
    test runner processes.
772

  
773
    """
774
    (opts, args) = parse_arguments(sys.argv[1:])
775

  
776
    global API, TOKEN
777
    API = opts.api
778
    TOKEN = opts.token
779

  
780
    # Cleanup stale servers from previous runs
781
    if opts.show_stale:
782
        cleanup_servers(delete_stale=opts.delete_stale)
783
        return 0
784

  
785
    # Initialize a kamaki instance, get flavors, images
786
    c = Client(API, TOKEN)
787
    DIMAGES = c.list_images(detail=True)
788
    DFLAVORS = c.list_flavors(detail=True)
789

  
790
    # FIXME: logging, log, LOG PID, TEST_RUN_ID, arguments
791
    # FIXME: Network testing? Create, destroy, connect, ping, disconnect VMs?
792
    # Run them: FIXME: In parallel, FAILEARLY, catchbreak?
793
    #unittest.main(verbosity=2, catchbreak=True)
794

  
795
    runner = unittest.TextTestRunner(verbosity=2, failfast=not opts.nofailfast)
796
    # The following cases run sequentially
797
    seq_cases = [UnauthorizedTestCase, FlavorsTestCase, ImagesTestCase]
798
    _run_cases_in_parallel(seq_cases, fanout=3, runner=runner)
799

  
800
    # The following cases run in parallel
801
    par_cases = []
802

  
803
    if opts.force_imageid == 'all':
804
        test_images = DIMAGES
805
    else:
806
        test_images = filter(lambda x: x["id"] == opts.force_imageid, DIMAGES)
807

  
808
    for image in test_images:
809
        imageid = image["id"]
810
        imagename = image["name"]
811
        if opts.force_flavorid:
812
            flavorid = opts.force_flavorid
813
        else:
814
            flavorid = choice([f["id"] for f in DFLAVORS if f["disk"] >= 20])
815
        personality = None   # FIXME
816
        servername = "%s%s for %s" % (SNF_TEST_PREFIX, TEST_RUN_ID, imagename)
817
        is_windows = imagename.lower().find("windows") >= 0
818
        case = _spawn_server_test_case(imageid=imageid, flavorid=flavorid,
819
                                       imagename=imagename,
820
                                       personality=personality,
821
                                       servername=servername,
822
                                       is_windows=is_windows,
823
                                       action_timeout=opts.action_timeout,
824
                                       build_warning=opts.build_warning,
825
                                       build_fail=opts.build_fail,
826
                                       query_interval=opts.query_interval)
827
        par_cases.append(case)
828

  
829
    _run_cases_in_parallel(par_cases, fanout=opts.fanout, runner=runner)
830

  
831
if __name__ == "__main__":
832
    sys.exit(main())
b/snf-tools/snf-burnin
1
#!/usr/bin/env python
2

  
3
# Copyright 2011 GRNET S.A. All rights reserved.
4
#
5
# Redistribution and use in source and binary forms, with or
6
# without modification, are permitted provided that the following
7
# conditions are met:
8
#
9
#   1. Redistributions of source code must retain the above
10
#      copyright notice, this list of conditions and the following
11
#      disclaimer.
12
#
13
#   2. Redistributions in binary form must reproduce the above
14
#      copyright notice, this list of conditions and the following
15
#      disclaimer in the documentation and/or other materials
16
#      provided with the distribution.
17
#
18
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
19
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
20
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
21
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
22
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
25
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
26
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
28
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29
# POSSIBILITY OF SUCH DAMAGE.
30
#
31
# The views and conclusions contained in the software and
32
# documentation are those of the authors and should not be
33
# interpreted as representing official policies, either expressed
34
# or implied, of GRNET S.A.
35

  
36
"""Perform integration testing on a running Synnefo deployment"""
37

  
38
import __main__
39
import datetime
40
import inspect
41
import logging
42
import os
43
import paramiko
44
import prctl
45
import subprocess
46
import signal
47
import socket
48
import struct
49
import sys
50
import time
51

  
52
from IPy import IP
53
from multiprocessing import Process, Queue
54
from random import choice
55

  
56
from kamaki.client import Client, ClientError
57
from vncauthproxy.d3des import generate_response as d3des_generate_response
58

  
59
# Use backported unittest functionality if Python < 2.7
60
try:
61
    import unittest2 as unittest
62
except ImportError:
63
    if sys.version_info < (2, 7):
64
        raise Exception("The unittest2 package is required for Python < 2.7")
65
    import unittest
66

  
67

  
68
API = None
69
TOKEN = None
70
DEFAULT_API = "http://dev67.dev.grnet.gr:8000/api/v1.1"
71
DEFAULT_TOKEN = "46e427d657b20defe352804f0eb6f8a2"
72
# A unique id identifying this test run
73
TEST_RUN_ID = datetime.datetime.strftime(datetime.datetime.now(),
74
                                         "%Y%m%d%H%M%S")
75
SNF_TEST_PREFIX = "snf-test-"
76

  
77
# Setup logging (FIXME - verigak)
78
logging.basicConfig(format="%(message)s")
79
log = logging.getLogger("burnin")
80
log.setLevel(logging.INFO)
81

  
82

  
83
class UnauthorizedTestCase(unittest.TestCase):
84
    def test_unauthorized_access(self):
85
        """Test access without a valid token fails"""
86
        c = Client(API, "123")
87
        with self.assertRaises(ClientError) as cm:
88
            c.list_servers()
89
        self.assertEqual(cm.exception.status, 401)
90

  
91

  
92
class ImagesTestCase(unittest.TestCase):
93
    """Test image lists for consistency"""
94
    @classmethod
95
    def setUpClass(cls):
96
        """Initialize kamaki, get (detailed) list of images"""
97
        log.info("Getting simple and detailed list of images")
98
        cls.client = Client(API, TOKEN)
99
        cls.images = cls.client.list_images()
100
        cls.dimages = cls.client.list_images(detail=True)
101

  
102
    def test_001_list_images(self):
103
        """Test image list actually returns images"""
104
        self.assertGreater(len(self.images), 0)
105

  
106
    def test_002_list_images_detailed(self):
107
        """Test detailed image list is the same length as list"""
108
        self.assertEqual(len(self.dimages), len(self.images))
109

  
110
    def test_003_same_image_names(self):
111
        """Test detailed and simple image list contain same names"""
112
        names = sorted(map(lambda x: x["name"], self.images))
113
        dnames = sorted(map(lambda x: x["name"], self.dimages))
114
        self.assertEqual(names, dnames)
115

  
116
    def test_004_unique_image_names(self):
117
        """Test images have unique names"""
118
        names = sorted(map(lambda x: x["name"], self.images))
119
        self.assertEqual(sorted(list(set(names))), names)
120

  
121
    def test_005_image_metadata(self):
122
        """Test every image has specific metadata defined"""
123
        keys = frozenset(["OS", "description", "size"])
124
        for i in self.dimages:
125
            self.assertTrue(keys.issubset(i["metadata"]["values"].keys()))
126

  
127

  
128
class FlavorsTestCase(unittest.TestCase):
129
    """Test flavor lists for consistency"""
130
    @classmethod
131
    def setUpClass(cls):
132
        """Initialize kamaki, get (detailed) list of flavors"""
133
        log.info("Getting simple and detailed list of flavors")
134
        cls.client = Client(API, TOKEN)
135
        cls.flavors = cls.client.list_flavors()
136
        cls.dflavors = cls.client.list_flavors(detail=True)
137

  
138
    def test_001_list_flavors(self):
139
        """Test flavor list actually returns flavors"""
140
        self.assertGreater(len(self.flavors), 0)
141

  
142
    def test_002_list_flavors_detailed(self):
143
        """Test detailed flavor list is the same length as list"""
144
        self.assertEquals(len(self.dflavors), len(self.flavors))
145

  
146
    def test_003_same_flavor_names(self):
147
        """Test detailed and simple flavor list contain same names"""
148
        names = sorted(map(lambda x: x["name"], self.flavors))
149
        dnames = sorted(map(lambda x: x["name"], self.dflavors))
150
        self.assertEqual(names, dnames)
151

  
152
    def test_004_unique_flavor_names(self):
153
        """Test flavors have unique names"""
154
        names = sorted(map(lambda x: x["name"], self.flavors))
155
        self.assertEqual(sorted(list(set(names))), names)
156

  
157
    def test_005_well_formed_flavor_names(self):
158
        """Test flavors have names of the form CxxRyyDzz
159

  
160
        Where xx is vCPU count, yy is RAM in MiB, zz is Disk in GiB
161

  
162
        """
163
        for f in self.dflavors:
164
            self.assertEqual("C%dR%dD%d" % (f["cpu"], f["ram"], f["disk"]),
165
                             f["name"],
166
                             "Flavor %s does not match its specs." % f["name"])
167

  
168

  
169
class ServersTestCase(unittest.TestCase):
170
    """Test server lists for consistency"""
171
    @classmethod
172
    def setUpClass(cls):
173
        """Initialize kamaki, get (detailed) list of servers"""
174
        log.info("Getting simple and detailed list of servers")
175
        cls.client = Client(API, TOKEN)
176
        cls.servers = cls.client.list_servers()
177
        cls.dservers = cls.client.list_servers(detail=True)
178

  
179
    def test_001_list_servers(self):
180
        """Test server list actually returns servers"""
181
        self.assertGreater(len(self.servers), 0)
182

  
183
    def test_002_list_servers_detailed(self):
184
        """Test detailed server list is the same length as list"""
185
        self.assertEqual(len(self.dservers), len(self.servers))
186

  
187
    def test_003_same_server_names(self):
188
        """Test detailed and simple flavor list contain same names"""
189
        names = sorted(map(lambda x: x["name"], self.servers))
190
        dnames = sorted(map(lambda x: x["name"], self.dservers))
191
        self.assertEqual(names, dnames)
192

  
193

  
194
# This class gets replicated into actual TestCases dynamically
195
class SpawnServerTestCase(unittest.TestCase):
196
    """Test scenario for server of the specified image"""
197

  
198
    @classmethod
199
    def setUpClass(cls):
200
        """Initialize a kamaki instance"""
201
        log.info("Spawning server for image `%s'", cls.imagename)
202
        cls.client = Client(API, TOKEN)
203

  
204
    def _get_ipv4(self, server):
205
        """Get the public IPv4 of a server from the detailed server info"""
206
        public_addrs = filter(lambda x: x["id"] == "public",
207
                              server["addresses"]["values"])
208
        self.assertEqual(len(public_addrs), 1)
209
        ipv4_addrs = filter(lambda x: x["version"] == 4,
210
                            public_addrs[0]["values"])
211
        self.assertEqual(len(ipv4_addrs), 1)
212
        return ipv4_addrs[0]["addr"]
213

  
214
    def _get_ipv6(self, server):
215
        """Get the public IPv6 of a server from the detailed server info"""
216
        public_addrs = filter(lambda x: x["id"] == "public",
217
                              server["addresses"]["values"])
218
        self.assertEqual(len(public_addrs), 1)
219
        ipv6_addrs = filter(lambda x: x["version"] == 6,
220
                            public_addrs[0]["values"])
221
        self.assertEqual(len(ipv6_addrs), 1)
222
        return ipv6_addrs[0]["addr"]
223

  
224
    def _connect_loginname(self, os):
225
        """Return the login name for connections based on the server OS"""
226
        if os in ("ubuntu", "kubuntu", "fedora"):
227
            return "user"
228
        elif os == "windows":
229
            return "Administrator"
230
        else:
231
            return "root"
232

  
233
    def _verify_server_status(self, current_status, new_status):
234
        """Verify a server has switched to a specified status"""
235
        server = self.client.get_server_details(self.serverid)
236
        if server["status"] not in (current_status, new_status):
237
            return None  # Do not raise exception, return so the test fails
238
        self.assertEquals(server["status"], new_status)
239

  
240
    def _get_connected_tcp_socket(self, family, host, port):
241
        """Get a connected socket from the specified family to host:port"""
242
        sock = None
243
        for res in \
244
            socket.getaddrinfo(host, port, family, socket.SOCK_STREAM, 0,
245
                               socket.AI_PASSIVE):
246
            af, socktype, proto, canonname, sa = res
247
            try:
248
                sock = socket.socket(af, socktype, proto)
249
            except socket.error as msg:
250
                sock = None
251
                continue
252
            try:
253
                sock.connect(sa)
254
            except socket.error as msg:
255
                sock.close()
256
                sock = None
257
                continue
258
        self.assertIsNotNone(sock)
259
        return sock
260

  
261
    def _ping_once(self, ipv6, ip):
262
        """Test server responds to a single IPv4 or IPv6 ping"""
263
        cmd = "ping%s -c 2 -w 3 %s" % ("6" if ipv6 else "", ip)
264
        ping = subprocess.Popen(cmd, shell=True,
265
                                stdout=subprocess.PIPE, stderr=subprocess.PIPE)
266
        (stdout, stderr) = ping.communicate()
267
        ret = ping.wait()
268
        self.assertEquals(ret, 0)
269

  
270
    def _get_hostname_over_ssh(self, hostip, username, password):
271
        ssh = paramiko.SSHClient()
272
        ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
273
        try:
274
            ssh.connect(hostip, username=username, password=password)
275
        except socket.error:
276
            raise AssertionError
277
        stdin, stdout, stderr = ssh.exec_command("hostname")
278
        lines = stdout.readlines()
279
        self.assertEqual(len(lines), 1)
280
        return lines[0]
281

  
282
    def _try_until_timeout_expires(self, warn_timeout, fail_timeout,
283
                                   opmsg, callable, *args, **kwargs):
284
        if warn_timeout == fail_timeout:
285
            warn_timeout = fail_timeout + 1
286
        warn_tmout = time.time() + warn_timeout
287
        fail_tmout = time.time() + fail_timeout
288
        while True:
289
            self.assertLess(time.time(), fail_tmout,
290
                            "operation `%s' timed out" % opmsg)
291
            if time.time() > warn_tmout:
292
                log.warning("Server %d: `%s' operation `%s' not done yet",
293
                            self.serverid, self.servername, opmsg)
294
            try:
295
                log.info("%s... " % opmsg)
296
                return callable(*args, **kwargs)
297
            except AssertionError:
298
                pass
299
            time.sleep(self.query_interval)
300

  
301
    def _insist_on_tcp_connection(self, family, host, port):
302
        familystr = {socket.AF_INET: "IPv4", socket.AF_INET6: "IPv6",
303
                     socket.AF_UNSPEC: "Unspecified-IPv4/6"}
304
        msg = "connect over %s to %s:%s" % \
305
              (familystr.get(family, "Unknown"), host, port)
306
        sock = self._try_until_timeout_expires(
307
                self.action_timeout, self.action_timeout,
308
                msg, self._get_connected_tcp_socket,
309
                family, host, port)
310
        return sock
311

  
312
    def _insist_on_status_transition(self, current_status, new_status,
313
                                    fail_timeout, warn_timeout=None):
314
        msg = "Server %d: `%s', waiting for %s -> %s" % \
315
              (self.serverid, self.servername, current_status, new_status)
316
        if warn_timeout is None:
317
            warn_timeout = fail_timeout
318
        self._try_until_timeout_expires(warn_timeout, fail_timeout,
319
                                        msg, self._verify_server_status,
320
                                        current_status, new_status)
321
        # Ensure the status is actually the expected one
322
        server = self.client.get_server_details(self.serverid)
323
        self.assertEquals(server["status"], new_status)
324

  
325
    def _insist_on_ssh_hostname(self, hostip, username, password):
326
        msg = "SSH to %s, as %s/%s" % (hostip, username, password)
327
        hostname = self._try_until_timeout_expires(
328
                self.action_timeout, self.action_timeout,
329
                msg, self._get_hostname_over_ssh,
330
                hostip, username, password)
331

  
332
        # The hostname must be of the form 'prefix-id'
333
        self.assertTrue(hostname.endswith("-%d\n" % self.serverid))
334

  
335
    def _skipIf(self, condition, msg):
336
        if condition:
337
            self.skipTest(msg)
338

  
339
    def test_001_submit_create_server(self):
340
        """Test submit create server request"""
341
        server = self.client.create_server(self.servername, self.flavorid,
342
                                           self.imageid, self.personality)
343
        self.assertEqual(server["name"], self.servername)
344
        self.assertEqual(server["flavorRef"], self.flavorid)
345
        self.assertEqual(server["imageRef"], self.imageid)
346
        self.assertEqual(server["status"], "BUILD")
347

  
348
        # Update class attributes to reflect data on building server
349
        cls = type(self)
350
        cls.serverid = server["id"]
351
        cls.username = None
352
        cls.passwd = server["adminPass"]
353

  
354
    def test_002a_server_is_building_in_list(self):
355
        """Test server is in BUILD state, in server list"""
356
        servers = self.client.list_servers(detail=True)
357
        servers = filter(lambda x: x["name"] == self.servername, servers)
358
        self.assertEqual(len(servers), 1)
359
        server = servers[0]
360
        self.assertEqual(server["name"], self.servername)
361
        self.assertEqual(server["flavorRef"], self.flavorid)
362
        self.assertEqual(server["imageRef"], self.imageid)
363
        self.assertEqual(server["status"], "BUILD")
364

  
365
    def test_002b_server_is_building_in_details(self):
366
        """Test server is in BUILD state, in details"""
367
        server = self.client.get_server_details(self.serverid)
368
        self.assertEqual(server["name"], self.servername)
369
        self.assertEqual(server["flavorRef"], self.flavorid)
370
        self.assertEqual(server["imageRef"], self.imageid)
371
        self.assertEqual(server["status"], "BUILD")
372

  
373
    def test_002c_set_server_metadata(self):
374
        image = self.client.get_image_details(self.imageid)
375
        os = image["metadata"]["values"]["OS"]
376
        loginname = image["metadata"]["values"].get("loginname", None)
377
        self.client.update_server_metadata(self.serverid, OS=os)
378

  
379
        # Determine the username to use for future connections
380
        # to this host
381
        cls = type(self)
382
        cls.username = loginname
383
        if not cls.username:
384
            cls.username = self._connect_loginname(os)
385
        self.assertIsNotNone(cls.username)
386

  
387
    def test_002d_verify_server_metadata(self):
388
        """Test server metadata keys are set based on image metadata"""
389
        servermeta = self.client.get_server_metadata(self.serverid)
390
        imagemeta = self.client.get_image_metadata(self.imageid)
391
        self.assertEqual(servermeta["OS"], imagemeta["OS"])
392

  
393
    def test_003_server_becomes_active(self):
394
        """Test server becomes ACTIVE"""
395
        self._insist_on_status_transition("BUILD", "ACTIVE",
396
                                         self.build_fail, self.build_warning)
397

  
398
    def test_003a_get_server_oob_console(self):
399
        """Test getting OOB server console over VNC
400

  
401
        Implementation of RFB protocol follows
402
        http://www.realvnc.com/docs/rfbproto.pdf.
403

  
404
        """
405
        console = self.client.get_server_console(self.serverid)
406
        self.assertEquals(console['type'], "vnc")
407
        sock = self._insist_on_tcp_connection(socket.AF_UNSPEC,
408
                                        console["host"], console["port"])
409

  
410
        # Step 1. ProtocolVersion message (par. 6.1.1)
411
        version = sock.recv(1024)
412
        self.assertEquals(version, 'RFB 003.008\n')
413
        sock.send(version)
414

  
415
        # Step 2. Security (par 6.1.2): Only VNC Authentication supported
416
        sec = sock.recv(1024)
417
        self.assertEquals(list(sec), ['\x01', '\x02'])
418

  
419
        # Step 3. Request VNC Authentication (par 6.1.2)
420
        sock.send('\x02')
421

  
422
        # Step 4. Receive Challenge (par 6.2.2)
423
        challenge = sock.recv(1024)
424
        self.assertEquals(len(challenge), 16)
425

  
426
        # Step 5. DES-Encrypt challenge, use password as key (par 6.2.2)
427
        response = d3des_generate_response(
428
            (console["password"] + '\0' * 8)[:8], challenge)
429
        sock.send(response)
430

  
431
        # Step 6. SecurityResult (par 6.1.3)
432
        result = sock.recv(4)
433
        self.assertEquals(list(result), ['\x00', '\x00', '\x00', '\x00'])
434
        sock.close()
435

  
436
    def test_004_server_has_ipv4(self):
437
        """Test active server has a valid IPv4 address"""
438
        server = self.client.get_server_details(self.serverid)
439
        ipv4 = self._get_ipv4(server)
440
        self.assertEquals(IP(ipv4).version(), 4)
441

  
442
    def test_005_server_has_ipv6(self):
443
        """Test active server has a valid IPv6 address"""
444
        server = self.client.get_server_details(self.serverid)
445
        ipv6 = self._get_ipv6(server)
446
        self.assertEquals(IP(ipv6).version(), 6)
447

  
448
    def test_006_server_responds_to_ping_IPv4(self):
449
        """Test server responds to ping on IPv4 address"""
450
        server = self.client.get_server_details(self.serverid)
451
        ip = self._get_ipv4(server)
452
        self._try_until_timeout_expires(self.action_timeout,
453
                                        self.action_timeout,
454
                                        "PING IPv4 to %s" % ip,
455
                                        self._ping_once,
456
                                        False, ip)
457

  
458
    def test_007_server_responds_to_ping_IPv6(self):
459
        """Test server responds to ping on IPv6 address"""
460
        server = self.client.get_server_details(self.serverid)
461
        ip = self._get_ipv6(server)
462
        self._try_until_timeout_expires(self.action_timeout,
463
                                        self.action_timeout,
464
                                        "PING IPv6 to %s" % ip,
465
                                        self._ping_once,
466
                                        True, ip)
467

  
468
    def test_008_submit_shutdown_request(self):
469
        """Test submit request to shutdown server"""
470
        self.client.shutdown_server(self.serverid)
471

  
472
    def test_009_server_becomes_stopped(self):
473
        """Test server becomes STOPPED"""
474
        self._insist_on_status_transition("ACTIVE", "STOPPED",
475
                                         self.action_timeout,
476
                                         self.action_timeout)
477

  
478
    def test_010_submit_start_request(self):
479
        """Test submit start server request"""
480
        self.client.start_server(self.serverid)
481

  
482
    def test_011_server_becomes_active(self):
483
        """Test server becomes ACTIVE again"""
484
        self._insist_on_status_transition("STOPPED", "ACTIVE",
485
                                         self.action_timeout,
486
                                         self.action_timeout)
487

  
488
    def test_011a_server_responds_to_ping_IPv4(self):
489
        """Test server OS is actually up and running again"""
490
        self.test_006_server_responds_to_ping_IPv4()
491

  
492
    def test_012_ssh_to_server_IPv4(self):
493
        """Test SSH to server public IPv4 works, verify hostname"""
494
        self._skipIf(self.is_windows, "only valid for Linux servers")
495
        server = self.client.get_server_details(self.serverid)
496
        self._insist_on_ssh_hostname(self._get_ipv4(server),
497
                                     self.username, self.passwd)
498

  
499
    def test_013_ssh_to_server_IPv6(self):
500
        """Test SSH to server public IPv6 works, verify hostname"""
501
        self._skipIf(self.is_windows, "only valid for Linux servers")
502
        server = self.client.get_server_details(self.serverid)
503
        self._insist_on_ssh_hostname(self._get_ipv6(server),
504
                                     self.username, self.passwd)
505

  
506
    def test_014_rdp_to_server_IPv4(self):
507
        "Test RDP connection to server public IPv4 works"""
508
        self._skipIf(not self.is_windows, "only valid for Windows servers")
509
        server = self.client.get_server_details(self.serverid)
510
        ipv4 = self._get_ipv4(server)
511
        sock = _insist_on_tcp_connection(socket.AF_INET, ipv4, 3389)
512

  
513
        # No actual RDP processing done. We assume the RDP server is there
514
        # if the connection to the RDP port is successful.
515
        # FIXME: Use rdesktop, analyze exit code? see manpage [costasd]
516
        sock.close()
517

  
518
    def test_015_rdp_to_server_IPv6(self):
519
        "Test RDP connection to server public IPv6 works"""
520
        self._skipIf(not self.is_windows, "only valid for Windows servers")
521
        server = self.client.get_server_details(self.serverid)
522
        ipv6 = self._get_ipv6(server)
523
        sock = _get_tcp_connection(socket.AF_INET6, ipv6, 3389)
524

  
525
        # No actual RDP processing done. We assume the RDP server is there
526
        # if the connection to the RDP port is successful.
527
        sock.close()
528

  
529
    def test_016_personality_is_enforced(self):
530
        """Test file injection for personality enforcement"""
531
        self._skipIf(self.is_windows, "only implemented for Linux servers")
532
        self.assertTrue(False, "test not implemented, will fail")
533

  
534
    def test_017_submit_delete_request(self):
535
        """Test submit request to delete server"""
536
        self.client.delete_server(self.serverid)
537

  
538
    def test_018_server_becomes_deleted(self):
539
        """Test server becomes DELETED"""
540
        self._insist_on_status_transition("ACTIVE", "DELETED",
541
                                         self.action_timeout,
542
                                         self.action_timeout)
543

  
544
    def test_019_server_no_longer_in_server_list(self):
545
        """Test server is no longer in server list"""
546
        servers = self.client.list_servers()
547
        self.assertNotIn(self.serverid, [s["id"] for s in servers])
548

  
549

  
550
class TestRunnerProcess(Process):
551
    """A distinct process used to execute part of the tests in parallel"""
552
    def __init__(self, **kw):
553
        Process.__init__(self, **kw)
554
        kwargs = kw["kwargs"]
555
        self.testq = kwargs["testq"]
556
        self.runner = kwargs["runner"]
557

  
558
    def run(self):
559
        # Make sure this test runner process dies with the parent
560
        # and is not left behind.
561
        #
562
        # WARNING: This uses the prctl(2) call and is
563
        # Linux-specific.
564
        prctl.set_pdeathsig(signal.SIGHUP)
565

  
566
        while True:
567
            log.debug("I am process %d, GETting from queue is %s",
568
                     os.getpid(), self.testq)
569
            msg = self.testq.get()
570
            log.debug("Dequeued msg: %s", msg)
571

  
572
            if msg == "TEST_RUNNER_TERMINATE":
573
                raise SystemExit
574
            elif issubclass(msg, unittest.TestCase):
575
                # Assemble a TestSuite, and run it
576
                suite = unittest.TestLoader().loadTestsFromTestCase(msg)
577
                self.runner.run(suite)
578
            else:
579
                raise Exception("Cannot handle msg: %s" % msg)
580

  
581

  
582
def _run_cases_in_parallel(cases, fanout=1, runner=None):
583
    """Run instances of TestCase in parallel, in a number of distinct processes
584

  
585
    The cases iterable specifies the TestCases to be executed in parallel,
586
    by test runners running in distinct processes.
587
    The fanout parameter specifies the number of processes to spawn,
588
    and defaults to 1.
589
    The runner argument specifies the test runner class to use inside each
590
    runner process.
591

  
592
    """
593
    if runner is None:
594
        runner = unittest.TextTestRunner(verbosity=2, failfast=True)
595

  
596
    # testq: The master process enqueues TestCase objects into this queue,
597
    #        test runner processes pick them up for execution, in parallel.
598
    testq = Queue()
599
    runners = []
600
    for i in xrange(0, fanout):
601
        kwargs = dict(testq=testq, runner=runner)
602
        runners.append(TestRunnerProcess(kwargs=kwargs))
603

  
604
    log.info("Spawning %d test runner processes", len(runners))
605
    for p in runners:
606
        p.start()
607
    log.debug("Spawned %d test runners, PIDs are %s",
608
              len(runners), [p.pid for p in runners])
609

  
610
    # Enqueue test cases
611
    map(testq.put, cases)
612
    map(testq.put, ["TEST_RUNNER_TERMINATE"] * len(runners))
613

  
614
    log.debug("Joining %d processes", len(runners))
615
    for p in runners:
616
        p.join()
617
    log.debug("Done joining %d processes", len(runners))
618

  
619

  
620
def _spawn_server_test_case(**kwargs):
621
    """Construct a new unit test case class from SpawnServerTestCase"""
622

  
623
    name = "SpawnServerTestCase_%d" % kwargs["imageid"]
624
    cls = type(name, (SpawnServerTestCase,), kwargs)
625

  
626
    # Patch extra parameters into test names by manipulating method docstrings
627
    for (mname, m) in \
628
        inspect.getmembers(cls, lambda x: inspect.ismethod(x)):
629
            if hasattr(m, __doc__):
630
                m.__func__.__doc__ = "[%s] %s" % (imagename, m.__doc__)
631

  
632
    # Make sure the class can be pickled, by listing it among
633
    # the attributes of __main__. A PicklingError is raised otherwise.
634
    setattr(__main__, name, cls)
635
    return cls
636

  
637

  
638
def cleanup_servers(delete_stale=False):
639
    c = Client(API, TOKEN)
640
    servers = c.list_servers()
641
    stale = [s for s in servers if s["name"].startswith(SNF_TEST_PREFIX)]
642

  
643
    if len(stale) == 0:
644
        return
645

  
646
    print >> sys.stderr, "Found these stale servers from previous runs:"
647
    print "    " + \
648
          "\n    ".join(["%d: %s" % (s["id"], s["name"]) for s in stale])
649

  
650
    if delete_stale:
651
        print >> sys.stderr, "Deleting %d stale servers:" % len(stale)
... This diff was truncated because it exceeds the maximum size that can be displayed.

Also available in: Unified diff