Statistics
| Branch: | Tag: | Revision:

root / snf-cyclades-app / synnefo / tools / burnin.py @ 77054bf5

History | View | Annotate | Download (41.4 kB)

1
 #!/usr/bin/env python
2

    
3
# Copyright 2011 GRNET S.A. All rights reserved.
4
#
5
# Redistribution and use in source and binary forms, with or
6
# without modification, are permitted provided that the following
7
# conditions are met:
8
#
9
#   1. Redistributions of source code must retain the above
10
#      copyright notice, this list of conditions and the following
11
#      disclaimer.
12
#
13
#   2. Redistributions in binary form must reproduce the above
14
#      copyright notice, this list of conditions and the following
15
#      disclaimer in the documentation and/or other materials
16
#      provided with the distribution.
17
#
18
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
19
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
20
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
21
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
22
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
25
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
26
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
28
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29
# POSSIBILITY OF SUCH DAMAGE.
30
#
31
# The views and conclusions contained in the software and
32
# documentation are those of the authors and should not be
33
# interpreted as representing official policies, either expressed
34
# or implied, of GRNET S.A.
35

    
36
"""Perform integration testing on a running Synnefo deployment"""
37

    
38
import __main__
39
import datetime
40
import inspect
41
import logging
42
import os
43
import paramiko
44
import prctl
45
import subprocess
46
import signal
47
import socket
48
import struct
49
import sys
50
import time
51
import hashlib
52

    
53
from IPy import IP
54
from multiprocessing import Process, Queue
55
from random import choice
56

    
57
from kamaki.clients import ClientError, ComputeClient, CycladesClient
58
from kamaki.config import Config
59

    
60
from vncauthproxy.d3des import generate_response as d3des_generate_response
61

    
62
# Use backported unittest functionality if Python < 2.7
63
try:
64
    import unittest2 as unittest
65
except ImportError:
66
    if sys.version_info < (2, 7):
67
        raise Exception("The unittest2 package is required for Python < 2.7")
68
    import unittest
69

    
70

    
71
API = None
72
TOKEN = None
73
DEFAULT_API = "http://127.0.0.1:8000/api/v1.1"
74

    
75
# A unique id identifying this test run
76
TEST_RUN_ID = datetime.datetime.strftime(datetime.datetime.now(),
77
                                         "%Y%m%d%H%M%S")
78
SNF_TEST_PREFIX = "snf-test-"
79

    
80
# Setup logging (FIXME - verigak)
81
logging.basicConfig(format="%(message)s")
82
log = logging.getLogger("burnin")
83
log.setLevel(logging.INFO)
84

    
85
class UnauthorizedTestCase(unittest.TestCase):
86
    def test_unauthorized_access(self):
87
        """Test access without a valid token fails"""
88
        falseToken = '12345'
89
        conf = Config()
90
        conf.set('compute_token', falseToken)
91
        c=ComputeClient(conf)
92

    
93
        with self.assertRaises(ClientError) as cm:
94
            c.list_servers()
95
        self.assertEqual(cm.exception.status, 401)
96

    
97

    
98
class ImagesTestCase(unittest.TestCase):
99
    """Test image lists for consistency"""
100
    @classmethod
101
    def setUpClass(cls):
102
        """Initialize kamaki, get (detailed) list of images"""
103
        log.info("Getting simple and detailed list of images")
104

    
105
        conf = Config()
106
        conf.set('compute_token', TOKEN)
107
        cls.client = ComputeClient(conf)
108
        cls.images = cls.client.list_images()
109
        cls.dimages = cls.client.list_images(detail=True)
110

    
111
    def test_001_list_images(self):
112
        """Test image list actually returns images"""
113
        self.assertGreater(len(self.images), 0)
114

    
115
    def test_002_list_images_detailed(self):
116
        """Test detailed image list is the same length as list"""
117
        self.assertEqual(len(self.dimages), len(self.images))
118

    
119
    def test_003_same_image_names(self):
120
        """Test detailed and simple image list contain same names"""
121
        names = sorted(map(lambda x: x["name"], self.images))
122
        dnames = sorted(map(lambda x: x["name"], self.dimages))
123
        self.assertEqual(names, dnames)
124

    
125
    def test_004_unique_image_names(self):
126
        """Test images have unique names"""
127
        names = sorted(map(lambda x: x["name"], self.images))
128
        self.assertEqual(sorted(list(set(names))), names)
129

    
130
    def test_005_image_metadata(self):
131
        """Test every image has specific metadata defined"""
132
        keys = frozenset(["os", "description", "size"])
133
        for i in self.dimages:
134
            self.assertTrue(keys.issubset(i["metadata"]["values"].keys()))
135

    
136

    
137
class FlavorsTestCase(unittest.TestCase):
138
    """Test flavor lists for consistency"""
139
    @classmethod
140
    def setUpClass(cls):
141
        """Initialize kamaki, get (detailed) list of flavors"""
142
        log.info("Getting simple and detailed list of flavors")
143

    
144
        conf = Config()
145
        conf.set('compute_token', TOKEN)
146
        cls.client = ComputeClient(conf)
147
        cls.flavors = cls.client.list_flavors()
148
        cls.dflavors = cls.client.list_flavors(detail=True)
149

    
150
    def test_001_list_flavors(self):
151
        """Test flavor list actually returns flavors"""
152
        self.assertGreater(len(self.flavors), 0)
153

    
154
    def test_002_list_flavors_detailed(self):
155
        """Test detailed flavor list is the same length as list"""
156
        self.assertEquals(len(self.dflavors), len(self.flavors))
157

    
158
    def test_003_same_flavor_names(self):
159
        """Test detailed and simple flavor list contain same names"""
160
        names = sorted(map(lambda x: x["name"], self.flavors))
161
        dnames = sorted(map(lambda x: x["name"], self.dflavors))
162
        self.assertEqual(names, dnames)
163

    
164
    def test_004_unique_flavor_names(self):
165
        """Test flavors have unique names"""
166
        names = sorted(map(lambda x: x["name"], self.flavors))
167
        self.assertEqual(sorted(list(set(names))), names)
168

    
169
    def test_005_well_formed_flavor_names(self):
170
        """Test flavors have names of the form CxxRyyDzz
171

172
        Where xx is vCPU count, yy is RAM in MiB, zz is Disk in GiB
173

174
        """
175
        for f in self.dflavors:
176
            self.assertEqual("C%dR%dD%d" % (f["cpu"], f["ram"], f["disk"]),
177
                             f["name"],
178
                             "Flavor %s does not match its specs." % f["name"])
179

    
180

    
181
class ServersTestCase(unittest.TestCase):
182
    """Test server lists for consistency"""
183
    @classmethod
184
    def setUpClass(cls):
185
        """Initialize kamaki, get (detailed) list of servers"""
186
        log.info("Getting simple and detailed list of servers")
187

    
188
        conf = Config()
189
        conf.set('compute_token', TOKEN)
190
        cls.client = ComputeClient(conf)
191
        cls.servers = cls.client.list_servers()
192
        cls.dservers = cls.client.list_servers(detail=True)
193

    
194
    def test_001_list_servers(self):
195
        """Test server list actually returns servers"""
196
        self.assertGreater(len(self.servers), 0)
197

    
198
    def test_002_list_servers_detailed(self):
199
        """Test detailed server list is the same length as list"""
200
        self.assertEqual(len(self.dservers), len(self.servers))
201

    
202
    def test_003_same_server_names(self):
203
        """Test detailed and simple flavor list contain same names"""
204
        names = sorted(map(lambda x: x["name"], self.servers))
205
        dnames = sorted(map(lambda x: x["name"], self.dservers))
206
        self.assertEqual(names, dnames)
207

    
208

    
209
# This class gets replicated into actual TestCases dynamically
210
class SpawnServerTestCase(unittest.TestCase):
211
    """Test scenario for server of the specified image"""
212

    
213
    @classmethod
214
    def setUpClass(cls):
215
        """Initialize a kamaki instance"""
216
        log.info("Spawning server for image `%s'", cls.imagename)
217

    
218
        conf = Config()
219
        conf.set('compute_token', TOKEN)
220
        cls.client = ComputeClient(conf)
221
        cls.cyclades = CycladesClient(conf)
222

    
223
    def _get_ipv4(self, server):
224
        """Get the public IPv4 of a server from the detailed server info"""
225

    
226
        public_addrs = filter(lambda x: x["id"] == "public",
227
                              server["addresses"]["values"])
228
        self.assertEqual(len(public_addrs), 1)
229
        ipv4_addrs = filter(lambda x: x["version"] == 4,
230
                            public_addrs[0]["values"])
231
        self.assertEqual(len(ipv4_addrs), 1)
232
        return ipv4_addrs[0]["addr"]
233

    
234
    def _get_ipv6(self, server):
235
        """Get the public IPv6 of a server from the detailed server info"""
236
        public_addrs = filter(lambda x: x["id"] == "public",
237
                              server["addresses"]["values"])
238
        self.assertEqual(len(public_addrs), 1)
239
        ipv6_addrs = filter(lambda x: x["version"] == 6,
240
                            public_addrs[0]["values"])
241
        self.assertEqual(len(ipv6_addrs), 1)
242
        return ipv6_addrs[0]["addr"]
243

    
244
    def _connect_loginname(self, os):
245
        """Return the login name for connections based on the server OS"""
246
        if os in ("Ubuntu", "Kubuntu", "Fedora"):
247
            return "user"
248
        elif os in ("windows", "windows_alpha1"):
249
            return "Administrator"
250
        else:
251
            return "root"
252

    
253
    def _verify_server_status(self, current_status, new_status):
254
        """Verify a server has switched to a specified status"""
255
        server = self.client.get_server_details(self.serverid)
256
        if server["status"] not in (current_status, new_status):
257
            return None  # Do not raise exception, return so the test fails
258
        self.assertEquals(server["status"], new_status)
259

    
260
    def _get_connected_tcp_socket(self, family, host, port):
261
        """Get a connected socket from the specified family to host:port"""
262
        sock = None
263
        for res in \
264
            socket.getaddrinfo(host, port, family, socket.SOCK_STREAM, 0,
265
                               socket.AI_PASSIVE):
266
            af, socktype, proto, canonname, sa = res
267
            try:
268
                sock = socket.socket(af, socktype, proto)
269
            except socket.error as msg:
270
                sock = None
271
                continue
272
            try:
273
                sock.connect(sa)
274
            except socket.error as msg:
275
                sock.close()
276
                sock = None
277
                continue
278
        self.assertIsNotNone(sock)
279
        return sock
280

    
281
    def _ping_once(self, ipv6, ip):
282
        """Test server responds to a single IPv4 or IPv6 ping"""
283
        cmd = "ping%s -c 2 -w 3 %s" % ("6" if ipv6 else "", ip)
284
        ping = subprocess.Popen(cmd, shell=True,
285
                                stdout=subprocess.PIPE, stderr=subprocess.PIPE)
286
        (stdout, stderr) = ping.communicate()
287
        ret = ping.wait()
288
        self.assertEquals(ret, 0)
289

    
290
    def _get_hostname_over_ssh(self, hostip, username, password):
291
        ssh = paramiko.SSHClient()
292
        ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
293
        try:
294
            ssh.connect(hostip, username=username, password=password)
295
        except socket.error:
296
            raise AssertionError
297
        stdin, stdout, stderr = ssh.exec_command("hostname")
298
        lines = stdout.readlines()
299
        self.assertEqual(len(lines), 1)
300
        return lines[0]
301

    
302
    def _try_until_timeout_expires(self, warn_timeout, fail_timeout,
303
                                   opmsg, callable, *args, **kwargs):
304
        if warn_timeout == fail_timeout:
305
            warn_timeout = fail_timeout + 1
306
        warn_tmout = time.time() + warn_timeout
307
        fail_tmout = time.time() + fail_timeout
308
        while True:
309
            self.assertLess(time.time(), fail_tmout,
310
                            "operation `%s' timed out" % opmsg)
311
            if time.time() > warn_tmout:
312
                log.warning("Server %d: `%s' operation `%s' not done yet",
313
                            self.serverid, self.servername, opmsg)
314
            try:
315
                log.info("%s... " % opmsg)
316
                return callable(*args, **kwargs)
317
            except AssertionError:
318
                pass
319
            time.sleep(self.query_interval)
320

    
321
    def _insist_on_tcp_connection(self, family, host, port):
322
        familystr = {socket.AF_INET: "IPv4", socket.AF_INET6: "IPv6",
323
                     socket.AF_UNSPEC: "Unspecified-IPv4/6"}
324
        msg = "connect over %s to %s:%s" % \
325
              (familystr.get(family, "Unknown"), host, port)
326
        sock = self._try_until_timeout_expires(
327
                self.action_timeout, self.action_timeout,
328
                msg, self._get_connected_tcp_socket,
329
                family, host, port)
330
        return sock
331

    
332
    def _insist_on_status_transition(self, current_status, new_status,
333
                                    fail_timeout, warn_timeout=None):
334
        msg = "Server %d: `%s', waiting for %s -> %s" % \
335
              (self.serverid, self.servername, current_status, new_status)
336
        if warn_timeout is None:
337
            warn_timeout = fail_timeout
338
        self._try_until_timeout_expires(warn_timeout, fail_timeout,
339
                                        msg, self._verify_server_status,
340
                                        current_status, new_status)
341
        # Ensure the status is actually the expected one
342
        server = self.client.get_server_details(self.serverid)
343
        self.assertEquals(server["status"], new_status)
344

    
345
    def _insist_on_ssh_hostname(self, hostip, username, password):
346
        msg = "SSH to %s, as %s/%s" % (hostip, username, password)
347
        hostname = self._try_until_timeout_expires(
348
                self.action_timeout, self.action_timeout,
349
                msg, self._get_hostname_over_ssh,
350
                hostip, username, password)
351

    
352
        # The hostname must be of the form 'prefix-id'
353
        self.assertTrue(hostname.endswith("-%d\n" % self.serverid))
354

    
355

    
356
    def _file_md5(filename, block_size = 2**20):
357
        f = open(filename)
358
        md5 = hashlib.md5()
359
        while True:
360
            data = f.read(block_size)
361
            if not data:
362
                break
363
            md5.update(data)
364
            f.close()
365
    
366
        return md5.digest()
367

    
368
    
369
    def _check_file_through_ssh(self, hostip, username, pavssword, path):
370
        msg = "SSH to %s, as %s/%s" % (hostip, username, password)
371
        try:
372
            ssh = paramiko.SSHClient()
373
            ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
374
            ssh.connect(hostip, username=username, password=password)
375
        except socket.error:
376
            raise AssertionError
377
        
378
        transport.connect(username = username, password = password)
379
        remotepath = path
380
        localpath = '/tmp/'+SNF_TEST_PREFIX+'injection'
381
        sftp = paramiko.SFTPClient.from_transport(transport)
382
        sftp.get(remotepath, localpath)
383
        
384
        sftp.close()
385
        transport.close()
386

    
387
        # Check if files are the same
388
        return _file_md5(localpath) == _file_md5('test.txt')
389

    
390
    def _skipIf(self, condition, msg):
391
        if condition:
392
            self.skipTest(msg)
393

    
394
    def test_001_submit_create_server(self):
395
        """Test submit create server request"""
396
        server = self.client.create_server(self.servername, self.flavorid,
397
                                           self.imageid, self.personality)
398
        self.assertEqual(server["name"], self.servername)
399
        self.assertEqual(server["flavorRef"], self.flavorid)
400
        self.assertEqual(server["imageRef"], self.imageid)
401
        self.assertEqual(server["status"], "BUILD")
402

    
403
        # Update class attributes to reflect data on building server
404
        cls = type(self)
405
        cls.serverid = server["id"]
406
        cls.username = None
407
        cls.passwd = server["adminPass"]
408

    
409
    def test_002a_server_is_building_in_list(self):
410
        """Test server is in BUILD state, in server list"""
411
        servers = self.client.list_servers(detail=True)
412
        servers = filter(lambda x: x["name"] == self.servername, servers)
413
        self.assertEqual(len(servers), 1)
414
        server = servers[0]
415
        self.assertEqual(server["name"], self.servername)
416
        self.assertEqual(server["flavorRef"], self.flavorid)
417
        self.assertEqual(server["imageRef"], self.imageid)
418
        self.assertEqual(server["status"], "BUILD")
419

    
420
    def test_002b_server_is_building_in_details(self):
421
        """Test server is in BUILD state, in details"""
422
        server = self.client.get_server_details(self.serverid)
423
        self.assertEqual(server["name"], self.servername)
424
        self.assertEqual(server["flavorRef"], self.flavorid)
425
        self.assertEqual(server["imageRef"], self.imageid)
426
        self.assertEqual(server["status"], "BUILD")
427

    
428
    def test_002c_set_server_metadata(self):
429
        image = self.client.get_image_details(self.imageid)
430
        os = image["metadata"]["values"]["os"]
431
        loginname = image["metadata"]["values"].get("users", None)
432
        self.client.update_server_metadata(self.serverid, OS=os)
433

    
434
        # Determine the username to use for future connections
435
        # to this host
436
        cls = type(self)
437
        cls.username = loginname
438
        if not cls.username:
439
            cls.username = self._connect_loginname(os)
440
        self.assertIsNotNone(cls.username)
441

    
442
    def test_002d_verify_server_metadata(self):
443
        """Test server metadata keys are set based on image metadata"""
444
        servermeta = self.client.get_server_metadata(self.serverid)
445
        imagemeta = self.client.get_image_metadata(self.imageid)
446
        self.assertEqual(servermeta["OS"], imagemeta["os"])
447

    
448
    def test_003_server_becomes_active(self):
449
        """Test server becomes ACTIVE"""
450
        self._insist_on_status_transition("BUILD", "ACTIVE",
451
                                         self.build_fail, self.build_warning)
452

    
453
    def test_003a_get_server_oob_console(self):
454
        """Test getting OOB server console over VNC
455

456
        Implementation of RFB protocol follows
457
        http://www.realvnc.com/docs/rfbproto.pdf.
458

459
        """
460
        
461
        console = self.cyclades.get_server_console(self.serverid)
462
        self.assertEquals(console['type'], "vnc")
463
        sock = self._insist_on_tcp_connection(socket.AF_UNSPEC,
464
                                        console["host"], console["port"])
465

    
466
        # Step 1. ProtocolVersion message (par. 6.1.1)
467
        version = sock.recv(1024)
468
        self.assertEquals(version, 'RFB 003.008\n')
469
        sock.send(version)
470

    
471
        # Step 2. Security (par 6.1.2): Only VNC Authentication supported
472
        sec = sock.recv(1024)
473
        self.assertEquals(list(sec), ['\x01', '\x02'])
474

    
475
        # Step 3. Request VNC Authentication (par 6.1.2)
476
        sock.send('\x02')
477

    
478
        # Step 4. Receive Challenge (par 6.2.2)
479
        challenge = sock.recv(1024)
480
        self.assertEquals(len(challenge), 16)
481

    
482
        # Step 5. DES-Encrypt challenge, use password as key (par 6.2.2)
483
        response = d3des_generate_response(
484
            (console["password"] + '\0' * 8)[:8], challenge)
485
        sock.send(response)
486

    
487
        # Step 6. SecurityResult (par 6.1.3)
488
        result = sock.recv(4)
489
        self.assertEquals(list(result), ['\x00', '\x00', '\x00', '\x00'])
490
        sock.close()
491

    
492
    def test_004_server_has_ipv4(self):
493
        """Test active server has a valid IPv4 address"""
494
        server = self.client.get_server_details(self.serverid)
495
        ipv4 = self._get_ipv4(server)
496
        self.assertEquals(IP(ipv4).version(), 4)
497

    
498
    def test_005_server_has_ipv6(self):
499
        """Test active server has a valid IPv6 address"""
500
        server = self.client.get_server_details(self.serverid)
501
        ipv6 = self._get_ipv6(server)
502
        self.assertEquals(IP(ipv6).version(), 6)
503

    
504
    def test_006_server_responds_to_ping_IPv4(self):
505
        """Test server responds to ping on IPv4 address"""
506
        server = self.client.get_server_details(self.serverid)
507
        ip = self._get_ipv4(server)
508
        self._try_until_timeout_expires(self.action_timeout,
509
                                        self.action_timeout,
510
                                        "PING IPv4 to %s" % ip,
511
                                        self._ping_once,
512
                                        False, ip)
513

    
514
    def test_007_server_responds_to_ping_IPv6(self):
515
        """Test server responds to ping on IPv6 address"""
516
        server = self.client.get_server_details(self.serverid)
517
        ip = self._get_ipv6(server)
518
        self._try_until_timeout_expires(self.action_timeout,
519
                                        self.action_timeout,
520
                                        "PING IPv6 to %s" % ip,
521
                                        self._ping_once,
522
                                        True, ip)
523

    
524
    def test_008_submit_shutdown_request(self):
525
        """Test submit request to shutdown server"""
526
        self.cyclades.shutdown_server(self.serverid)
527

    
528
    def test_009_server_becomes_stopped(self):
529
        """Test server becomes STOPPED"""
530
        self._insist_on_status_transition("ACTIVE", "STOPPED",
531
                                         self.action_timeout,
532
                                         self.action_timeout)
533

    
534
    def test_010_submit_start_request(self):
535
        """Test submit start server request"""
536
        self.cyclades.start_server(self.serverid)
537

    
538
    def test_011_server_becomes_active(self):
539
        """Test server becomes ACTIVE again"""
540
        self._insist_on_status_transition("STOPPED", "ACTIVE",
541
                                         self.action_timeout,
542
                                         self.action_timeout)
543

    
544
    def test_011a_server_responds_to_ping_IPv4(self):
545
        """Test server OS is actually up and running again"""
546
        self.test_006_server_responds_to_ping_IPv4()
547

    
548
    def test_012_ssh_to_server_IPv4(self):
549
        """Test SSH to server public IPv4 works, verify hostname"""
550
        self._skipIf(self.is_windows, "only valid for Linux servers")
551
        server = self.client.get_server_details(self.serverid)
552
        self._insist_on_ssh_hostname(self._get_ipv4(server),
553
                                     self.username, self.passwd)
554

    
555
    def test_013_ssh_to_server_IPv6(self):
556
        """Test SSH to server public IPv6 works, verify hostname"""
557
        self._skipIf(self.is_windows, "only valid for Linux servers")
558
        server = self.client.get_server_details(self.serverid)
559
        self._insist_on_ssh_hostname(self._get_ipv6(server),
560
                                     self.username, self.passwd)
561

    
562
    def test_014_rdp_to_server_IPv4(self):
563
        "Test RDP connection to server public IPv4 works"""
564
        self._skipIf(not self.is_windows, "only valid for Windows servers")
565
        server = self.client.get_server_details(self.serverid)
566
        ipv4 = self._get_ipv4(server)
567
        sock = _insist_on_tcp_connection(socket.AF_INET, ipv4, 3389)
568

    
569
        # No actual RDP processing done. We assume the RDP server is there
570
        # if the connection to the RDP port is successful.
571
        # FIXME: Use rdesktop, analyze exit code? see manpage [costasd]
572
        sock.close()
573

    
574
    def test_015_rdp_to_server_IPv6(self):
575
        "Test RDP connection to server public IPv6 works"""
576
        self._skipIf(not self.is_windows, "only valid for Windows servers")
577
        server = self.client.get_server_details(self.serverid)
578
        ipv6 = self._get_ipv6(server)
579
        sock = _get_tcp_connection(socket.AF_INET6, ipv6, 3389)
580

    
581
        # No actual RDP processing done. We assume the RDP server is there
582
        # if the connection to the RDP port is successful.
583
        sock.close()
584

    
585
    def test_016_personality_is_enforced(self):
586
        """Test file injection for personality enforcement"""
587
        self._skipIf(self.is_windows, "only implemented for Linux servers")
588
        
589

    
590
        #Create new server
591
        server = self.client.create_server(self.servername, self.flavorid,
592
                                           self.imageid, self.personality) #/path/to/file
593
        self.assertEqual(server["name"], self.servername)
594
        self.assertEqual(server["flavorRef"], self.flavorid)
595
        self.assertEqual(server["imageRef"], self.imageid)
596
        self.assertEqual(server["status"], "BUILD")
597
        
598
        #Test if is in active state
599
        servers = self.client.list_servers(detail=True)
600
        servers = filter(lambda x: x["name"] == self.servername, servers)
601
        self.assertEqual(len(servers), 1)
602

    
603
        #Test if server is building in details
604
        server = self.client.get_server_details(self.serverid)
605
        self.assertEqual(server["name"], self.servername)
606
        self.assertEqual(server["flavorRef"], self.flavorid)
607
        self.assertEqual(server["imageRef"], self.imageid)
608
        self.assertEqual(server["status"], "BUILD")
609

    
610
        #Insist on transition
611
        self._insist_on_status_transition("BUILD", "ACTIVE",
612
                                          self.build_fail, self.build_warning)
613
        
614
        #Test if file injected exists
615
        equal = self._check_file_through_ssh(self._get_ipv4(server), self.username, self.password)
616
        
617
        self.assertTrue(equal)
618

    
619
    def test_017_submit_delete_request(self):
620
        """Test submit request to delete server"""
621
        self.client.delete_server(self.serverid)
622

    
623
    def test_018_server_becomes_deleted(self):
624
        """Test server becomes DELETED"""
625
        self._insist_on_status_transition("ACTIVE", "DELETED",
626
                                         self.action_timeout,
627
                                         self.action_timeout)
628

    
629
    def test_019_server_no_longer_in_server_list(self):
630
        """Test server is no longer in server list"""
631
        servers = self.client.list_servers()
632
        self.assertNotIn(self.serverid, [s["id"] for s in servers])
633

    
634

    
635
class NetworkTestCase(unittest.TestCase):
636
    """ Testing networking in cyclades """
637
    @classmethod
638
    def setUpClass(cls):
639
        "Initialize kamaki, get list of current networks"
640
        conf = Config()
641
        conf.set('compute_token', TOKEN)
642
        cls.client = CycladesClient(conf)
643
        cls.compute = ComputeClient(conf)
644

    
645
        images = cls.compute.list_images(detail = True)
646
        flavors = cls.compute.list_flavors(detail = True)
647
        imageid = choice([im['id'] for im in images])
648
        flavorid = choice([f["id"] for f in flavors if f["disk"] >= 20])
649

    
650
        for image in images:
651
            if image['id'] == imageid:
652
                imagename = image['name']
653

    
654
        servername = "%s%s for %s" % (SNF_TEST_PREFIX, TEST_RUN_ID, imagename)
655
        is_windows = imagename.lower().find("windows") >= 0
656
        setupCase =  _spawn_server_test_case(imageid=str(imageid), flavorid=flavorid,
657
                                             imagename=imagename,
658
                                             personality=None,
659
                                             servername=servername,
660
                                             is_windows=is_windows,
661
                                             action_timeout=200,
662
                                             build_warning=1200,
663
                                             build_fail=500,
664
                                             query_interval=3)
665

    
666
        #Using already implemented tests for serverlist population
667
        suite = unittest.TestSuite()
668
        suite.addTest(setupCase('test_001_submit_create_server'))
669
        suite.addTest(setupCase('test_002a_server_is_building_in_list'))
670
        suite.addTest(setupCase('test_002b_server_is_building_in_details'))        
671
        suite.addTest(setupCase('test_003_server_becomes_active'))
672
        unittest.TextTestRunner(verbosity=2).run(suite)
673

    
674
    def test_001_create_network(self):
675
        """Test submit create network request"""
676
        name = SNF_TEST_PREFIX+TEST_RUN_ID
677
        network =  self.client.create_network(name)        
678
        previous_num = len(self.client.list_networks())
679

    
680
        #Test if right name is assigned
681
        self.assertEqual(network['name'], name)
682
        
683
        # Update class attributes
684
        cls = type(self)
685
        cls.networkid = network['id']
686
        networks = self.client.list_networks()
687

    
688
        #Test if new network is created
689
        self.assertTrue(len(networks) > previous_num)
690
        
691
    
692
    def test_002_connect_to_network(self):
693
        """Test VM to network connection"""
694
        servers = self.compute.list_servers()
695
        server = choice(servers)
696
        self.client.connect_server(server['id'], self.networkid)
697
        
698
        #Update class attributes
699
        cls = type(self)
700
        cls.serverid = server['id']
701

    
702
        connected = (self.client.get_network_details(self.networkid))
703
        connections = len(connected['servers']['values'])
704
        
705
        time.sleep(60)
706

    
707
        #FIXME: Insist on new connection
708
        self.assertTrue(connections>=1)
709
        
710

    
711
    def test_003_disconnect_from_network(self):
712
        prev_state = (self.client.get_network_details(self.networkid))
713
        prev_conn = len(prev_state['servers']['values'])
714

    
715
        self.client.disconnect_server(self.serverid, self.networkid)
716
        connected = (self.client.get_network_details(self.networkid))
717
        curr_conn = len(connected['servers']['values'])
718

    
719
        #FIXME: Insist on deleting
720
        self.assertTrue(curr_conn < prev_conn)
721

    
722
    def test_004_destroy_network(self):
723
        """Test submit delete network request"""
724
        self.client.delete_network(self.networkid)
725
        
726
        networks = self.client.list_networks()
727
        self.assertEqual(len(networks),1)
728

    
729

    
730
class TestRunnerProcess(Process):
731
    """A distinct process used to execute part of the tests in parallel"""
732
    def __init__(self, **kw):
733
        Process.__init__(self, **kw)
734
        kwargs = kw["kwargs"]
735
        self.testq = kwargs["testq"]
736
        self.runner = kwargs["runner"]
737

    
738
    def run(self):
739
        # Make sure this test runner process dies with the parent
740
        # and is not left behind.
741
        #
742
        # WARNING: This uses the prctl(2) call and is
743
        # Linux-specific.
744
        prctl.set_pdeathsig(signal.SIGHUP)
745

    
746
        while True:
747
            log.debug("I am process %d, GETting from queue is %s",
748
                     os.getpid(), self.testq)
749
            msg = self.testq.get()
750
            log.debug("Dequeued msg: %s", msg)
751

    
752
            if msg == "TEST_RUNNER_TERMINATE":
753
                raise SystemExit
754
            elif issubclass(msg, unittest.TestCase):
755
                # Assemble a TestSuite, and run it
756
                suite = unittest.TestLoader().loadTestsFromTestCase(msg)
757
                self.runner.run(suite)
758
            else:
759
                raise Exception("Cannot handle msg: %s" % msg)
760

    
761

    
762

    
763
def _run_cases_in_parallel(cases, fanout=1, runner=None):
764
    """Run instances of TestCase in parallel, in a number of distinct processes
765

766
    The cases iterable specifies the TestCases to be executed in parallel,
767
    by test runners running in distinct processes.
768
    The fanout parameter specifies the number of processes to spawn,
769
    and defaults to 1.
770
    The runner argument specifies the test runner class to use inside each
771
    runner process.
772

773
    """
774
    if runner is None:
775
        runner = unittest.TextTestRunner(verbosity=2, failfast=True)
776

    
777
    # testq: The master process enqueues TestCase objects into this queue,
778
    #        test runner processes pick them up for execution, in parallel.
779
    testq = Queue()
780
    runners = []
781
    for i in xrange(0, fanout):
782
        kwargs = dict(testq=testq, runner=runner)
783
        runners.append(TestRunnerProcess(kwargs=kwargs))
784

    
785
    log.info("Spawning %d test runner processes", len(runners))
786
    for p in runners:
787
        p.start()
788
    log.debug("Spawned %d test runners, PIDs are %s",
789
              len(runners), [p.pid for p in runners])
790

    
791
    # Enqueue test cases
792
    map(testq.put, cases)
793
    map(testq.put, ["TEST_RUNNER_TERMINATE"] * len(runners))
794

    
795
    log.debug("Joining %d processes", len(runners))
796
    for p in runners:
797
        p.join()
798
    log.debug("Done joining %d processes", len(runners))
799

    
800

    
801
def _spawn_server_test_case(**kwargs):
802
    """Construct a new unit test case class from SpawnServerTestCase"""
803

    
804
    name = "SpawnServerTestCase_%s" % kwargs["imageid"]
805
    cls = type(name, (SpawnServerTestCase,), kwargs)
806

    
807
    # Patch extra parameters into test names by manipulating method docstrings
808
    for (mname, m) in \
809
        inspect.getmembers(cls, lambda x: inspect.ismethod(x)):
810
            if hasattr(m, __doc__):
811
                m.__func__.__doc__ = "[%s] %s" % (imagename, m.__doc__)
812

    
813
    # Make sure the class can be pickled, by listing it among
814
    # the attributes of __main__. A PicklingError is raised otherwise.
815
    setattr(__main__, name, cls)
816
    return cls
817

    
818

    
819
def cleanup_servers(delete_stale=False):
820

    
821
    conf = Config()
822
    conf.set('compute_token', TOKEN)
823
    c = ComputeClient(conf)
824

    
825
    servers = c.list_servers()
826
    stale = [s for s in servers if s["name"].startswith(SNF_TEST_PREFIX)]
827

    
828
    if len(stale) == 0:
829
        return
830

    
831
    print >> sys.stderr, "Found these stale servers from previous runs:"
832
    print "    " + \
833
          "\n    ".join(["%d: %s" % (s["id"], s["name"]) for s in stale])
834

    
835
    if delete_stale:
836
        print >> sys.stderr, "Deleting %d stale servers:" % len(stale)
837
        for server in stale:
838
            c.delete_server(server["id"])
839
        print >> sys.stderr, "    ...done"
840
    else:
841
        print >> sys.stderr, "Use --delete-stale to delete them."
842

    
843

    
844
def parse_arguments(args):
845
    from optparse import OptionParser
846

    
847
    kw = {}
848
    kw["usage"] = "%prog [options]"
849
    kw["description"] = \
850
        "%prog runs a number of test scenarios on a " \
851
        "Synnefo deployment."
852

    
853
    parser = OptionParser(**kw)
854
    parser.disable_interspersed_args()
855
    parser.add_option("--api",
856
                      action="store", type="string", dest="api",
857
                      help="The API URI to use to reach the Synnefo API",
858
                      default=DEFAULT_API)
859
    parser.add_option("--token",
860
                      action="store", type="string", dest="token",
861
                      help="The token to use for authentication to the API")
862
    parser.add_option("--nofailfast",
863
                      action="store_true", dest="nofailfast",
864
                      help="Do not fail immediately if one of the tests " \
865
                           "fails (EXPERIMENTAL)",
866
                      default=False)
867
    parser.add_option("--action-timeout",
868
                      action="store", type="int", dest="action_timeout",
869
                      metavar="TIMEOUT",
870
                      help="Wait SECONDS seconds for a server action to " \
871
                           "complete, then the test is considered failed",
872
                      default=100)
873
    parser.add_option("--build-warning",
874
                      action="store", type="int", dest="build_warning",
875
                      metavar="TIMEOUT",
876
                      help="Warn if TIMEOUT seconds have passed and a " \
877
                           "build operation is still pending",
878
                      default=600)
879
    parser.add_option("--build-fail",
880
                      action="store", type="int", dest="build_fail",
881
                      metavar="BUILD_TIMEOUT",
882
                      help="Fail the test if TIMEOUT seconds have passed " \
883
                           "and a build operation is still incomplete",
884
                      default=900)
885
    parser.add_option("--query-interval",
886
                      action="store", type="int", dest="query_interval",
887
                      metavar="INTERVAL",
888
                      help="Query server status when requests are pending " \
889
                           "every INTERVAL seconds",
890
                      default=3)
891
    parser.add_option("--fanout",
892
                      action="store", type="int", dest="fanout",
893
                      metavar="COUNT",
894
                      help="Spawn up to COUNT child processes to execute " \
895
                           "in parallel, essentially have up to COUNT " \
896
                           "server build requests outstanding (EXPERIMENTAL)",
897
                      default=1)
898
    parser.add_option("--force-flavor",
899
                      action="store", type="int", dest="force_flavorid",
900
                      metavar="FLAVOR ID",
901
                      help="Force all server creations to use the specified "\
902
                           "FLAVOR ID instead of a randomly chosen one, " \
903
                           "useful if disk space is scarce",
904
                      default=None)
905
    parser.add_option("--image-id",
906
                      action="store", type="string", dest="force_imageid",
907
                      metavar="IMAGE ID",
908
                      help="Test the specified image id, use 'all' to test " \
909
                           "all available images (mandatory argument)",
910
                      default=None)
911
    parser.add_option("--show-stale",
912
                      action="store_true", dest="show_stale",
913
                      help="Show stale servers from previous runs, whose "\
914
                           "name starts with `%s'" % SNF_TEST_PREFIX,
915
                      default=False)
916
    parser.add_option("--delete-stale",
917
                      action="store_true", dest="delete_stale",
918
                      help="Delete stale servers from previous runs, whose "\
919
                           "name starts with `%s'" % SNF_TEST_PREFIX,
920
                      default=False)
921

    
922
    # FIXME: Change the default for build-fanout to 10
923
    # FIXME: Allow the user to specify a specific set of Images to test
924

    
925
    (opts, args) = parser.parse_args(args)
926

    
927
    # Verify arguments
928
    if opts.delete_stale:
929
        opts.show_stale = True
930

    
931
    if not opts.show_stale:
932
        if not opts.force_imageid:
933
            print >>sys.stderr, "The --image-id argument is mandatory."
934
            parser.print_help()
935
            sys.exit(1)
936

    
937
        if opts.force_imageid != 'all':
938
            try:
939
                opts.force_imageid = str(opts.force_imageid)
940
            except ValueError:
941
                print >>sys.stderr, "Invalid value specified for --image-id." \
942
                                    "Use a valid id, or `all'."
943
                sys.exit(1)
944

    
945
    return (opts, args)
946

    
947

    
948
def main():
949
    """Assemble test cases into a test suite, and run it
950

951
    IMPORTANT: Tests have dependencies and have to be run in the specified
952
    order inside a single test case. They communicate through attributes of the
953
    corresponding TestCase class (shared fixtures). Distinct subclasses of
954
    TestCase MAY SHARE NO DATA, since they are run in parallel, in distinct
955
    test runner processes.
956

957
    """
958
    (opts, args) = parse_arguments(sys.argv[1:])
959

    
960
    global API, TOKEN
961
    API = opts.api
962
    TOKEN = opts.token
963

    
964
    # Cleanup stale servers from previous runs
965
    if opts.show_stale:
966
        cleanup_servers(delete_stale=opts.delete_stale)
967
        return 0
968

    
969
    # Initialize a kamaki instance, get flavors, images
970

    
971
    conf = Config()
972
    conf.set('compute_token', TOKEN)
973
    c = ComputeClient(conf)
974

    
975
    DIMAGES = c.list_images(detail=True)
976
    DFLAVORS = c.list_flavors(detail=True)
977

    
978
    # FIXME: logging, log, LOG PID, TEST_RUN_ID, arguments
979
    # Run them: FIXME: In parallel, FAILEARLY, catchbreak?
980
    #unittest.main(verbosity=2, catchbreak=True)
981

    
982
    if opts.force_imageid == 'all':
983
        test_images = DIMAGES
984
    else:
985
        test_images = filter(lambda x: x["id"] == opts.force_imageid, DIMAGES)
986

    
987
    for image in test_images:
988
        imageid = str(image["id"])
989
        flavorid = choice([f["id"] for f in DFLAVORS if f["disk"] >= 20])
990
        imagename = image["name"]
991
        personality = None   # FIXME
992
        servername = "%s%s for %s" % (SNF_TEST_PREFIX, TEST_RUN_ID, imagename)
993
        is_windows = imagename.lower().find("windows") >= 0
994
        
995
        ServerTestCase = _spawn_server_test_case(imageid=imageid, flavorid=flavorid,
996
                                                 imagename=imagename,
997
                                                 personality=personality,
998
                                                 servername=servername,
999
                                                 is_windows=is_windows,
1000
                                                 action_timeout=opts.action_timeout,
1001
                                                 build_warning=opts.build_warning,
1002
                                                 build_fail=opts.build_fail,
1003
                                                 query_interval=opts.query_interval)
1004

    
1005

    
1006
    #Running all the testcases sequentially
1007
    #seq_cases = [UnauthorizedTestCase, FlavorsTestCase, ImagesTestCase, ServerTestCase, NetworkTestCase]
1008

    
1009
    seq_cases = [NetworkTestCase]
1010
    for case in seq_cases:
1011
        suite = unittest.TestLoader().loadTestsFromTestCase(case)
1012
        unittest.TextTestRunner(verbosity=2).run(suite)
1013
        
1014
    
1015

    
1016
    # # The Following cases run sequentially
1017
    # seq_cases = [UnauthorizedTestCase, FlavorsTestCase, ImagesTestCase]
1018
    # _run_cases_in_parallel(seq_cases, fanout=3, runner=runner)
1019

    
1020
    # # The following cases run in parallel
1021
    # par_cases = []
1022

    
1023
    # if opts.force_imageid == 'all':
1024
    #     test_images = DIMAGES
1025
    # else:
1026
    #     test_images = filter(lambda x: x["id"] == opts.force_imageid, DIMAGES)
1027

    
1028
    # for image in test_images:
1029
    #     imageid = image["id"]
1030
    #     imagename = image["name"]
1031
    #     if opts.force_flavorid:
1032
    #         flavorid = opts.force_flavorid
1033
    #     else:
1034
    #         flavorid = choice([f["id"] for f in DFLAVORS if f["disk"] >= 20])
1035
    #     personality = None   # FIXME
1036
    #     servername = "%s%s for %s" % (SNF_TEST_PREFIX, TEST_RUN_ID, imagename)
1037
    #     is_windows = imagename.lower().find("windows") >= 0
1038
    #     case = _spawn_server_test_case(imageid=str(imageid), flavorid=flavorid,
1039
    #                                    imagename=imagename,
1040
    #                                    personality=personality,
1041
    #                                    servername=servername,
1042
    #                                    is_windows=is_windows,
1043
    #                                    action_timeout=opts.action_timeout,
1044
    #                                    build_warning=opts.build_warning,
1045
    #                                    build_fail=opts.build_fail,
1046
    #                                    query_interval=opts.query_interval)
1047
    #     par_cases.append(case)
1048

    
1049
    # _run_cases_in_parallel(par_cases, fanout=opts.fanout, runner=runner)
1050

    
1051
if __name__ == "__main__":
1052
    sys.exit(main())