Statistics
| Branch: | Tag: | Revision:

root / snf-cyclades-app / synnefo / tools / burnin.py @ 9659e075

History | View | Annotate | Download (41.6 kB)

1
 #!/usr/bin/env python
2

    
3
# Copyright 2011 GRNET S.A. All rights reserved.
4
#
5
# Redistribution and use in source and binary forms, with or
6
# without modification, are permitted provided that the following
7
# conditions are met:
8
#
9
#   1. Redistributions of source code must retain the above
10
#      copyright notice, this list of conditions and the following
11
#      disclaimer.
12
#
13
#   2. Redistributions in binary form must reproduce the above
14
#      copyright notice, this list of conditions and the following
15
#      disclaimer in the documentation and/or other materials
16
#      provided with the distribution.
17
#
18
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
19
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
20
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
21
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
22
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
25
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
26
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
28
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29
# POSSIBILITY OF SUCH DAMAGE.
30
#
31
# The views and conclusions contained in the software and
32
# documentation are those of the authors and should not be
33
# interpreted as representing official policies, either expressed
34
# or implied, of GRNET S.A.
35

    
36
"""Perform integration testing on a running Synnefo deployment"""
37

    
38
import __main__
39
import datetime
40
import inspect
41
import logging
42
import os
43
import paramiko
44
import prctl
45
import subprocess
46
import signal
47
import socket
48
import struct
49
import sys
50
import time
51
import hashlib
52
from base64 import b64encode
53
from pwd import getpwuid
54
from grp import getgrgid
55
from IPy import IP
56
from multiprocessing import Process, Queue
57
from random import choice
58

    
59
from kamaki.clients import ClientError, ComputeClient, CycladesClient
60
from kamaki.config import Config
61

    
62
from vncauthproxy.d3des import generate_response as d3des_generate_response
63

    
64
# Use backported unittest functionality if Python < 2.7
65
try:
66
    import unittest2 as unittest
67
except ImportError:
68
    if sys.version_info < (2, 7):
69
        raise Exception("The unittest2 package is required for Python < 2.7")
70
    import unittest
71

    
72

    
73
API = None
74
TOKEN = None
75
DEFAULT_API = "http://127.0.0.1:8000/api/v1.1"
76

    
77
# A unique id identifying this test run
78
TEST_RUN_ID = datetime.datetime.strftime(datetime.datetime.now(),
79
                                         "%Y%m%d%H%M%S")
80
SNF_TEST_PREFIX = "snf-test-"
81

    
82
# Setup logging (FIXME - verigak)
83
logging.basicConfig(format="%(message)s")
84
log = logging.getLogger("burnin")
85
log.setLevel(logging.INFO)
86

    
87
class UnauthorizedTestCase(unittest.TestCase):
88
    def test_unauthorized_access(self):
89
        """Test access without a valid token fails"""
90
        falseToken = '12345'
91
        conf = Config()
92
        conf.set('compute_token', falseToken)
93
        c=ComputeClient(conf)
94

    
95
        with self.assertRaises(ClientError) as cm:
96
            c.list_servers()
97
        self.assertEqual(cm.exception.status, 401)
98

    
99

    
100
class ImagesTestCase(unittest.TestCase):
101
    """Test image lists for consistency"""
102
    @classmethod
103
    def setUpClass(cls):
104
        """Initialize kamaki, get (detailed) list of images"""
105
        log.info("Getting simple and detailed list of images")
106

    
107
        conf = Config()
108
        conf.set('compute_token', TOKEN)
109
        cls.client = ComputeClient(conf)
110
        cls.images = cls.client.list_images()
111
        cls.dimages = cls.client.list_images(detail=True)
112

    
113
    def test_001_list_images(self):
114
        """Test image list actually returns images"""
115
        self.assertGreater(len(self.images), 0)
116

    
117
    def test_002_list_images_detailed(self):
118
        """Test detailed image list is the same length as list"""
119
        self.assertEqual(len(self.dimages), len(self.images))
120

    
121
    def test_003_same_image_names(self):
122
        """Test detailed and simple image list contain same names"""
123
        names = sorted(map(lambda x: x["name"], self.images))
124
        dnames = sorted(map(lambda x: x["name"], self.dimages))
125
        self.assertEqual(names, dnames)
126

    
127
    def test_004_unique_image_names(self):
128
        """Test images have unique names"""
129
        names = sorted(map(lambda x: x["name"], self.images))
130
        self.assertEqual(sorted(list(set(names))), names)
131

    
132
    def test_005_image_metadata(self):
133
        """Test every image has specific metadata defined"""
134
        keys = frozenset(["os", "description", "size"])
135
        for i in self.dimages:
136
            self.assertTrue(keys.issubset(i["metadata"]["values"].keys()))
137

    
138

    
139
class FlavorsTestCase(unittest.TestCase):
140
    """Test flavor lists for consistency"""
141
    @classmethod
142
    def setUpClass(cls):
143
        """Initialize kamaki, get (detailed) list of flavors"""
144
        log.info("Getting simple and detailed list of flavors")
145

    
146
        conf = Config()
147
        conf.set('compute_token', TOKEN)
148
        cls.client = ComputeClient(conf)
149
        cls.flavors = cls.client.list_flavors()
150
        cls.dflavors = cls.client.list_flavors(detail=True)
151

    
152
    def test_001_list_flavors(self):
153
        """Test flavor list actually returns flavors"""
154
        self.assertGreater(len(self.flavors), 0)
155

    
156
    def test_002_list_flavors_detailed(self):
157
        """Test detailed flavor list is the same length as list"""
158
        self.assertEquals(len(self.dflavors), len(self.flavors))
159

    
160
    def test_003_same_flavor_names(self):
161
        """Test detailed and simple flavor list contain same names"""
162
        names = sorted(map(lambda x: x["name"], self.flavors))
163
        dnames = sorted(map(lambda x: x["name"], self.dflavors))
164
        self.assertEqual(names, dnames)
165

    
166
    def test_004_unique_flavor_names(self):
167
        """Test flavors have unique names"""
168
        names = sorted(map(lambda x: x["name"], self.flavors))
169
        self.assertEqual(sorted(list(set(names))), names)
170

    
171
    def test_005_well_formed_flavor_names(self):
172
        """Test flavors have names of the form CxxRyyDzz
173

174
        Where xx is vCPU count, yy is RAM in MiB, zz is Disk in GiB
175

176
        """
177
        for f in self.dflavors:
178
            self.assertEqual("C%dR%dD%d" % (f["cpu"], f["ram"], f["disk"]),
179
                             f["name"],
180
                             "Flavor %s does not match its specs." % f["name"])
181

    
182

    
183
class ServersTestCase(unittest.TestCase):
184
    """Test server lists for consistency"""
185
    @classmethod
186
    def setUpClass(cls):
187
        """Initialize kamaki, get (detailed) list of servers"""
188
        log.info("Getting simple and detailed list of servers")
189

    
190
        conf = Config()
191
        conf.set('compute_token', TOKEN)
192
        cls.client = ComputeClient(conf)
193
        cls.servers = cls.client.list_servers()
194
        cls.dservers = cls.client.list_servers(detail=True)
195

    
196
    def test_001_list_servers(self):
197
        """Test server list actually returns servers"""
198
        self.assertGreater(len(self.servers), 0)
199

    
200
    def test_002_list_servers_detailed(self):
201
        """Test detailed server list is the same length as list"""
202
        self.assertEqual(len(self.dservers), len(self.servers))
203

    
204
    def test_003_same_server_names(self):
205
        """Test detailed and simple flavor list contain same names"""
206
        names = sorted(map(lambda x: x["name"], self.servers))
207
        dnames = sorted(map(lambda x: x["name"], self.dservers))
208
        self.assertEqual(names, dnames)
209

    
210

    
211
# This class gets replicated into actual TestCases dynamically
212
class SpawnServerTestCase(unittest.TestCase):
213
    """Test scenario for server of the specified image"""
214

    
215
    @classmethod
216
    def setUpClass(cls):
217
        """Initialize a kamaki instance"""
218
        log.info("Spawning server for image `%s'", cls.imagename)
219

    
220
        conf = Config()
221
        conf.set('compute_token', TOKEN)
222
        cls.client = ComputeClient(conf)
223
        cls.cyclades = CycladesClient(conf)
224

    
225
    def _get_ipv4(self, server):
226
        """Get the public IPv4 of a server from the detailed server info"""
227

    
228
        public_addrs = filter(lambda x: x["id"] == "public",
229
                              server["addresses"]["values"])
230
        self.assertEqual(len(public_addrs), 1)
231
        ipv4_addrs = filter(lambda x: x["version"] == 4,
232
                            public_addrs[0]["values"])
233
        self.assertEqual(len(ipv4_addrs), 1)
234
        return ipv4_addrs[0]["addr"]
235

    
236
    def _get_ipv6(self, server):
237
        """Get the public IPv6 of a server from the detailed server info"""
238
        public_addrs = filter(lambda x: x["id"] == "public",
239
                              server["addresses"]["values"])
240
        self.assertEqual(len(public_addrs), 1)
241
        ipv6_addrs = filter(lambda x: x["version"] == 6,
242
                            public_addrs[0]["values"])
243
        self.assertEqual(len(ipv6_addrs), 1)
244
        return ipv6_addrs[0]["addr"]
245

    
246
    def _connect_loginname(self, os):
247
        """Return the login name for connections based on the server OS"""
248
        if os in ("Ubuntu", "Kubuntu", "Fedora"):
249
            return "user"
250
        elif os in ("windows", "windows_alpha1"):
251
            return "Administrator"
252
        else:
253
            return "root"
254

    
255
    def _verify_server_status(self, current_status, new_status):
256
        """Verify a server has switched to a specified status"""
257
        server = self.client.get_server_details(self.serverid)
258
        if server["status"] not in (current_status, new_status):
259
            return None  # Do not raise exception, return so the test fails
260
        self.assertEquals(server["status"], new_status)
261

    
262
    def _get_connected_tcp_socket(self, family, host, port):
263
        """Get a connected socket from the specified family to host:port"""
264
        sock = None
265
        for res in \
266
            socket.getaddrinfo(host, port, family, socket.SOCK_STREAM, 0,
267
                               socket.AI_PASSIVE):
268
            af, socktype, proto, canonname, sa = res
269
            try:
270
                sock = socket.socket(af, socktype, proto)
271
            except socket.error as msg:
272
                sock = None
273
                continue
274
            try:
275
                sock.connect(sa)
276
            except socket.error as msg:
277
                sock.close()
278
                sock = None
279
                continue
280
        self.assertIsNotNone(sock)
281
        return sock
282

    
283
    def _ping_once(self, ipv6, ip):
284
        """Test server responds to a single IPv4 or IPv6 ping"""
285
        cmd = "ping%s -c 2 -w 3 %s" % ("6" if ipv6 else "", ip)
286
        ping = subprocess.Popen(cmd, shell=True,
287
                                stdout=subprocess.PIPE, stderr=subprocess.PIPE)
288
        (stdout, stderr) = ping.communicate()
289
        ret = ping.wait()
290
        self.assertEquals(ret, 0)
291

    
292
    def _get_hostname_over_ssh(self, hostip, username, password):
293
        ssh = paramiko.SSHClient()
294
        ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
295
        try:
296
            ssh.connect(hostip, username=username, password=password)
297
        except socket.error:
298
            raise AssertionError
299
        stdin, stdout, stderr = ssh.exec_command("hostname")
300
        lines = stdout.readlines()
301
        self.assertEqual(len(lines), 1)
302
        return lines[0]
303

    
304
    def _try_until_timeout_expires(self, warn_timeout, fail_timeout,
305
                                   opmsg, callable, *args, **kwargs):
306
        if warn_timeout == fail_timeout:
307
            warn_timeout = fail_timeout + 1
308
        warn_tmout = time.time() + warn_timeout
309
        fail_tmout = time.time() + fail_timeout
310
        while True:
311
            self.assertLess(time.time(), fail_tmout,
312
                            "operation `%s' timed out" % opmsg)
313
            if time.time() > warn_tmout:
314
                log.warning("Server %d: `%s' operation `%s' not done yet",
315
                            self.serverid, self.servername, opmsg)
316
            try:
317
                log.info("%s... " % opmsg)
318
                return callable(*args, **kwargs)
319
            except AssertionError:
320
                pass
321
            time.sleep(self.query_interval)
322

    
323
    def _insist_on_tcp_connection(self, family, host, port):
324
        familystr = {socket.AF_INET: "IPv4", socket.AF_INET6: "IPv6",
325
                     socket.AF_UNSPEC: "Unspecified-IPv4/6"}
326
        msg = "connect over %s to %s:%s" % \
327
              (familystr.get(family, "Unknown"), host, port)
328
        sock = self._try_until_timeout_expires(
329
                self.action_timeout, self.action_timeout,
330
                msg, self._get_connected_tcp_socket,
331
                family, host, port)
332
        return sock
333

    
334
    def _insist_on_status_transition(self, current_status, new_status,
335
                                    fail_timeout, warn_timeout=None):
336
        msg = "Server %d: `%s', waiting for %s -> %s" % \
337
              (self.serverid, self.servername, current_status, new_status)
338
        if warn_timeout is None:
339
            warn_timeout = fail_timeout
340
        self._try_until_timeout_expires(warn_timeout, fail_timeout,
341
                                        msg, self._verify_server_status,
342
                                        current_status, new_status)
343
        # Ensure the status is actually the expected one
344
        server = self.client.get_server_details(self.serverid)
345
        self.assertEquals(server["status"], new_status)
346

    
347
    def _insist_on_ssh_hostname(self, hostip, username, password):
348
        msg = "SSH to %s, as %s/%s" % (hostip, username, password)
349
        hostname = self._try_until_timeout_expires(
350
                self.action_timeout, self.action_timeout,
351
                msg, self._get_hostname_over_ssh,
352
                hostip, username, password)
353

    
354
        # The hostname must be of the form 'prefix-id'
355
        self.assertTrue(hostname.endswith("-%d\n" % self.serverid))
356

    
357

    
358
    def _file_md5(filename, block_size = 2**20):
359
        f = open(filename)
360
        md5 = hashlib.md5()
361
        while True:
362
            data = f.read(block_size)
363
            if not data:
364
                break
365
            md5.update(data)
366
            f.close()
367
    
368
        return md5.digest()
369

    
370
    
371
    def _check_file_through_ssh(self, hostip, username, password, remotepath, content):
372
        msg = "Trying file injection through SSH to %s, as %s/%s" % (hostip, username, password)
373
        try:
374
            ssh = paramiko.SSHClient()
375
            ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
376
            ssh.connect(hostip, username=username, password=password)
377
        except socket.error:
378
            raise AssertionError
379
        
380
        transport = paramiko.Transport((hostip,22))
381
        transport.connect(username = username, password = password)
382

    
383
        localpath = '/tmp/'+SNF_TEST_PREFIX+'injection'
384
        sftp = paramiko.SFTPClient.from_transport(transport)
385
        sftp.get(remotepath, localpath)
386
        
387
        sftp.close()
388
        transport.close()
389

    
390
        f = open(localpath)
391
        remote_content = b64encode(f.read())
392

    
393
        # Check if files are the same
394
        return remote_content == content
395

    
396
    def _skipIf(self, condition, msg):
397
        if condition:
398
            self.skipTest(msg)
399

    
400
    def test_001_submit_create_server(self):
401
        """Test submit create server request"""
402
        server = self.client.create_server(self.servername, self.flavorid,
403
                                           self.imageid, self.personality)
404

    
405
        self.assertEqual(server["name"], self.servername)
406
        self.assertEqual(server["flavorRef"], self.flavorid)
407
        self.assertEqual(server["imageRef"], self.imageid)
408
        self.assertEqual(server["status"], "BUILD")
409

    
410
        # Update class attributes to reflect data on building server
411
        cls = type(self)
412
        cls.serverid = server["id"]
413
        cls.username = None
414
        cls.passwd = server["adminPass"]
415

    
416
    def test_002a_server_is_building_in_list(self):
417
        """Test server is in BUILD state, in server list"""
418
        servers = self.client.list_servers(detail=True)
419
        servers = filter(lambda x: x["name"] == self.servername, servers)
420
        self.assertEqual(len(servers), 1)
421
        server = servers[0]
422
        self.assertEqual(server["name"], self.servername)
423
        self.assertEqual(server["flavorRef"], self.flavorid)
424
        self.assertEqual(server["imageRef"], self.imageid)
425
        self.assertEqual(server["status"], "BUILD")
426

    
427
    def test_002b_server_is_building_in_details(self):
428
        """Test server is in BUILD state, in details"""
429
        server = self.client.get_server_details(self.serverid)
430
        self.assertEqual(server["name"], self.servername)
431
        self.assertEqual(server["flavorRef"], self.flavorid)
432
        self.assertEqual(server["imageRef"], self.imageid)
433
        self.assertEqual(server["status"], "BUILD")
434

    
435
    def test_002c_set_server_metadata(self):
436
        image = self.client.get_image_details(self.imageid)
437
        os = image["metadata"]["values"]["os"]
438
        loginname = image["metadata"]["values"].get("users", None)
439
        self.client.update_server_metadata(self.serverid, OS=os)
440

    
441
        # Determine the username to use for future connections
442
        # to this host
443
        cls = type(self)
444
        cls.username = loginname
445
        if not cls.username:
446
            cls.username = self._connect_loginname(os)
447
        self.assertIsNotNone(cls.username)
448

    
449
    def test_002d_verify_server_metadata(self):
450
        """Test server metadata keys are set based on image metadata"""
451
        servermeta = self.client.get_server_metadata(self.serverid)
452
        imagemeta = self.client.get_image_metadata(self.imageid)
453
        self.assertEqual(servermeta["OS"], imagemeta["os"])
454

    
455
    def test_003_server_becomes_active(self):
456
        """Test server becomes ACTIVE"""
457
        self._insist_on_status_transition("BUILD", "ACTIVE",
458
                                         self.build_fail, self.build_warning)
459

    
460
    # def test_003a_get_server_oob_console(self):
461
    #     """Test getting OOB server console over VNC
462

    
463
    #     Implementation of RFB protocol follows
464
    #     http://www.realvnc.com/docs/rfbproto.pdf.
465

    
466
    #     """
467
        
468
    #     console = self.cyclades.get_server_console(self.serverid)
469
    #     self.assertEquals(console['type'], "vnc")
470
    #     sock = self._insist_on_tcp_connection(socket.AF_UNSPEC,
471
    #                                     console["host"], console["port"])
472

    
473
    #     # Step 1. ProtocolVersion message (par. 6.1.1)
474
    #     version = sock.recv(1024)
475
    #     self.assertEquals(version, 'RFB 003.008\n')
476
    #     sock.send(version)
477

    
478
    #     # Step 2. Security (par 6.1.2): Only VNC Authentication supported
479
    #     sec = sock.recv(1024)
480
    #     self.assertEquals(list(sec), ['\x01', '\x02'])
481

    
482
    #     # Step 3. Request VNC Authentication (par 6.1.2)
483
    #     sock.send('\x02')
484

    
485
    #     # Step 4. Receive Challenge (par 6.2.2)
486
    #     challenge = sock.recv(1024)
487
    #     self.assertEquals(len(challenge), 16)
488

    
489
    #     # Step 5. DES-Encrypt challenge, use password as key (par 6.2.2)
490
    #     response = d3des_generate_response(
491
    #         (console["password"] + '\0' * 8)[:8], challenge)
492
    #     sock.send(response)
493

    
494
    #     # Step 6. SecurityResult (par 6.1.3)
495
    #     result = sock.recv(4)
496
    #     self.assertEquals(list(result), ['\x00', '\x00', '\x00', '\x00'])
497
    #     sock.close()
498
        
499
    def test_004_server_has_ipv4(self):
500
        """Test active server has a valid IPv4 address"""
501
        server = self.client.get_server_details(self.serverid)
502
        ipv4 = self._get_ipv4(server)
503
        self.assertEquals(IP(ipv4).version(), 4)
504

    
505
    # def test_005_server_has_ipv6(self):
506
    #     """Test active server has a valid IPv6 address"""
507
    #     server = self.client.get_server_details(self.serverid)
508
    #     ipv6 = self._get_ipv6(server)
509
    #     self.assertEquals(IP(ipv6).version(), 6)
510

    
511
    def test_006_server_responds_to_ping_IPv4(self):
512
        """Test server responds to ping on IPv4 address"""
513
        server = self.client.get_server_details(self.serverid)
514
        ip = self._get_ipv4(server)
515
        self._try_until_timeout_expires(self.action_timeout,
516
                                        self.action_timeout,
517
                                        "PING IPv4 to %s" % ip,
518
                                        self._ping_once,
519
                                        False, ip)
520

    
521
    # def test_007_server_responds_to_ping_IPv6(self):
522
    #     """Test server responds to ping on IPv6 address"""
523
    #     server = self.client.get_server_details(self.serverid)
524
    #     ip = self._get_ipv6(server)
525
    #     self._try_until_timeout_expires(self.action_timeout,
526
    #                                     self.action_timeout,
527
    #                                     "PING IPv6 to %s" % ip,
528
    #                                     self._ping_once,
529
    #                                     True, ip)
530

    
531
    def test_008_submit_shutdown_request(self):
532
        """Test submit request to shutdown server"""
533
        self.cyclades.shutdown_server(self.serverid)
534

    
535
    def test_009_server_becomes_stopped(self):
536
        """Test server becomes STOPPED"""
537
        self._insist_on_status_transition("ACTIVE", "STOPPED",
538
                                         self.action_timeout,
539
                                         self.action_timeout)
540

    
541
    def test_010_submit_start_request(self):
542
        """Test submit start server request"""
543
        self.cyclades.start_server(self.serverid)
544

    
545
    def test_011_server_becomes_active(self):
546
        """Test server becomes ACTIVE again"""
547
        self._insist_on_status_transition("STOPPED", "ACTIVE",
548
                                         self.action_timeout,
549
                                         self.action_timeout)
550

    
551
    def test_011a_server_responds_to_ping_IPv4(self):
552
        """Test server OS is actually up and running again"""
553
        self.test_006_server_responds_to_ping_IPv4()
554

    
555
    def test_012_ssh_to_server_IPv4(self):
556
        """Test SSH to server public IPv4 works, verify hostname"""
557
        self._skipIf(self.is_windows, "only valid for Linux servers")
558
        server = self.client.get_server_details(self.serverid)
559
        self._insist_on_ssh_hostname(self._get_ipv4(server),
560
                                     self.username, self.passwd)
561

    
562
    # def test_013_ssh_to_server_IPv6(self):
563
    #     """Test SSH to server public IPv6 works, verify hostname"""
564
    #     self._skipIf(self.is_windows, "only valid for Linux servers")
565
    #     server = self.client.get_server_details(self.serverid)
566
    #     self._insist_on_ssh_hostname(self._get_ipv6(server),
567
    #                                  self.username, self.passwd)
568

    
569
    def test_014_rdp_to_server_IPv4(self):
570
        "Test RDP connection to server public IPv4 works"""
571
        self._skipIf(not self.is_windows, "only valid for Windows servers")
572
        server = self.client.get_server_details(self.serverid)
573
        ipv4 = self._get_ipv4(server)
574
        sock = _insist_on_tcp_connection(socket.AF_INET, ipv4, 3389)
575

    
576
        # No actual RDP processing done. We assume the RDP server is there
577
        # if the connection to the RDP port is successful.
578
        # FIXME: Use rdesktop, analyze exit code? see manpage [costasd]
579
        sock.close()
580

    
581
    # def test_015_rdp_to_server_IPv6(self):
582
    #     "Test RDP connection to server public IPv6 works"""
583
    #     self._skipIf(not self.is_windows, "only valid for Windows servers")
584
    #     server = self.client.get_server_details(self.serverid)
585
    #     ipv6 = self._get_ipv6(server)
586
    #     sock = _get_tcp_connection(socket.AF_INET6, ipv6, 3389)
587

    
588
    #     # No actual RDP processing done. We assume the RDP server is there
589
    #     # if the connection to the RDP port is successful.
590
    #     sock.close()
591

    
592
    def test_016_personality_is_enforced(self):
593
        """Test file injection for personality enforcement"""
594
        self._skipIf(self.is_windows, "only implemented for Linux servers")
595
        self._skipIf(self.personality == None, "No personality file selected")
596

    
597
        server = self.client.get_server_details(self.serverid)
598

    
599
        for inj_file in self.personality:
600
            equal_files = self._check_file_through_ssh(self._get_ipv4(server), inj_file['owner'], 
601
                                                       self.passwd, inj_file['path'], inj_file['contents'])
602
            self.assertTrue(equal_files)
603
        
604

    
605
    def test_017_submit_delete_request(self):
606
        """Test submit request to delete server"""
607
        self.client.delete_server(self.serverid)
608

    
609
    def test_018_server_becomes_deleted(self):
610
        """Test server becomes DELETED"""
611
        self._insist_on_status_transition("ACTIVE", "DELETED",
612
                                         self.action_timeout,
613
                                         self.action_timeout)
614

    
615
    def test_019_server_no_longer_in_server_list(self):
616
        """Test server is no longer in server list"""
617
        servers = self.client.list_servers()
618
        self.assertNotIn(self.serverid, [s["id"] for s in servers])
619

    
620

    
621
class NetworkTestCase(unittest.TestCase):
622
    """ Testing networking in cyclades """
623
    @classmethod
624
    def setUpClass(cls):
625
        "Initialize kamaki, get list of current networks"
626
        conf = Config()
627
        conf.set('compute_token', TOKEN)
628
        cls.client = CycladesClient(conf)
629
        cls.compute = ComputeClient(conf)
630

    
631
        images = cls.compute.list_images(detail = True)
632
        flavors = cls.compute.list_flavors(detail = True)
633
        imageid = choice([im['id'] for im in images])
634
        flavorid = choice([f["id"] for f in flavors if f["disk"] >= 20])
635

    
636
        for image in images:
637
            if image['id'] == imageid:
638
                imagename = image['name']
639

    
640
        servername = "%s%s for %s" % (SNF_TEST_PREFIX, TEST_RUN_ID, imagename)
641
        is_windows = imagename.lower().find("windows") >= 0
642
        setupCase =  _spawn_server_test_case(imageid=str(imageid), flavorid=flavorid,
643
                                             imagename=imagename,
644
                                             personality=None,
645
                                             servername=servername,
646
                                             is_windows=is_windows,
647
                                             action_timeout=200,
648
                                             build_warning=1200,
649
                                             build_fail=500,
650
                                             query_interval=3)
651

    
652
        #Using already implemented tests for serverlist population
653
        suite = unittest.TestSuite()
654
        suite.addTest(setupCase('test_001_submit_create_server'))
655
        suite.addTest(setupCase('test_002a_server_is_building_in_list'))
656
        suite.addTest(setupCase('test_002b_server_is_building_in_details'))        
657
        suite.addTest(setupCase('test_003_server_becomes_active'))
658
        unittest.TextTestRunner(verbosity=2).run(suite)
659

    
660
    def test_001_create_network(self):
661
        """Test submit create network request"""
662
        name = SNF_TEST_PREFIX+TEST_RUN_ID
663
        network =  self.client.create_network(name)        
664
        previous_num = len(self.client.list_networks())
665

    
666
        #Test if right name is assigned
667
        self.assertEqual(network['name'], name)
668
        
669
        # Update class attributes
670
        cls = type(self)
671
        cls.networkid = network['id']
672
        networks = self.client.list_networks()
673

    
674
        #Test if new network is created
675
        self.assertTrue(len(networks) > previous_num)
676
        
677
    
678
    def test_002_connect_to_network(self):
679
        """Test VM to network connection"""
680
        servers = self.compute.list_servers()
681
        server = choice(servers)
682
        self.client.connect_server(server['id'], self.networkid)
683
        
684
        #Update class attributes
685
        cls = type(self)
686
        cls.serverid = server['id']
687

    
688
        connected = (self.client.get_network_details(self.networkid))
689
        connections = len(connected['servers']['values'])
690
        
691
        time.sleep(60)
692

    
693
        #FIXME: Insist on new connection
694
        self.assertTrue(connections>=1)
695
        
696

    
697
    def test_003_disconnect_from_network(self):
698
        prev_state = (self.client.get_network_details(self.networkid))
699
        prev_conn = len(prev_state['servers']['values'])
700

    
701
        self.client.disconnect_server(self.serverid, self.networkid)
702
        connected = (self.client.get_network_details(self.networkid))
703
        curr_conn = len(connected['servers']['values'])
704

    
705
        #FIXME: Insist on deleting
706
        self.assertTrue(curr_conn < prev_conn)
707

    
708
    def test_004_destroy_network(self):
709
        """Test submit delete network request"""
710
        self.client.delete_network(self.networkid)
711
        
712
        networks = self.client.list_networks()
713
        self.assertEqual(len(networks),1)
714

    
715

    
716
class TestRunnerProcess(Process):
717
    """A distinct process used to execute part of the tests in parallel"""
718
    def __init__(self, **kw):
719
        Process.__init__(self, **kw)
720
        kwargs = kw["kwargs"]
721
        self.testq = kwargs["testq"]
722
        self.runner = kwargs["runner"]
723

    
724
    def run(self):
725
        # Make sure this test runner process dies with the parent
726
        # and is not left behind.
727
        #
728
        # WARNING: This uses the prctl(2) call and is
729
        # Linux-specific.
730
        prctl.set_pdeathsig(signal.SIGHUP)
731

    
732
        while True:
733
            log.debug("I am process %d, GETting from queue is %s",
734
                     os.getpid(), self.testq)
735
            msg = self.testq.get()
736
            log.debug("Dequeued msg: %s", msg)
737

    
738
            if msg == "TEST_RUNNER_TERMINATE":
739
                raise SystemExit
740
            elif issubclass(msg, unittest.TestCase):
741
                # Assemble a TestSuite, and run it
742
                suite = unittest.TestLoader().loadTestsFromTestCase(msg)
743
                self.runner.run(suite)
744
            else:
745
                raise Exception("Cannot handle msg: %s" % msg)
746

    
747

    
748

    
749
def _run_cases_in_parallel(cases, fanout=1, runner=None):
750
    """Run instances of TestCase in parallel, in a number of distinct processes
751

752
    The cases iterable specifies the TestCases to be executed in parallel,
753
    by test runners running in distinct processes.
754
    The fanout parameter specifies the number of processes to spawn,
755
    and defaults to 1.
756
    The runner argument specifies the test runner class to use inside each
757
    runner process.
758

759
    """
760
    if runner is None:
761
        runner = unittest.TextTestRunner(verbosity=2, failfast=True)
762

    
763
    # testq: The master process enqueues TestCase objects into this queue,
764
    #        test runner processes pick them up for execution, in parallel.
765
    testq = Queue()
766
    runners = []
767
    for i in xrange(0, fanout):
768
        kwargs = dict(testq=testq, runner=runner)
769
        runners.append(TestRunnerProcess(kwargs=kwargs))
770

    
771
    log.info("Spawning %d test runner processes", len(runners))
772
    for p in runners:
773
        p.start()
774
    log.debug("Spawned %d test runners, PIDs are %s",
775
              len(runners), [p.pid for p in runners])
776

    
777
    # Enqueue test cases
778
    map(testq.put, cases)
779
    map(testq.put, ["TEST_RUNNER_TERMINATE"] * len(runners))
780

    
781
    log.debug("Joining %d processes", len(runners))
782
    for p in runners:
783
        p.join()
784
    log.debug("Done joining %d processes", len(runners))
785

    
786

    
787
def _spawn_server_test_case(**kwargs):
788
    """Construct a new unit test case class from SpawnServerTestCase"""
789

    
790
    name = "SpawnServerTestCase_%s" % kwargs["imageid"]
791
    cls = type(name, (SpawnServerTestCase,), kwargs)
792

    
793
    # Patch extra parameters into test names by manipulating method docstrings
794
    for (mname, m) in \
795
        inspect.getmembers(cls, lambda x: inspect.ismethod(x)):
796
            if hasattr(m, __doc__):
797
                m.__func__.__doc__ = "[%s] %s" % (imagename, m.__doc__)
798

    
799
    # Make sure the class can be pickled, by listing it among
800
    # the attributes of __main__. A PicklingError is raised otherwise.
801
    setattr(__main__, name, cls)
802
    return cls
803

    
804

    
805
def cleanup_servers(delete_stale=False):
806

    
807
    conf = Config()
808
    conf.set('compute_token', TOKEN)
809
    c = ComputeClient(conf)
810

    
811
    servers = c.list_servers()
812
    stale = [s for s in servers if s["name"].startswith(SNF_TEST_PREFIX)]
813

    
814
    if len(stale) == 0:
815
        return
816

    
817
    print >> sys.stderr, "Found these stale servers from previous runs:"
818
    print "    " + \
819
          "\n    ".join(["%d: %s" % (s["id"], s["name"]) for s in stale])
820

    
821
    if delete_stale:
822
        print >> sys.stderr, "Deleting %d stale servers:" % len(stale)
823
        for server in stale:
824
            c.delete_server(server["id"])
825
        print >> sys.stderr, "    ...done"
826
    else:
827
        print >> sys.stderr, "Use --delete-stale to delete them."
828

    
829

    
830
def parse_arguments(args):
831
    from optparse import OptionParser
832

    
833
    kw = {}
834
    kw["usage"] = "%prog [options]"
835
    kw["description"] = \
836
        "%prog runs a number of test scenarios on a " \
837
        "Synnefo deployment."
838

    
839
    parser = OptionParser(**kw)
840
    parser.disable_interspersed_args()
841
    parser.add_option("--api",
842
                      action="store", type="string", dest="api",
843
                      help="The API URI to use to reach the Synnefo API",
844
                      default=DEFAULT_API)
845
    parser.add_option("--token",
846
                      action="store", type="string", dest="token",
847
                      help="The token to use for authentication to the API")
848
    parser.add_option("--nofailfast",
849
                      action="store_true", dest="nofailfast",
850
                      help="Do not fail immediately if one of the tests " \
851
                           "fails (EXPERIMENTAL)",
852
                      default=False)
853
    parser.add_option("--action-timeout",
854
                      action="store", type="int", dest="action_timeout",
855
                      metavar="TIMEOUT",
856
                      help="Wait SECONDS seconds for a server action to " \
857
                           "complete, then the test is considered failed",
858
                      default=100)
859
    parser.add_option("--build-warning",
860
                      action="store", type="int", dest="build_warning",
861
                      metavar="TIMEOUT",
862
                      help="Warn if TIMEOUT seconds have passed and a " \
863
                           "build operation is still pending",
864
                      default=600)
865
    parser.add_option("--build-fail",
866
                      action="store", type="int", dest="build_fail",
867
                      metavar="BUILD_TIMEOUT",
868
                      help="Fail the test if TIMEOUT seconds have passed " \
869
                           "and a build operation is still incomplete",
870
                      default=900)
871
    parser.add_option("--query-interval",
872
                      action="store", type="int", dest="query_interval",
873
                      metavar="INTERVAL",
874
                      help="Query server status when requests are pending " \
875
                           "every INTERVAL seconds",
876
                      default=3)
877
    parser.add_option("--fanout",
878
                      action="store", type="int", dest="fanout",
879
                      metavar="COUNT",
880
                      help="Spawn up to COUNT child processes to execute " \
881
                           "in parallel, essentially have up to COUNT " \
882
                           "server build requests outstanding (EXPERIMENTAL)",
883
                      default=1)
884
    parser.add_option("--force-flavor",
885
                      action="store", type="int", dest="force_flavorid",
886
                      metavar="FLAVOR ID",
887
                      help="Force all server creations to use the specified "\
888
                           "FLAVOR ID instead of a randomly chosen one, " \
889
                           "useful if disk space is scarce",
890
                      default=None)
891
    parser.add_option("--image-id",
892
                      action="store", type="string", dest="force_imageid",
893
                      metavar="IMAGE ID",
894
                      help="Test the specified image id, use 'all' to test " \
895
                           "all available images (mandatory argument)",
896
                      default=None)
897
    parser.add_option("--show-stale",
898
                      action="store_true", dest="show_stale",
899
                      help="Show stale servers from previous runs, whose "\
900
                           "name starts with `%s'" % SNF_TEST_PREFIX,
901
                      default=False)
902
    parser.add_option("--delete-stale",
903
                      action="store_true", dest="delete_stale",
904
                      help="Delete stale servers from previous runs, whose "\
905
                           "name starts with `%s'" % SNF_TEST_PREFIX,
906
                      default=False)
907
    parser.add_option("--force-personality",
908
                      action="store", dest="personality_path",
909
                      help="Force a personality file injection. File path required. ",
910
                      default=None)
911
    
912

    
913
    # FIXME: Change the default for build-fanout to 10
914
    # FIXME: Allow the user to specify a specific set of Images to test
915

    
916
    (opts, args) = parser.parse_args(args)
917

    
918
    # Verify arguments
919
    if opts.delete_stale:
920
        opts.show_stale = True
921

    
922
    if not opts.show_stale:
923
        if not opts.force_imageid:
924
            print >>sys.stderr, "The --image-id argument is mandatory."
925
            parser.print_help()
926
            sys.exit(1)
927

    
928
        if opts.force_imageid != 'all':
929
            try:
930
                opts.force_imageid = str(opts.force_imageid)
931
            except ValueError:
932
                print >>sys.stderr, "Invalid value specified for --image-id." \
933
                                    "Use a valid id, or `all'."
934
                sys.exit(1)
935

    
936
    return (opts, args)
937

    
938

    
939
def main():
940
    """Assemble test cases into a test suite, and run it
941

942
    IMPORTANT: Tests have dependencies and have to be run in the specified
943
    order inside a single test case. They communicate through attributes of the
944
    corresponding TestCase class (shared fixtures). Distinct subclasses of
945
    TestCase MAY SHARE NO DATA, since they are run in parallel, in distinct
946
    test runner processes.
947

948
    """
949
    (opts, args) = parse_arguments(sys.argv[1:])
950

    
951
    global API, TOKEN
952
    API = opts.api
953
    TOKEN = opts.token
954

    
955
    # Cleanup stale servers from previous runs
956
    if opts.show_stale:
957
        cleanup_servers(delete_stale=opts.delete_stale)
958
        return 0
959

    
960
    # Initialize a kamaki instance, get flavors, images
961

    
962
    conf = Config()
963
    conf.set('compute_token', TOKEN)
964
    c = ComputeClient(conf)
965

    
966
    DIMAGES = c.list_images(detail=True)
967
    DFLAVORS = c.list_flavors(detail=True)
968

    
969
    # FIXME: logging, log, LOG PID, TEST_RUN_ID, arguments
970
    # Run them: FIXME: In parallel, FAILEARLY, catchbreak?
971
    #unittest.main(verbosity=2, catchbreak=True)
972

    
973
    if opts.force_imageid == 'all':
974
        test_images = DIMAGES
975
    else:
976
        test_images = filter(lambda x: x["id"] == opts.force_imageid, DIMAGES)
977

    
978
    for image in test_images:
979
        imageid = str(image["id"])
980
        flavorid = choice([f["id"] for f in DFLAVORS if f["disk"] >= 20])
981
        imagename = image["name"]
982
        
983
        
984
        if opts.personality_path != None:
985
            f = open(opts.personality_path)
986
            content = b64encode(f.read())
987
            personality = []
988
            st = os.stat(opts.personality_path)
989
            personality.append({
990
                    'path': '/root/test_inj_file',
991
                    'owner': 'root',
992
                    'group': 'root',
993
                    'mode': 0x7777 & st.st_mode,
994
                    'contents': content
995
                    })
996
        else:
997
            personality = None
998

    
999
        servername = "%s%s for %s" % (SNF_TEST_PREFIX, TEST_RUN_ID, imagename)
1000
        is_windows = imagename.lower().find("windows") >= 0
1001
        
1002
        ServerTestCase = _spawn_server_test_case(imageid=imageid, flavorid=flavorid,
1003
                                                 imagename=imagename,
1004
                                                 personality=personality,
1005
                                                 servername=servername,
1006
                                                 is_windows=is_windows,
1007
                                                 action_timeout=opts.action_timeout,
1008
                                                 build_warning=opts.build_warning,
1009
                                                 build_fail=opts.build_fail,
1010
                                                 query_interval=opts.query_interval,
1011
                                                 )
1012

    
1013

    
1014
    #Running all the testcases sequentially
1015
    #seq_cases = [UnauthorizedTestCase, FlavorsTestCase, ImagesTestCase, ServerTestCase, NetworkTestCase]
1016

    
1017
    seq_cases = [ServerTestCase]
1018
    for case in seq_cases:
1019
        suite = unittest.TestLoader().loadTestsFromTestCase(case)
1020
        unittest.TextTestRunner(verbosity=2).run(suite)
1021
        
1022
    
1023

    
1024
    # # The Following cases run sequentially
1025
    # seq_cases = [UnauthorizedTestCase, FlavorsTestCase, ImagesTestCase]
1026
    # _run_cases_in_parallel(seq_cases, fanout=3, runner=runner)
1027

    
1028
    # # The following cases run in parallel
1029
    # par_cases = []
1030

    
1031
    # if opts.force_imageid == 'all':
1032
    #     test_images = DIMAGES
1033
    # else:
1034
    #     test_images = filter(lambda x: x["id"] == opts.force_imageid, DIMAGES)
1035

    
1036
    # for image in test_images:
1037
    #     imageid = image["id"]
1038
    #     imagename = image["name"]
1039
    #     if opts.force_flavorid:
1040
    #         flavorid = opts.force_flavorid
1041
    #     else:
1042
    #         flavorid = choice([f["id"] for f in DFLAVORS if f["disk"] >= 20])
1043
    #     personality = None   # FIXME
1044
    #     servername = "%s%s for %s" % (SNF_TEST_PREFIX, TEST_RUN_ID, imagename)
1045
    #     is_windows = imagename.lower().find("windows") >= 0
1046
    #     case = _spawn_server_test_case(imageid=str(imageid), flavorid=flavorid,
1047
    #                                    imagename=imagename,
1048
    #                                    personality=personality,
1049
    #                                    servername=servername,
1050
    #                                    is_windows=is_windows,
1051
    #                                    action_timeout=opts.action_timeout,
1052
    #                                    build_warning=opts.build_warning,
1053
    #                                    build_fail=opts.build_fail,
1054
    #                                    query_interval=opts.query_interval)
1055
    #     par_cases.append(case)
1056

    
1057
    # _run_cases_in_parallel(par_cases, fanout=opts.fanout, runner=runner)
1058

    
1059
if __name__ == "__main__":
1060
    sys.exit(main())