Statistics
| Branch: | Tag: | Revision:

root / snf-cyclades-app / synnefo / tools / burnin.py @ 65462ca9

History | View | Annotate | Download (43.3 kB)

1
#!/usr/bin/env python
2

    
3
# Copyright 2011 GRNET S.A. All rights reserved.
4
#
5
# Redistribution and use in source and binary forms, with or
6
# without modification, are permitted provided that the following
7
# conditions are met:
8
#
9
#   1. Redistributions of source code must retain the above
10
#      copyright notice, this list of conditions and the following
11
#      disclaimer.
12
#
13
#   2. Redistributions in binary form must reproduce the above
14
#      copyright notice, this list of conditions and the following
15
#      disclaimer in the documentation and/or other materials
16
#      provided with the distribution.
17
#
18
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
19
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
20
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
21
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
22
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
25
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
26
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
28
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29
# POSSIBILITY OF SUCH DAMAGE.
30
#
31
# The views and conclusions contained in the software and
32
# documentation are those of the authors and should not be
33
# interpreted as representing official policies, either expressed
34
# or implied, of GRNET S.A.
35

    
36
"""Perform integration testing on a running Synnefo deployment"""
37

    
38
import __main__
39
import datetime
40
import inspect
41
import logging
42
import os
43
import paramiko
44
import prctl
45
import subprocess
46
import signal
47
import socket
48
import struct
49
import sys
50
import time
51
import hashlib
52
from base64 import b64encode
53
from pwd import getpwuid
54
from grp import getgrgid
55
from IPy import IP
56
from multiprocessing import Process, Queue
57
from random import choice
58

    
59
from kamaki.clients import ClientError, ComputeClient, CycladesClient
60
from kamaki.config import Config
61

    
62
from vncauthproxy.d3des import generate_response as d3des_generate_response
63

    
64
# Use backported unittest functionality if Python < 2.7
65
try:
66
    import unittest2 as unittest
67
except ImportError:
68
    if sys.version_info < (2, 7):
69
        raise Exception("The unittest2 package is required for Python < 2.7")
70
    import unittest
71

    
72

    
73
API = None
74
TOKEN = None
75
DEFAULT_API = "http://127.0.0.1:8000/api/v1.1"
76

    
77
# A unique id identifying this test run
78
TEST_RUN_ID = datetime.datetime.strftime(datetime.datetime.now(),
79
                                         "%Y%m%d%H%M%S")
80
SNF_TEST_PREFIX = "snf-test-"
81

    
82
# Setup logging (FIXME - verigak)
83
logging.basicConfig(format="%(message)s")
84
log = logging.getLogger("burnin")
85
log.setLevel(logging.INFO)
86

    
87
class UnauthorizedTestCase(unittest.TestCase):
88
    def test_unauthorized_access(self):
89
        """Test access without a valid token fails"""
90
        falseToken = '12345'
91
        conf = Config()
92
        conf.set('compute_token', falseToken)
93
        c=ComputeClient(conf)
94

    
95
        with self.assertRaises(ClientError) as cm:
96
            c.list_servers()
97
        self.assertEqual(cm.exception.status, 401)
98

    
99

    
100
class ImagesTestCase(unittest.TestCase):
101
    """Test image lists for consistency"""
102
    @classmethod
103
    def setUpClass(cls):
104
        """Initialize kamaki, get (detailed) list of images"""
105
        log.info("Getting simple and detailed list of images")
106

    
107
        conf = Config()
108
        conf.set('compute_token', TOKEN)
109
        cls.client = ComputeClient(conf)
110
        cls.images = cls.client.list_images()
111
        cls.dimages = cls.client.list_images(detail=True)
112

    
113
    def test_001_list_images(self):
114
        """Test image list actually returns images"""
115
        self.assertGreater(len(self.images), 0)
116

    
117
    def test_002_list_images_detailed(self):
118
        """Test detailed image list is the same length as list"""
119
        self.assertEqual(len(self.dimages), len(self.images))
120

    
121
    def test_003_same_image_names(self):
122
        """Test detailed and simple image list contain same names"""
123
        names = sorted(map(lambda x: x["name"], self.images))
124
        dnames = sorted(map(lambda x: x["name"], self.dimages))
125
        self.assertEqual(names, dnames)
126

    
127
    def test_004_unique_image_names(self):
128
        """Test images have unique names"""
129
        names = sorted(map(lambda x: x["name"], self.images))
130
        self.assertEqual(sorted(list(set(names))), names)
131

    
132
    def test_005_image_metadata(self):
133
        """Test every image has specific metadata defined"""
134
        keys = frozenset(["os", "description", "size"])
135
        for i in self.dimages:
136
            self.assertTrue(keys.issubset(i["metadata"]["values"].keys()))
137

    
138

    
139
class FlavorsTestCase(unittest.TestCase):
140
    """Test flavor lists for consistency"""
141
    @classmethod
142
    def setUpClass(cls):
143
        """Initialize kamaki, get (detailed) list of flavors"""
144
        log.info("Getting simple and detailed list of flavors")
145

    
146
        conf = Config()
147
        conf.set('compute_token', TOKEN)
148
        cls.client = ComputeClient(conf)
149
        cls.flavors = cls.client.list_flavors()
150
        cls.dflavors = cls.client.list_flavors(detail=True)
151

    
152
    def test_001_list_flavors(self):
153
        """Test flavor list actually returns flavors"""
154
        self.assertGreater(len(self.flavors), 0)
155

    
156
    def test_002_list_flavors_detailed(self):
157
        """Test detailed flavor list is the same length as list"""
158
        self.assertEquals(len(self.dflavors), len(self.flavors))
159

    
160
    def test_003_same_flavor_names(self):
161
        """Test detailed and simple flavor list contain same names"""
162
        names = sorted(map(lambda x: x["name"], self.flavors))
163
        dnames = sorted(map(lambda x: x["name"], self.dflavors))
164
        self.assertEqual(names, dnames)
165

    
166
    def test_004_unique_flavor_names(self):
167
        """Test flavors have unique names"""
168
        names = sorted(map(lambda x: x["name"], self.flavors))
169
        self.assertEqual(sorted(list(set(names))), names)
170

    
171
    def test_005_well_formed_flavor_names(self):
172
        """Test flavors have names of the form CxxRyyDzz
173

174
        Where xx is vCPU count, yy is RAM in MiB, zz is Disk in GiB
175

176
        """
177
        for f in self.dflavors:
178
            self.assertEqual("C%dR%dD%d" % (f["cpu"], f["ram"], f["disk"]),
179
                             f["name"],
180
                             "Flavor %s does not match its specs." % f["name"])
181

    
182

    
183
class ServersTestCase(unittest.TestCase):
184
    """Test server lists for consistency"""
185
    @classmethod
186
    def setUpClass(cls):
187
        """Initialize kamaki, get (detailed) list of servers"""
188
        log.info("Getting simple and detailed list of servers")
189

    
190
        conf = Config()
191
        conf.set('compute_token', TOKEN)
192
        cls.client = ComputeClient(conf)
193
        cls.servers = cls.client.list_servers()
194
        cls.dservers = cls.client.list_servers(detail=True)
195

    
196
    def test_001_list_servers(self):
197
        """Test server list actually returns servers"""
198
        self.assertGreater(len(self.servers), 0)
199

    
200
    def test_002_list_servers_detailed(self):
201
        """Test detailed server list is the same length as list"""
202
        self.assertEqual(len(self.dservers), len(self.servers))
203

    
204
    def test_003_same_server_names(self):
205
        """Test detailed and simple flavor list contain same names"""
206
        names = sorted(map(lambda x: x["name"], self.servers))
207
        dnames = sorted(map(lambda x: x["name"], self.dservers))
208
        self.assertEqual(names, dnames)
209

    
210

    
211
# This class gets replicated into actual TestCases dynamically
212
class SpawnServerTestCase(unittest.TestCase):
213
    """Test scenario for server of the specified image"""
214

    
215
    @classmethod
216
    def setUpClass(cls):
217
        """Initialize a kamaki instance"""
218
        log.info("Spawning server for image `%s'", cls.imagename)
219

    
220
        conf = Config()
221
        conf.set('compute_token', TOKEN)
222
        cls.client = ComputeClient(conf)
223
        cls.cyclades = CycladesClient(conf)
224

    
225
    def _get_ipv4(self, server):
226
        """Get the public IPv4 of a server from the detailed server info"""
227

    
228
        public_addrs = filter(lambda x: x["id"] == "public",
229
                              server["addresses"]["values"])
230
        self.assertEqual(len(public_addrs), 1)
231
        ipv4_addrs = filter(lambda x: x["version"] == 4,
232
                            public_addrs[0]["values"])
233
        self.assertEqual(len(ipv4_addrs), 1)
234
        return ipv4_addrs[0]["addr"]
235

    
236
    def _get_ipv6(self, server):
237
        """Get the public IPv6 of a server from the detailed server info"""
238
        public_addrs = filter(lambda x: x["id"] == "public",
239
                              server["addresses"]["values"])
240
        self.assertEqual(len(public_addrs), 1)
241
        ipv6_addrs = filter(lambda x: x["version"] == 6,
242
                            public_addrs[0]["values"])
243
        self.assertEqual(len(ipv6_addrs), 1)
244
        return ipv6_addrs[0]["addr"]
245

    
246
    def _connect_loginname(self, os):
247
        """Return the login name for connections based on the server OS"""
248
        if os in ("Ubuntu", "Kubuntu", "Fedora"):
249
            return "user"
250
        elif os in ("windows", "windows_alpha1"):
251
            return "Administrator"
252
        else:
253
            return "root"
254

    
255
    def _verify_server_status(self, current_status, new_status):
256
        """Verify a server has switched to a specified status"""
257
        server = self.client.get_server_details(self.serverid)
258
        if server["status"] not in (current_status, new_status):
259
            return None  # Do not raise exception, return so the test fails
260
        self.assertEquals(server["status"], new_status)
261

    
262
    def _get_connected_tcp_socket(self, family, host, port):
263
        """Get a connected socket from the specified family to host:port"""
264
        sock = None
265
        for res in \
266
            socket.getaddrinfo(host, port, family, socket.SOCK_STREAM, 0,
267
                               socket.AI_PASSIVE):
268
            af, socktype, proto, canonname, sa = res
269
            try:
270
                sock = socket.socket(af, socktype, proto)
271
            except socket.error as msg:
272
                sock = None
273
                continue
274
            try:
275
                sock.connect(sa)
276
            except socket.error as msg:
277
                sock.close()
278
                sock = None
279
                continue
280
        self.assertIsNotNone(sock)
281
        return sock
282

    
283
    def _ping_once(self, ipv6, ip):
284
        """Test server responds to a single IPv4 or IPv6 ping"""
285
        cmd = "ping%s -c 2 -w 3 %s" % ("6" if ipv6 else "", ip)
286
        ping = subprocess.Popen(cmd, shell=True,
287
                                stdout=subprocess.PIPE, stderr=subprocess.PIPE)
288
        (stdout, stderr) = ping.communicate()
289
        ret = ping.wait()
290
        self.assertEquals(ret, 0)
291

    
292
    def _get_hostname_over_ssh(self, hostip, username, password):
293
        ssh = paramiko.SSHClient()
294
        ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
295
        try:
296
            ssh.connect(hostip, username=username, password=password)
297
        except socket.error:
298
            raise AssertionError
299
        stdin, stdout, stderr = ssh.exec_command("hostname")
300
        lines = stdout.readlines()
301
        self.assertEqual(len(lines), 1)
302
        return lines[0]
303

    
304
    def _try_until_timeout_expires(self, warn_timeout, fail_timeout,
305
                                   opmsg, callable, *args, **kwargs):
306
        if warn_timeout == fail_timeout:
307
            warn_timeout = fail_timeout + 1
308
        warn_tmout = time.time() + warn_timeout
309
        fail_tmout = time.time() + fail_timeout
310
        while True:
311
            self.assertLess(time.time(), fail_tmout,
312
                            "operation `%s' timed out" % opmsg)
313
            if time.time() > warn_tmout:
314
                log.warning("Server %d: `%s' operation `%s' not done yet",
315
                            self.serverid, self.servername, opmsg)
316
            try:
317
                log.info("%s... " % opmsg)
318
                return callable(*args, **kwargs)
319
            except AssertionError:
320
                pass
321
            time.sleep(self.query_interval)
322

    
323
    def _insist_on_tcp_connection(self, family, host, port):
324
        familystr = {socket.AF_INET: "IPv4", socket.AF_INET6: "IPv6",
325
                     socket.AF_UNSPEC: "Unspecified-IPv4/6"}
326
        msg = "connect over %s to %s:%s" % \
327
              (familystr.get(family, "Unknown"), host, port)
328
        sock = self._try_until_timeout_expires(
329
                self.action_timeout, self.action_timeout,
330
                msg, self._get_connected_tcp_socket,
331
                family, host, port)
332
        return sock
333

    
334
    def _insist_on_status_transition(self, current_status, new_status,
335
                                    fail_timeout, warn_timeout=None):
336
        msg = "Server %d: `%s', waiting for %s -> %s" % \
337
              (self.serverid, self.servername, current_status, new_status)
338
        if warn_timeout is None:
339
            warn_timeout = fail_timeout
340
        self._try_until_timeout_expires(warn_timeout, fail_timeout,
341
                                        msg, self._verify_server_status,
342
                                        current_status, new_status)
343
        # Ensure the status is actually the expected one
344
        server = self.client.get_server_details(self.serverid)
345
        self.assertEquals(server["status"], new_status)
346

    
347
    def _insist_on_ssh_hostname(self, hostip, username, password):
348
        msg = "SSH to %s, as %s/%s" % (hostip, username, password)
349
        hostname = self._try_until_timeout_expires(
350
                self.action_timeout, self.action_timeout,
351
                msg, self._get_hostname_over_ssh,
352
                hostip, username, password)
353

    
354
        # The hostname must be of the form 'prefix-id'
355
        self.assertTrue(hostname.endswith("-%d\n" % self.serverid))
356

    
357
    def _check_file_through_ssh(self, hostip, username, password, remotepath, content):
358
        msg = "Trying file injection through SSH to %s, as %s/%s" % (hostip, username, password)
359
        log.info(msg)
360
        try:
361
            ssh = paramiko.SSHClient()
362
            ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
363
            ssh.connect(hostip, username=username, password=password)
364
        except socket.error:
365
            raise AssertionError
366
        
367
        transport = paramiko.Transport((hostip,22))
368
        transport.connect(username = username, password = password)
369

    
370
        localpath = '/tmp/'+SNF_TEST_PREFIX+'injection'
371
        sftp = paramiko.SFTPClient.from_transport(transport)
372
        sftp.get(remotepath, localpath)
373
        
374
        sftp.close()
375
        transport.close()
376

    
377
        f = open(localpath)
378
        remote_content = b64encode(f.read())
379

    
380
        # Check if files are the same
381
        return (remote_content == content)
382

    
383
    def _skipIf(self, condition, msg):
384
        if condition:
385
            self.skipTest(msg)
386

    
387
    def test_001_submit_create_server(self):
388
        """Test submit create server request"""
389
        server = self.client.create_server(self.servername, self.flavorid,
390
                                           self.imageid, self.personality)
391

    
392
        self.assertEqual(server["name"], self.servername)
393
        self.assertEqual(server["flavorRef"], self.flavorid)
394
        self.assertEqual(server["imageRef"], self.imageid)
395
        self.assertEqual(server["status"], "BUILD")
396

    
397
        # Update class attributes to reflect data on building server
398
        cls = type(self)
399
        cls.serverid = server["id"]
400
        cls.username = None
401
        cls.passwd = server["adminPass"]
402

    
403
    def test_002a_server_is_building_in_list(self):
404
        """Test server is in BUILD state, in server list"""
405
        servers = self.client.list_servers(detail=True)
406
        servers = filter(lambda x: x["name"] == self.servername, servers)
407
        self.assertEqual(len(servers), 1)
408
        server = servers[0]
409
        self.assertEqual(server["name"], self.servername)
410
        self.assertEqual(server["flavorRef"], self.flavorid)
411
        self.assertEqual(server["imageRef"], self.imageid)
412
        self.assertEqual(server["status"], "BUILD")
413

    
414
    def test_002b_server_is_building_in_details(self):
415
        """Test server is in BUILD state, in details"""
416
        server = self.client.get_server_details(self.serverid)
417
        self.assertEqual(server["name"], self.servername)
418
        self.assertEqual(server["flavorRef"], self.flavorid)
419
        self.assertEqual(server["imageRef"], self.imageid)
420
        self.assertEqual(server["status"], "BUILD")
421

    
422
    def test_002c_set_server_metadata(self):
423
        image = self.client.get_image_details(self.imageid)
424
        os = image["metadata"]["values"]["os"]
425
        loginname = image["metadata"]["values"].get("users", None)
426
        self.client.update_server_metadata(self.serverid, OS=os)
427

    
428
        # Determine the username to use for future connections
429
        # to this host
430
        cls = type(self)
431
        cls.username = loginname
432
        if not cls.username:
433
            cls.username = self._connect_loginname(os)
434
        self.assertIsNotNone(cls.username)
435

    
436
    def test_002d_verify_server_metadata(self):
437
        """Test server metadata keys are set based on image metadata"""
438
        servermeta = self.client.get_server_metadata(self.serverid)
439
        imagemeta = self.client.get_image_metadata(self.imageid)
440
        self.assertEqual(servermeta["OS"], imagemeta["os"])
441

    
442
    def test_003_server_becomes_active(self):
443
        """Test server becomes ACTIVE"""
444
        self._insist_on_status_transition("BUILD", "ACTIVE",
445
                                         self.build_fail, self.build_warning)
446

    
447
    def test_003a_get_server_oob_console(self):
448
        """Test getting OOB server console over VNC
449

450
        Implementation of RFB protocol follows
451
        http://www.realvnc.com/docs/rfbproto.pdf.
452

453
        """
454
        
455
        console = self.cyclades.get_server_console(self.serverid)
456
        self.assertEquals(console['type'], "vnc")
457
        sock = self._insist_on_tcp_connection(socket.AF_UNSPEC,
458
                                        console["host"], console["port"])
459

    
460
        # Step 1. ProtocolVersion message (par. 6.1.1)
461
        version = sock.recv(1024)
462
        self.assertEquals(version, 'RFB 003.008\n')
463
        sock.send(version)
464

    
465
        # Step 2. Security (par 6.1.2): Only VNC Authentication supported
466
        sec = sock.recv(1024)
467
        self.assertEquals(list(sec), ['\x01', '\x02'])
468

    
469
        # Step 3. Request VNC Authentication (par 6.1.2)
470
        sock.send('\x02')
471

    
472
        # Step 4. Receive Challenge (par 6.2.2)
473
        challenge = sock.recv(1024)
474
        self.assertEquals(len(challenge), 16)
475

    
476
        # Step 5. DES-Encrypt challenge, use password as key (par 6.2.2)
477
        response = d3des_generate_response(
478
            (console["password"] + '\0' * 8)[:8], challenge)
479
        sock.send(response)
480

    
481
        # Step 6. SecurityResult (par 6.1.3)
482
        result = sock.recv(4)
483
        self.assertEquals(list(result), ['\x00', '\x00', '\x00', '\x00'])
484
        sock.close()
485
        
486
    def test_004_server_has_ipv4(self):
487
        """Test active server has a valid IPv4 address"""
488
        server = self.client.get_server_details(self.serverid)
489
        ipv4 = self._get_ipv4(server)
490
        self.assertEquals(IP(ipv4).version(), 4)
491

    
492
    def test_005_server_has_ipv6(self):
493
        """Test active server has a valid IPv6 address"""
494
        server = self.client.get_server_details(self.serverid)
495
        ipv6 = self._get_ipv6(server)
496
        self.assertEquals(IP(ipv6).version(), 6)
497

    
498
    def test_006_server_responds_to_ping_IPv4(self):
499
        """Test server responds to ping on IPv4 address"""
500
        server = self.client.get_server_details(self.serverid)
501
        ip = self._get_ipv4(server)
502
        self._try_until_timeout_expires(self.action_timeout,
503
                                        self.action_timeout,
504
                                        "PING IPv4 to %s" % ip,
505
                                        self._ping_once,
506
                                        False, ip)
507

    
508
    def test_007_server_responds_to_ping_IPv6(self):
509
        """Test server responds to ping on IPv6 address"""
510
        server = self.client.get_server_details(self.serverid)
511
        ip = self._get_ipv6(server)
512
        self._try_until_timeout_expires(self.action_timeout,
513
                                        self.action_timeout,
514
                                        "PING IPv6 to %s" % ip,
515
                                        self._ping_once,
516
                                        True, ip)
517

    
518
    def test_008_submit_shutdown_request(self):
519
        """Test submit request to shutdown server"""
520
        self.cyclades.shutdown_server(self.serverid)
521

    
522
    def test_009_server_becomes_stopped(self):
523
        """Test server becomes STOPPED"""
524
        self._insist_on_status_transition("ACTIVE", "STOPPED",
525
                                         self.action_timeout,
526
                                         self.action_timeout)
527

    
528
    def test_010_submit_start_request(self):
529
        """Test submit start server request"""
530
        self.cyclades.start_server(self.serverid)
531

    
532
    def test_011_server_becomes_active(self):
533
        """Test server becomes ACTIVE again"""
534
        self._insist_on_status_transition("STOPPED", "ACTIVE",
535
                                         self.action_timeout,
536
                                         self.action_timeout)
537

    
538
    def test_011a_server_responds_to_ping_IPv4(self):
539
        """Test server OS is actually up and running again"""
540
        self.test_006_server_responds_to_ping_IPv4()
541

    
542
    def test_012_ssh_to_server_IPv4(self):
543
        """Test SSH to server public IPv4 works, verify hostname"""
544
        self._skipIf(self.is_windows, "only valid for Linux servers")
545
        server = self.client.get_server_details(self.serverid)
546
        self._insist_on_ssh_hostname(self._get_ipv4(server),
547
                                     self.username, self.passwd)
548

    
549
    def test_013_ssh_to_server_IPv6(self):
550
        """Test SSH to server public IPv6 works, verify hostname"""
551
        self._skipIf(self.is_windows, "only valid for Linux servers")
552
        server = self.client.get_server_details(self.serverid)
553
        self._insist_on_ssh_hostname(self._get_ipv6(server),
554
                                     self.username, self.passwd)
555

    
556
    def test_014_rdp_to_server_IPv4(self):
557
        "Test RDP connection to server public IPv4 works"""
558
        self._skipIf(not self.is_windows, "only valid for Windows servers")
559
        server = self.client.get_server_details(self.serverid)
560
        ipv4 = self._get_ipv4(server)
561
        sock = _insist_on_tcp_connection(socket.AF_INET, ipv4, 3389)
562

    
563
        # No actual RDP processing done. We assume the RDP server is there
564
        # if the connection to the RDP port is successful.
565
        # FIXME: Use rdesktop, analyze exit code? see manpage [costasd]
566
        sock.close()
567

    
568
    def test_015_rdp_to_server_IPv6(self):
569
        "Test RDP connection to server public IPv6 works"""
570
        self._skipIf(not self.is_windows, "only valid for Windows servers")
571
        server = self.client.get_server_details(self.serverid)
572
        ipv6 = self._get_ipv6(server)
573
        sock = _get_tcp_connection(socket.AF_INET6, ipv6, 3389)
574

    
575
        # No actual RDP processing done. We assume the RDP server is there
576
        # if the connection to the RDP port is successful.
577
        sock.close()
578

    
579
    def test_016_personality_is_enforced(self):
580
        """Test file injection for personality enforcement"""
581
        self._skipIf(self.is_windows, "only implemented for Linux servers")
582
        self._skipIf(self.personality == None, "No personality file selected")
583

    
584
        server = self.client.get_server_details(self.serverid)
585

    
586
        for inj_file in self.personality:
587
            equal_files = self._check_file_through_ssh(self._get_ipv4(server), inj_file['owner'], 
588
                                                       self.passwd, inj_file['path'], inj_file['contents'])
589
            self.assertTrue(equal_files)
590
        
591

    
592
    def test_017_submit_delete_request(self):
593
        """Test submit request to delete server"""
594
        self.client.delete_server(self.serverid)
595

    
596
    def test_018_server_becomes_deleted(self):
597
        """Test server becomes DELETED"""
598
        self._insist_on_status_transition("ACTIVE", "DELETED",
599
                                         self.action_timeout,
600
                                         self.action_timeout)
601

    
602
    def test_019_server_no_longer_in_server_list(self):
603
        """Test server is no longer in server list"""
604
        servers = self.client.list_servers()
605
        self.assertNotIn(self.serverid, [s["id"] for s in servers])
606

    
607

    
608
class NetworkTestCase(unittest.TestCase):
609
    """ Testing networking in cyclades """
610
  
611
    @classmethod
612
    def setUpClass(cls):
613
        "Initialize kamaki, get list of current networks"
614
        conf = Config()
615
        conf.set('compute_token', TOKEN)
616
        cls.client = CycladesClient(conf)
617
        cls.compute = ComputeClient(conf)
618

    
619
        images = cls.compute.list_images(detail = True)
620
        flavors = cls.compute.list_flavors(detail = True)
621
        imageid = choice([im['id'] for im in images])
622
        flavorid = choice([f["id"] for f in flavors if f["disk"] >= 20])
623

    
624
        for image in images:
625
            if image['id'] == imageid:
626
                imagename = image['name']
627

    
628
        servername = "%s%s for %s" % (SNF_TEST_PREFIX, TEST_RUN_ID, imagename)
629
        is_windows = imagename.lower().find("windows") >= 0
630

    
631
        #Run testcases for server spawning in order to ensure it is done right
632
        setupCase =  _spawn_server_test_case(imageid=str(imageid), flavorid=flavorid,
633
                                             imagename=imagename,
634
                                             personality=None,
635
                                             servername=servername,
636
                                             is_windows=is_windows,
637
                                             action_timeout=200,
638
                                             build_warning=1200,
639
                                             build_fail=500,
640
                                             query_interval=3)
641

    
642
        #Using already implemented tests for serverlist population
643
        suite = unittest.TestSuite()
644
        suite.addTest(setupCase('test_001_submit_create_server'))
645
        suite.addTest(setupCase('test_002a_server_is_building_in_list'))
646
        suite.addTest(setupCase('test_002b_server_is_building_in_details'))        
647
        suite.addTest(setupCase('test_003_server_becomes_active'))
648
        unittest.TextTestRunner(verbosity=2).run(suite)
649

    
650
    def test_001_create_network(self):
651
        """Test submit create network request"""
652
        name = SNF_TEST_PREFIX+TEST_RUN_ID
653
        previous_num = len(self.client.list_networks())
654
        network =  self.client.create_network(name)        
655
       
656
        #Test if right name is assigned
657
        self.assertEqual(network['name'], name)
658
        
659
        # Update class attributes
660
        cls = type(self)
661
        cls.networkid = network['id']
662
        networks = self.client.list_networks()
663

    
664
        #Test if new network is created
665
        self.assertTrue(len(networks) > previous_num)
666
        
667
    
668
    def test_002_connect_to_network(self):
669
        """Test connect VM to network"""
670
        servers = self.compute.list_servers()
671

    
672
        #Pick a server created only for the test
673
        server = choice([s for s in servers if s['name'].startswith(SNF_TEST_PREFIX)])
674
        self.client.connect_server(server['id'], self.networkid)
675
        
676
        #Update class attributes
677
        cls = type(self)
678
        cls.serverid = server['id']
679

    
680
        #Insist on connecting until action timeout
681
        connected = (self.client.get_network_details(self.networkid))
682
        fail_tmout = time.time()+self.action_timeout
683

    
684
        while True:
685
            connections = connected['servers']['values']
686
            if (self.serverid in connections):
687
                conn_exists = True
688
            if time.time() > fail_tmout:
689
                self.assertLess(time.time(), fail_tmout)
690
            else:
691
                time.sleep(self.query_interval)
692

    
693
        self.assertTrue(conn_exists)
694
            
695

    
696
    def test_003_disconnect_from_network(self):
697
        prev_state = self.client.get_network_details(self.networkid)
698
        prev_conn = len(prev_state['servers']['values'])
699

    
700
        self.client.disconnect_server(self.serverid, self.networkid)
701
        time.sleep(15)
702

    
703
        #Insist on deleting until action timeout
704
        connected = (self.client.get_network_details(self.networkid))
705
        fail_tmout = time.time()+self.action_timeout
706

    
707
        while True:
708
            connections = connected['servers']['values']
709
            if (self.serverid not in connections):
710
                conn_exists = False
711
            if time.time() > fail_tmout:
712
                self.assertLess(time.time(), fail_tmout)
713
            else:
714
                time.sleep(self.query_interval)
715

    
716
        self.assertFalse(conn_exists)
717

    
718
    def test_004_destroy_network(self):
719
        """Test submit delete network request"""
720
        self.client.delete_network(self.networkid)        
721
        networks = self.client.list_networks()
722

    
723
        curr_net = []
724
        for net in networks:
725
            curr_net.appent(net['id'])
726

    
727
        self.assertTrue(self.networkid not in curr_net)
728
        
729
    def test_005_cleanup_servers(self):
730
        """Cleanup servers created for this test"""
731
        self.compute.delete_server(self.server_id)
732
        fail_tmout = time.time()+self.action_timeout
733

    
734
        #Ensure server gets deleted
735
        while True:
736
            status = self.compute.get_server_details(self.serverid)
737
            if status == 'DELETED':
738
                deleted = True
739
            elif time.time() > fail_tmout: 
740
                self.assertLess(time.time(), fail_tmout)
741
            else:
742
                time.sleep(self.query_interval)
743

    
744
        self.assertTrue(deleted)
745

    
746
class TestRunnerProcess(Process):
747
    """A distinct process used to execute part of the tests in parallel"""
748
    def __init__(self, **kw):
749
        Process.__init__(self, **kw)
750
        kwargs = kw["kwargs"]
751
        self.testq = kwargs["testq"]
752
        self.runner = kwargs["runner"]
753

    
754
    def run(self):
755
        # Make sure this test runner process dies with the parent
756
        # and is not left behind.
757
        #
758
        # WARNING: This uses the prctl(2) call and is
759
        # Linux-specific.
760
        prctl.set_pdeathsig(signal.SIGHUP)
761

    
762
        while True:
763
            log.debug("I am process %d, GETting from queue is %s",
764
                     os.getpid(), self.testq)
765
            msg = self.testq.get()
766
            log.debug("Dequeued msg: %s", msg)
767

    
768
            if msg == "TEST_RUNNER_TERMINATE":
769
                raise SystemExit
770
            elif issubclass(msg, unittest.TestCase):
771
                # Assemble a TestSuite, and run it
772
                suite = unittest.TestLoader().loadTestsFromTestCase(msg)
773
                self.runner.run(suite)
774
            else:
775
                raise Exception("Cannot handle msg: %s" % msg)
776

    
777

    
778

    
779
def _run_cases_in_parallel(cases, fanout=1, runner=None):
780
    """Run instances of TestCase in parallel, in a number of distinct processes
781

782
    The cases iterable specifies the TestCases to be executed in parallel,
783
    by test runners running in distinct processes.
784
    The fanout parameter specifies the number of processes to spawn,
785
    and defaults to 1.
786
    The runner argument specifies the test runner class to use inside each
787
    runner process.
788

789
    """
790
    if runner is None:
791
        runner = unittest.TextTestRunner(verbosity=2, failfast=True)
792

    
793
    # testq: The master process enqueues TestCase objects into this queue,
794
    #        test runner processes pick them up for execution, in parallel.
795
    testq = Queue()
796
    runners = []
797
    for i in xrange(0, fanout):
798
        kwargs = dict(testq=testq, runner=runner)
799
        runners.append(TestRunnerProcess(kwargs=kwargs))
800

    
801
    log.info("Spawning %d test runner processes", len(runners))
802
    for p in runners:
803
        p.start()
804
    log.debug("Spawned %d test runners, PIDs are %s",
805
              len(runners), [p.pid for p in runners])
806

    
807
    # Enqueue test cases
808
    map(testq.put, cases)
809
    map(testq.put, ["TEST_RUNNER_TERMINATE"] * len(runners))
810

    
811
    log.debug("Joining %d processes", len(runners))
812
    for p in runners:
813
        p.join()
814
    log.debug("Done joining %d processes", len(runners))
815

    
816

    
817
def _spawn_server_test_case(**kwargs):
818
    """Construct a new unit test case class from SpawnServerTestCase"""
819

    
820
    name = "SpawnServerTestCase_%s" % kwargs["imageid"]
821
    cls = type(name, (SpawnServerTestCase,), kwargs)
822

    
823
    # Patch extra parameters into test names by manipulating method docstrings
824
    for (mname, m) in \
825
        inspect.getmembers(cls, lambda x: inspect.ismethod(x)):
826
            if hasattr(m, __doc__):
827
                m.__func__.__doc__ = "[%s] %s" % (imagename, m.__doc__)
828

    
829
    # Make sure the class can be pickled, by listing it among
830
    # the attributes of __main__. A PicklingError is raised otherwise.
831
    setattr(__main__, name, cls)
832
    return cls 
833

    
834
def _spawn_network_test_case(**kwargs):
835
    """Construct a new unit test case class from NetworkTestCase"""
836

    
837
    name = "NetworkTestCase"+TEST_RUN_ID
838
    cls = type(name, (NetworkTestCase,), kwargs)
839

    
840
    # Make sure the class can be pickled, by listing it among
841
    # the attributes of __main__. A PicklingError is raised otherwise.
842
    setattr(__main__, name, cls)
843
    return cls 
844

    
845

    
846
def cleanup_servers(delete_stale=False):
847

    
848
    conf = Config()
849
    conf.set('compute_token', TOKEN)
850
    c = ComputeClient(conf)
851

    
852
    servers = c.list_servers()
853
    stale = [s for s in servers if s["name"].startswith(SNF_TEST_PREFIX)]
854

    
855
    if len(stale) == 0:
856
        return
857

    
858
    print >> sys.stderr, "Found these stale servers from previous runs:"
859
    print "    " + \
860
          "\n    ".join(["%d: %s" % (s["id"], s["name"]) for s in stale])
861

    
862
    if delete_stale:
863
        print >> sys.stderr, "Deleting %d stale servers:" % len(stale)
864
        for server in stale:
865
            c.delete_server(server["id"])
866
        print >> sys.stderr, "    ...done"
867
    else:
868
        print >> sys.stderr, "Use --delete-stale to delete them."
869

    
870

    
871
def parse_arguments(args):
872
    from optparse import OptionParser
873

    
874
    kw = {}
875
    kw["usage"] = "%prog [options]"
876
    kw["description"] = \
877
        "%prog runs a number of test scenarios on a " \
878
        "Synnefo deployment."
879

    
880
    parser = OptionParser(**kw)
881
    parser.disable_interspersed_args()
882
    parser.add_option("--api",
883
                      action="store", type="string", dest="api",
884
                      help="The API URI to use to reach the Synnefo API",
885
                      default=DEFAULT_API)
886
    parser.add_option("--token",
887
                      action="store", type="string", dest="token",
888
                      help="The token to use for authentication to the API")
889
    parser.add_option("--nofailfast",
890
                      action="store_true", dest="nofailfast",
891
                      help="Do not fail immediately if one of the tests " \
892
                           "fails (EXPERIMENTAL)",
893
                      default=False)
894
    parser.add_option("--action-timeout",
895
                      action="store", type="int", dest="action_timeout",
896
                      metavar="TIMEOUT",
897
                      help="Wait SECONDS seconds for a server action to " \
898
                           "complete, then the test is considered failed",
899
                      default=100)
900
    parser.add_option("--build-warning",
901
                      action="store", type="int", dest="build_warning",
902
                      metavar="TIMEOUT",
903
                      help="Warn if TIMEOUT seconds have passed and a " \
904
                           "build operation is still pending",
905
                      default=600)
906
    parser.add_option("--build-fail",
907
                      action="store", type="int", dest="build_fail",
908
                      metavar="BUILD_TIMEOUT",
909
                      help="Fail the test if TIMEOUT seconds have passed " \
910
                           "and a build operation is still incomplete",
911
                      default=900)
912
    parser.add_option("--query-interval",
913
                      action="store", type="int", dest="query_interval",
914
                      metavar="INTERVAL",
915
                      help="Query server status when requests are pending " \
916
                           "every INTERVAL seconds",
917
                      default=3)
918
    parser.add_option("--fanout",
919
                      action="store", type="int", dest="fanout",
920
                      metavar="COUNT",
921
                      help="Spawn up to COUNT child processes to execute " \
922
                           "in parallel, essentially have up to COUNT " \
923
                           "server build requests outstanding (EXPERIMENTAL)",
924
                      default=1)
925
    parser.add_option("--force-flavor",
926
                      action="store", type="int", dest="force_flavorid",
927
                      metavar="FLAVOR ID",
928
                      help="Force all server creations to use the specified "\
929
                           "FLAVOR ID instead of a randomly chosen one, " \
930
                           "useful if disk space is scarce",
931
                      default=None)
932
    parser.add_option("--image-id",
933
                      action="store", type="string", dest="force_imageid",
934
                      metavar="IMAGE ID",
935
                      help="Test the specified image id, use 'all' to test " \
936
                           "all available images (mandatory argument)",
937
                      default=None)
938
    parser.add_option("--show-stale",
939
                      action="store_true", dest="show_stale",
940
                      help="Show stale servers from previous runs, whose "\
941
                           "name starts with `%s'" % SNF_TEST_PREFIX,
942
                      default=False)
943
    parser.add_option("--delete-stale",
944
                      action="store_true", dest="delete_stale",
945
                      help="Delete stale servers from previous runs, whose "\
946
                           "name starts with `%s'" % SNF_TEST_PREFIX,
947
                      default=False)
948
    parser.add_option("--force-personality",
949
                      action="store", type="string", dest="personality_path",
950
                      help="Force a personality file injection. File path required. ",
951
                      default=None)
952
    
953

    
954
    # FIXME: Change the default for build-fanout to 10
955
    # FIXME: Allow the user to specify a specific set of Images to test
956

    
957
    (opts, args) = parser.parse_args(args)
958

    
959
    # Verify arguments
960
    if opts.delete_stale:
961
        opts.show_stale = True
962

    
963
    if not opts.show_stale:
964
        if not opts.force_imageid:
965
            print >>sys.stderr, "The --image-id argument is mandatory."
966
            parser.print_help()
967
            sys.exit(1)
968

    
969
        if opts.force_imageid != 'all':
970
            try:
971
                opts.force_imageid = str(opts.force_imageid)
972
            except ValueError:
973
                print >>sys.stderr, "Invalid value specified for --image-id." \
974
                                    "Use a valid id, or `all'."
975
                sys.exit(1)
976

    
977
    return (opts, args)
978

    
979

    
980
def main():
981
    """Assemble test cases into a test suite, and run it
982

983
    IMPORTANT: Tests have dependencies and have to be run in the specified
984
    order inside a single test case. They communicate through attributes of the
985
    corresponding TestCase class (shared fixtures). Distinct subclasses of
986
    TestCase MAY SHARE NO DATA, since they are run in parallel, in distinct
987
    test runner processes.
988

989
    """
990
    (opts, args) = parse_arguments(sys.argv[1:])
991

    
992
    global API, TOKEN
993
    API = opts.api
994
    TOKEN = opts.token
995

    
996
    # Cleanup stale servers from previous runs
997
    if opts.show_stale:
998
        cleanup_servers(delete_stale=opts.delete_stale)
999
        return 0
1000

    
1001
    # Initialize a kamaki instance, get flavors, images
1002

    
1003
    conf = Config()
1004
    conf.set('compute_token', TOKEN)
1005
    c = ComputeClient(conf)
1006

    
1007
    DIMAGES = c.list_images(detail=True)
1008
    DFLAVORS = c.list_flavors(detail=True)
1009

    
1010
    # FIXME: logging, log, LOG PID, TEST_RUN_ID, arguments
1011
    # Run them: FIXME: In parallel, FAILEARLY, catchbreak?
1012
    #unittest.main(verbosity=2, catchbreak=True)
1013

    
1014
    if opts.force_imageid == 'all':
1015
        test_images = DIMAGES
1016
    else:
1017
        test_images = filter(lambda x: x["id"] == opts.force_imageid, DIMAGES)
1018

    
1019
    for image in test_images:
1020
        imageid = str(image["id"])
1021
        flavorid = choice([f["id"] for f in DFLAVORS if f["disk"] >= 20])
1022
        imagename = image["name"]
1023
        
1024
        
1025
        if opts.personality_path != None:
1026
            f = open(opts.personality_path)
1027
            content = b64encode(f.read())
1028
            personality = []
1029
            st = os.stat(opts.personality_path)
1030
            personality.append({
1031
                    'path': '/root/test_inj_file',
1032
                    'owner': 'root',
1033
                    'group': 'root',
1034
                    'mode': 0x7777 & st.st_mode,
1035
                    'contents': content
1036
                    })
1037
        else:
1038
            personality = None
1039

    
1040
        servername = "%s%s for %s" % (SNF_TEST_PREFIX, TEST_RUN_ID, imagename)
1041
        is_windows = imagename.lower().find("windows") >= 0
1042
        
1043
    ServerTestCase = _spawn_server_test_case(imageid=imageid, flavorid=flavorid,
1044
                                             imagename=imagename,
1045
                                             personality=personality,
1046
                                             servername=servername,
1047
                                             is_windows=is_windows,
1048
                                             action_timeout=opts.action_timeout,
1049
                                             build_warning=opts.build_warning,
1050
                                             build_fail=opts.build_fail,
1051
                                             query_interval=opts.query_interval,
1052
                                             )
1053

    
1054

    
1055
    #Running all the testcases sequentially
1056
    #seq_cases = [UnauthorizedTestCase, FlavorsTestCase, ImagesTestCase, ServerTestCase, NetworkTestCase]
1057
    
1058
    newNetworkTestCase = _spawn_network_test_case(action_timeout = opts.action_timeout,
1059
                                                  query_interval = opts.query_interval)
1060
    
1061
    seq_cases = [newNetworkTestCase]
1062
    for case in seq_cases:
1063
        suite = unittest.TestLoader().loadTestsFromTestCase(case)
1064
        unittest.TextTestRunner(verbosity=2).run(suite)
1065
        
1066
    
1067

    
1068
    # # The Following cases run sequentially
1069
    # seq_cases = [UnauthorizedTestCase, FlavorsTestCase, ImagesTestCase]
1070
    # _run_cases_in_parallel(seq_cases, fanout=3, runner=runner)
1071

    
1072
    # # The following cases run in parallel
1073
    # par_cases = []
1074

    
1075
    # if opts.force_imageid == 'all':
1076
    #     test_images = DIMAGES
1077
    # else:
1078
    #     test_images = filter(lambda x: x["id"] == opts.force_imageid, DIMAGES)
1079

    
1080
    # for image in test_images:
1081
    #     imageid = image["id"]
1082
    #     imagename = image["name"]
1083
    #     if opts.force_flavorid:
1084
    #         flavorid = opts.force_flavorid
1085
    #     else:
1086
    #         flavorid = choice([f["id"] for f in DFLAVORS if f["disk"] >= 20])
1087
    #     personality = None   # FIXME
1088
    #     servername = "%s%s for %s" % (SNF_TEST_PREFIX, TEST_RUN_ID, imagename)
1089
    #     is_windows = imagename.lower().find("windows") >= 0
1090
    #     case = _spawn_server_test_case(imageid=str(imageid), flavorid=flavorid,
1091
    #                                    imagename=imagename,
1092
    #                                    personality=personality,
1093
    #                                    servername=servername,
1094
    #                                    is_windows=is_windows,
1095
    #                                    action_timeout=opts.action_timeout,
1096
    #                                    build_warning=opts.build_warning,
1097
    #                                    build_fail=opts.build_fail,
1098
    #                                    query_interval=opts.query_interval)
1099
    #     par_cases.append(case)
1100

    
1101
    # _run_cases_in_parallel(par_cases, fanout=opts.fanout, runner=runner)
1102

    
1103
if __name__ == "__main__":
1104
    sys.exit(main())