Statistics
| Branch: | Tag: | Revision:

root / snf-tools / synnefo_tools / burnin.py @ e8a6a168

History | View | Annotate | Download (55.7 kB)

1
#!/usr/bin/env python
2

    
3
# Copyright 2011 GRNET S.A. All rights reserved.
4
#
5
# Redistribution and use in source and binary forms, with or
6
# without modification, are permitted provided that the following
7
# conditions are met:
8
#
9
#   1. Redistributions of source code must retain the above
10
#      copyright notice, this list of conditions and the following
11
#      disclaimer.
12
#
13
#   2. Redistributions in binary form must reproduce the above
14
#      copyright notice, this list of conditions and the following
15
#      disclaimer in the documentation and/or other materials
16
#      provided with the distribution.
17
#
18
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
19
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
20
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
21
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
22
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
25
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
26
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
28
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29
# POSSIBILITY OF SUCH DAMAGE.
30
#
31
# The views and conclusions contained in the software and
32
# documentation are those of the authors and should not be
33
# interpreted as representing official policies, either expressed
34
# or implied, of GRNET S.A.
35

    
36
"""Perform integration testing on a running Synnefo deployment"""
37

    
38
import __main__
39
import datetime
40
import inspect
41
import logging
42
import os
43
import paramiko
44
import prctl
45
import subprocess
46
import signal
47
import socket
48
import sys
49
import time
50
from base64 import b64encode
51
from IPy import IP
52
from multiprocessing import Process, Queue
53
from random import choice
54

    
55
from kamaki.clients.compute import ComputeClient
56
from kamaki.clients.cyclades import CycladesClient
57
from kamaki.clients import ClientError
58

    
59
from fabric.api import *
60

    
61
from vncauthproxy.d3des import generate_response as d3des_generate_response
62

    
63
# Use backported unittest functionality if Python < 2.7
64
try:
65
    import unittest2 as unittest
66
except ImportError:
67
    if sys.version_info < (2, 7):
68
        raise Exception("The unittest2 package is required for Python < 2.7")
69
    import unittest
70

    
71

    
72
API = None
73
TOKEN = None
74
DEFAULT_API = "https://cyclades.okeanos.grnet.gr/api/v1.1"
75

    
76
# A unique id identifying this test run
77
TEST_RUN_ID = datetime.datetime.strftime(datetime.datetime.now(),
78
                                         "%Y%m%d%H%M%S")
79
SNF_TEST_PREFIX = "snf-test-"
80

    
81
# Setup logging (FIXME - verigak)
82
logging.basicConfig(format="%(message)s")
83
log = logging.getLogger("burnin")
84
log.setLevel(logging.INFO)
85

    
86

    
87
class UnauthorizedTestCase(unittest.TestCase):
88
    def test_unauthorized_access(self):
89
        """Test access without a valid token fails"""
90
        log.info("Authentication test")
91

    
92
        falseToken = '12345'
93
        c = ComputeClient(API, falseToken)
94

    
95
        with self.assertRaises(ClientError) as cm:
96
            c.list_servers()
97
            self.assertEqual(cm.exception.status, 401)
98

    
99

    
100
class ImagesTestCase(unittest.TestCase):
101
    """Test image lists for consistency"""
102
    @classmethod
103
    def setUpClass(cls):
104
        """Initialize kamaki, get (detailed) list of images"""
105
        log.info("Getting simple and detailed list of images")
106

    
107
        cls.client = ComputeClient(API, TOKEN)
108
        cls.images = cls.client.list_images()
109
        cls.dimages = cls.client.list_images(detail=True)
110

    
111
    def test_001_list_images(self):
112
        """Test image list actually returns images"""
113
        self.assertGreater(len(self.images), 0)
114

    
115
    def test_002_list_images_detailed(self):
116
        """Test detailed image list is the same length as list"""
117
        self.assertEqual(len(self.dimages), len(self.images))
118

    
119
    def test_003_same_image_names(self):
120
        """Test detailed and simple image list contain same names"""
121
        names = sorted(map(lambda x: x["name"], self.images))
122
        dnames = sorted(map(lambda x: x["name"], self.dimages))
123
        self.assertEqual(names, dnames)
124

    
125
    def test_004_unique_image_names(self):
126
        """Test images have unique names"""
127
        names = sorted(map(lambda x: x["name"], self.images))
128
        self.assertEqual(sorted(list(set(names))), names)
129

    
130
    def test_005_image_metadata(self):
131
        """Test every image has specific metadata defined"""
132
        keys = frozenset(["os", "description", "size"])
133
        for i in self.dimages:
134
            self.assertTrue(keys.issubset(i["metadata"]["values"].keys()))
135

    
136

    
137
class FlavorsTestCase(unittest.TestCase):
138
    """Test flavor lists for consistency"""
139
    @classmethod
140
    def setUpClass(cls):
141
        """Initialize kamaki, get (detailed) list of flavors"""
142
        log.info("Getting simple and detailed list of flavors")
143

    
144
        cls.client = ComputeClient(API, TOKEN)
145
        cls.flavors = cls.client.list_flavors()
146
        cls.dflavors = cls.client.list_flavors(detail=True)
147

    
148
    def test_001_list_flavors(self):
149
        """Test flavor list actually returns flavors"""
150
        self.assertGreater(len(self.flavors), 0)
151

    
152
    def test_002_list_flavors_detailed(self):
153
        """Test detailed flavor list is the same length as list"""
154
        self.assertEquals(len(self.dflavors), len(self.flavors))
155

    
156
    def test_003_same_flavor_names(self):
157
        """Test detailed and simple flavor list contain same names"""
158
        names = sorted(map(lambda x: x["name"], self.flavors))
159
        dnames = sorted(map(lambda x: x["name"], self.dflavors))
160
        self.assertEqual(names, dnames)
161

    
162
    def test_004_unique_flavor_names(self):
163
        """Test flavors have unique names"""
164
        names = sorted(map(lambda x: x["name"], self.flavors))
165
        self.assertEqual(sorted(list(set(names))), names)
166

    
167
    def test_005_well_formed_flavor_names(self):
168
        """Test flavors have names of the form CxxRyyDzz
169

170
        Where xx is vCPU count, yy is RAM in MiB, zz is Disk in GiB
171

172
        """
173
        for f in self.dflavors:
174
            self.assertEqual("C%dR%dD%d" % (f["cpu"], f["ram"], f["disk"]),
175
                             f["name"],
176
                             "Flavor %s does not match its specs." % f["name"])
177

    
178

    
179
class ServersTestCase(unittest.TestCase):
180
    """Test server lists for consistency"""
181
    @classmethod
182
    def setUpClass(cls):
183
        """Initialize kamaki, get (detailed) list of servers"""
184
        log.info("Getting simple and detailed list of servers")
185

    
186
        cls.client = ComputeClient(API, TOKEN)
187
        cls.servers = cls.client.list_servers()
188
        cls.dservers = cls.client.list_servers(detail=True)
189

    
190
    # def test_001_list_servers(self):
191
    #     """Test server list actually returns servers"""
192
    #     self.assertGreater(len(self.servers), 0)
193

    
194
    def test_002_list_servers_detailed(self):
195
        """Test detailed server list is the same length as list"""
196
        self.assertEqual(len(self.dservers), len(self.servers))
197

    
198
    def test_003_same_server_names(self):
199
        """Test detailed and simple flavor list contain same names"""
200
        names = sorted(map(lambda x: x["name"], self.servers))
201
        dnames = sorted(map(lambda x: x["name"], self.dservers))
202
        self.assertEqual(names, dnames)
203

    
204

    
205
# This class gets replicated into actual TestCases dynamically
206
class SpawnServerTestCase(unittest.TestCase):
207
    """Test scenario for server of the specified image"""
208

    
209
    @classmethod
210
    def setUpClass(cls):
211
        """Initialize a kamaki instance"""
212
        log.info("Spawning server for image `%s'", cls.imagename)
213

    
214
        cls.client = ComputeClient(API, TOKEN)
215
        cls.cyclades = CycladesClient(API, TOKEN)
216

    
217
    def _get_ipv4(self, server):
218
        """Get the public IPv4 of a server from the detailed server info"""
219

    
220
        public_addrs = filter(lambda x: x["id"] == "public",
221
                              server["addresses"]["values"])
222
        self.assertEqual(len(public_addrs), 1)
223
        ipv4_addrs = filter(lambda x: x["version"] == 4,
224
                            public_addrs[0]["values"])
225
        self.assertEqual(len(ipv4_addrs), 1)
226
        return ipv4_addrs[0]["addr"]
227

    
228
    def _get_ipv6(self, server):
229
        """Get the public IPv6 of a server from the detailed server info"""
230
        public_addrs = filter(lambda x: x["id"] == "public",
231
                              server["addresses"]["values"])
232
        self.assertEqual(len(public_addrs), 1)
233
        ipv6_addrs = filter(lambda x: x["version"] == 6,
234
                            public_addrs[0]["values"])
235
        self.assertEqual(len(ipv6_addrs), 1)
236
        return ipv6_addrs[0]["addr"]
237

    
238
    def _connect_loginname(self, os):
239
        """Return the login name for connections based on the server OS"""
240
        if os in ("Ubuntu", "Kubuntu", "Fedora"):
241
            return "user"
242
        elif os in ("windows", "windows_alpha1"):
243
            return "Administrator"
244
        else:
245
            return "root"
246

    
247
    def _verify_server_status(self, current_status, new_status):
248
        """Verify a server has switched to a specified status"""
249
        server = self.client.get_server_details(self.serverid)
250
        if server["status"] not in (current_status, new_status):
251
            return None  # Do not raise exception, return so the test fails
252
        self.assertEquals(server["status"], new_status)
253

    
254
    def _get_connected_tcp_socket(self, family, host, port):
255
        """Get a connected socket from the specified family to host:port"""
256
        sock = None
257
        for res in \
258
            socket.getaddrinfo(host, port, family, socket.SOCK_STREAM, 0,
259
                               socket.AI_PASSIVE):
260
            af, socktype, proto, canonname, sa = res
261
            try:
262
                sock = socket.socket(af, socktype, proto)
263
            except socket.error as msg:
264
                sock = None
265
                continue
266
            try:
267
                sock.connect(sa)
268
            except socket.error as msg:
269
                sock.close()
270
                sock = None
271
                continue
272
        self.assertIsNotNone(sock)
273
        return sock
274

    
275
    def _ping_once(self, ipv6, ip):
276
        """Test server responds to a single IPv4 or IPv6 ping"""
277
        cmd = "ping%s -c 2 -w 3 %s" % ("6" if ipv6 else "", ip)
278
        ping = subprocess.Popen(cmd, shell=True,
279
                                stdout=subprocess.PIPE, stderr=subprocess.PIPE)
280
        (stdout, stderr) = ping.communicate()
281
        ret = ping.wait()
282
        self.assertEquals(ret, 0)
283

    
284
    def _get_hostname_over_ssh(self, hostip, username, password):
285
        ssh = paramiko.SSHClient()
286
        ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
287
        try:
288
            ssh.connect(hostip, username=username, password=password)
289
        except socket.error:
290
            raise AssertionError
291
        stdin, stdout, stderr = ssh.exec_command("hostname")
292
        lines = stdout.readlines()
293
        self.assertEqual(len(lines), 1)
294
        return lines[0]
295

    
296
    def _try_until_timeout_expires(self, warn_timeout, fail_timeout,
297
                                   opmsg, callable, *args, **kwargs):
298
        if warn_timeout == fail_timeout:
299
            warn_timeout = fail_timeout + 1
300
        warn_tmout = time.time() + warn_timeout
301
        fail_tmout = time.time() + fail_timeout
302
        while True:
303
            self.assertLess(time.time(), fail_tmout,
304
                            "operation `%s' timed out" % opmsg)
305
            if time.time() > warn_tmout:
306
                log.warning("Server %d: `%s' operation `%s' not done yet",
307
                            self.serverid, self.servername, opmsg)
308
            try:
309
                log.info("%s... " % opmsg)
310
                return callable(*args, **kwargs)
311
            except AssertionError:
312
                pass
313
            time.sleep(self.query_interval)
314

    
315
    def _insist_on_tcp_connection(self, family, host, port):
316
        familystr = {socket.AF_INET: "IPv4", socket.AF_INET6: "IPv6",
317
                     socket.AF_UNSPEC: "Unspecified-IPv4/6"}
318
        msg = "connect over %s to %s:%s" % \
319
              (familystr.get(family, "Unknown"), host, port)
320
        sock = self._try_until_timeout_expires(
321
                self.action_timeout, self.action_timeout,
322
                msg, self._get_connected_tcp_socket,
323
                family, host, port)
324
        return sock
325

    
326
    def _insist_on_status_transition(self, current_status, new_status,
327
                                    fail_timeout, warn_timeout=None):
328
        msg = "Server %d: `%s', waiting for %s -> %s" % \
329
              (self.serverid, self.servername, current_status, new_status)
330
        if warn_timeout is None:
331
            warn_timeout = fail_timeout
332
        self._try_until_timeout_expires(warn_timeout, fail_timeout,
333
                                        msg, self._verify_server_status,
334
                                        current_status, new_status)
335
        # Ensure the status is actually the expected one
336
        server = self.client.get_server_details(self.serverid)
337
        self.assertEquals(server["status"], new_status)
338

    
339
    def _insist_on_ssh_hostname(self, hostip, username, password):
340
        msg = "SSH to %s, as %s/%s" % (hostip, username, password)
341
        hostname = self._try_until_timeout_expires(
342
                self.action_timeout, self.action_timeout,
343
                msg, self._get_hostname_over_ssh,
344
                hostip, username, password)
345

    
346
        # The hostname must be of the form 'prefix-id'
347
        self.assertTrue(hostname.endswith("-%d\n" % self.serverid))
348

    
349
    def _check_file_through_ssh(self, hostip, username, password,
350
                                remotepath, content):
351
        msg = "Trying file injection through SSH to %s, as %s/%s" % \
352
            (hostip, username, password)
353
        log.info(msg)
354
        try:
355
            ssh = paramiko.SSHClient()
356
            ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
357
            ssh.connect(hostip, username=username, password=password)
358
        except socket.error:
359
            raise AssertionError
360

    
361
        transport = paramiko.Transport((hostip, 22))
362
        transport.connect(username=username, password=password)
363

    
364
        localpath = '/tmp/' + SNF_TEST_PREFIX + 'injection'
365
        sftp = paramiko.SFTPClient.from_transport(transport)
366
        sftp.get(remotepath, localpath)
367
        sftp.close()
368
        transport.close()
369

    
370
        f = open(localpath)
371
        remote_content = b64encode(f.read())
372

    
373
        # Check if files are the same
374
        return (remote_content == content)
375

    
376
    def _skipIf(self, condition, msg):
377
        if condition:
378
            self.skipTest(msg)
379

    
380
    def test_001_submit_create_server(self):
381
        """Test submit create server request"""
382

    
383
        log.info("Submit new server request")
384
        server = self.client.create_server(self.servername, self.flavorid,
385
                                           self.imageid, self.personality)
386

    
387
        log.info("Server id: " + str(server["id"]))
388
        log.info("Server password: " + server["adminPass"])
389
        self.assertEqual(server["name"], self.servername)
390
        self.assertEqual(server["flavorRef"], self.flavorid)
391
        self.assertEqual(server["imageRef"], self.imageid)
392
        self.assertEqual(server["status"], "BUILD")
393

    
394
        # Update class attributes to reflect data on building server
395
        cls = type(self)
396
        cls.serverid = server["id"]
397
        cls.username = None
398
        cls.passwd = server["adminPass"]
399

    
400
    def test_002a_server_is_building_in_list(self):
401
        """Test server is in BUILD state, in server list"""
402
        log.info("Server in BUILD state in server list")
403

    
404
        servers = self.client.list_servers(detail=True)
405
        servers = filter(lambda x: x["name"] == self.servername, servers)
406
        self.assertEqual(len(servers), 1)
407
        server = servers[0]
408
        self.assertEqual(server["name"], self.servername)
409
        self.assertEqual(server["flavorRef"], self.flavorid)
410
        self.assertEqual(server["imageRef"], self.imageid)
411
        self.assertEqual(server["status"], "BUILD")
412

    
413
    def test_002b_server_is_building_in_details(self):
414
        """Test server is in BUILD state, in details"""
415

    
416
        log.info("Server in BUILD state in details")
417

    
418
        server = self.client.get_server_details(self.serverid)
419
        self.assertEqual(server["name"], self.servername)
420
        self.assertEqual(server["flavorRef"], self.flavorid)
421
        self.assertEqual(server["imageRef"], self.imageid)
422
        self.assertEqual(server["status"], "BUILD")
423

    
424
    def test_002c_set_server_metadata(self):
425

    
426
        log.info("Creating server metadata")
427

    
428
        image = self.client.get_image_details(self.imageid)
429
        os = image["metadata"]["values"]["os"]
430
        loginname = image["metadata"]["values"].get("users", None)
431
        self.client.update_server_metadata(self.serverid, OS=os)
432

    
433
        userlist = loginname.split()
434

    
435
        # Determine the username to use for future connections
436
        # to this host
437
        cls = type(self)
438

    
439
        if "root" in userlist:
440
            cls.username = "root"
441
        elif users == None:
442
            cls.username = self._connect_loginname(os)
443
        else:
444
            cls.username = choice(userlist)
445

    
446
        self.assertIsNotNone(cls.username)
447

    
448
    def test_002d_verify_server_metadata(self):
449
        """Test server metadata keys are set based on image metadata"""
450

    
451
        log.info("Verifying image metadata")
452

    
453
        servermeta = self.client.get_server_metadata(self.serverid)
454
        imagemeta = self.client.get_image_metadata(self.imageid)
455

    
456
        self.assertEqual(servermeta["OS"], imagemeta["os"])
457

    
458
    def test_003_server_becomes_active(self):
459
        """Test server becomes ACTIVE"""
460

    
461
        log.info("Waiting for server to become ACTIVE")
462

    
463
        self._insist_on_status_transition("BUILD", "ACTIVE",
464
                                         self.build_fail, self.build_warning)
465

    
466
    def test_003a_get_server_oob_console(self):
467
        """Test getting OOB server console over VNC
468

469
        Implementation of RFB protocol follows
470
        http://www.realvnc.com/docs/rfbproto.pdf.
471

472
        """
473
        console = self.cyclades.get_server_console(self.serverid)
474
        self.assertEquals(console['type'], "vnc")
475
        sock = self._insist_on_tcp_connection(socket.AF_INET,
476
                                        console["host"], console["port"])
477

    
478
        # Step 1. ProtocolVersion message (par. 6.1.1)
479
        version = sock.recv(1024)
480
        self.assertEquals(version, 'RFB 003.008\n')
481
        sock.send(version)
482

    
483
        # Step 2. Security (par 6.1.2): Only VNC Authentication supported
484
        sec = sock.recv(1024)
485
        self.assertEquals(list(sec), ['\x01', '\x02'])
486

    
487
        # Step 3. Request VNC Authentication (par 6.1.2)
488
        sock.send('\x02')
489

    
490
        # Step 4. Receive Challenge (par 6.2.2)
491
        challenge = sock.recv(1024)
492
        self.assertEquals(len(challenge), 16)
493

    
494
        # Step 5. DES-Encrypt challenge, use password as key (par 6.2.2)
495
        response = d3des_generate_response(
496
            (console["password"] + '\0' * 8)[:8], challenge)
497
        sock.send(response)
498

    
499
        # Step 6. SecurityResult (par 6.1.3)
500
        result = sock.recv(4)
501
        self.assertEquals(list(result), ['\x00', '\x00', '\x00', '\x00'])
502
        sock.close()
503

    
504
    def test_004_server_has_ipv4(self):
505
        """Test active server has a valid IPv4 address"""
506

    
507
        log.info("Validate server's IPv4")
508

    
509
        server = self.client.get_server_details(self.serverid)
510
        ipv4 = self._get_ipv4(server)
511
        self.assertEquals(IP(ipv4).version(), 4)
512

    
513
    def test_005_server_has_ipv6(self):
514
        """Test active server has a valid IPv6 address"""
515

    
516
        log.info("Validate server's IPv6")
517

    
518
        server = self.client.get_server_details(self.serverid)
519
        ipv6 = self._get_ipv6(server)
520
        self.assertEquals(IP(ipv6).version(), 6)
521

    
522
    def test_006_server_responds_to_ping_IPv4(self):
523
        """Test server responds to ping on IPv4 address"""
524

    
525
        log.info("Testing if server responds to pings in IPv4")
526

    
527
        server = self.client.get_server_details(self.serverid)
528
        ip = self._get_ipv4(server)
529
        self._try_until_timeout_expires(self.action_timeout,
530
                                        self.action_timeout,
531
                                        "PING IPv4 to %s" % ip,
532
                                        self._ping_once,
533
                                        False, ip)
534

    
535
    def test_007_server_responds_to_ping_IPv6(self):
536
        """Test server responds to ping on IPv6 address"""
537

    
538
        log.info("Testing if server responds to pings in IPv6")
539

    
540
        server = self.client.get_server_details(self.serverid)
541
        ip = self._get_ipv6(server)
542
        self._try_until_timeout_expires(self.action_timeout,
543
                                        self.action_timeout,
544
                                        "PING IPv6 to %s" % ip,
545
                                        self._ping_once,
546
                                        True, ip)
547

    
548
    def test_008_submit_shutdown_request(self):
549
        """Test submit request to shutdown server"""
550

    
551
        log.info("Shutting down server")
552

    
553
        self.cyclades.shutdown_server(self.serverid)
554

    
555
    def test_009_server_becomes_stopped(self):
556
        """Test server becomes STOPPED"""
557

    
558
        log.info("Waiting until server becomes STOPPED")
559
        self._insist_on_status_transition("ACTIVE", "STOPPED",
560
                                         self.action_timeout,
561
                                         self.action_timeout)
562

    
563
    def test_010_submit_start_request(self):
564
        """Test submit start server request"""
565

    
566
        log.info("Starting server")
567

    
568
        self.cyclades.start_server(self.serverid)
569

    
570
    def test_011_server_becomes_active(self):
571
        """Test server becomes ACTIVE again"""
572

    
573
        log.info("Waiting until server becomes ACTIVE")
574
        self._insist_on_status_transition("STOPPED", "ACTIVE",
575
                                         self.action_timeout,
576
                                         self.action_timeout)
577

    
578
    def test_011a_server_responds_to_ping_IPv4(self):
579
        """Test server OS is actually up and running again"""
580

    
581
        log.info("Testing if server is actually up and running")
582

    
583
        self.test_006_server_responds_to_ping_IPv4()
584

    
585
    def test_012_ssh_to_server_IPv4(self):
586
        """Test SSH to server public IPv4 works, verify hostname"""
587

    
588
        self._skipIf(self.is_windows, "only valid for Linux servers")
589
        server = self.client.get_server_details(self.serverid)
590
        self._insist_on_ssh_hostname(self._get_ipv4(server),
591
                                     self.username, self.passwd)
592

    
593
    def test_013_ssh_to_server_IPv6(self):
594
        """Test SSH to server public IPv6 works, verify hostname"""
595
        self._skipIf(self.is_windows, "only valid for Linux servers")
596
        server = self.client.get_server_details(self.serverid)
597
        self._insist_on_ssh_hostname(self._get_ipv6(server),
598
                                     self.username, self.passwd)
599

    
600
    def test_014_rdp_to_server_IPv4(self):
601
        "Test RDP connection to server public IPv4 works"""
602
        self._skipIf(not self.is_windows, "only valid for Windows servers")
603
        server = self.client.get_server_details(self.serverid)
604
        ipv4 = self._get_ipv4(server)
605
        sock = _insist_on_tcp_connection(socket.AF_INET, ipv4, 3389)
606

    
607
        # No actual RDP processing done. We assume the RDP server is there
608
        # if the connection to the RDP port is successful.
609
        # FIXME: Use rdesktop, analyze exit code? see manpage [costasd]
610
        sock.close()
611

    
612
    def test_015_rdp_to_server_IPv6(self):
613
        "Test RDP connection to server public IPv6 works"""
614
        self._skipIf(not self.is_windows, "only valid for Windows servers")
615
        server = self.client.get_server_details(self.serverid)
616
        ipv6 = self._get_ipv6(server)
617
        sock = _get_tcp_connection(socket.AF_INET6, ipv6, 3389)
618

    
619
        # No actual RDP processing done. We assume the RDP server is there
620
        # if the connection to the RDP port is successful.
621
        sock.close()
622

    
623
    def test_016_personality_is_enforced(self):
624
        """Test file injection for personality enforcement"""
625
        self._skipIf(self.is_windows, "only implemented for Linux servers")
626
        self._skipIf(self.personality == None, "No personality file selected")
627

    
628
        log.info("Trying to inject file for personality enforcement")
629

    
630
        server = self.client.get_server_details(self.serverid)
631

    
632
        for inj_file in self.personality:
633
            equal_files = self._check_file_through_ssh(self._get_ipv4(server),
634
                                                       inj_file['owner'],
635
                                                       self.passwd,
636
                                                       inj_file['path'],
637
                                                       inj_file['contents'])
638
            self.assertTrue(equal_files)
639

    
640
    def test_017_submit_delete_request(self):
641
        """Test submit request to delete server"""
642

    
643
        log.info("Deleting server")
644

    
645
        self.client.delete_server(self.serverid)
646

    
647
    def test_018_server_becomes_deleted(self):
648
        """Test server becomes DELETED"""
649

    
650
        log.info("Testing if server becomes DELETED")
651

    
652
        self._insist_on_status_transition("ACTIVE", "DELETED",
653
                                         self.action_timeout,
654
                                         self.action_timeout)
655

    
656
    def test_019_server_no_longer_in_server_list(self):
657
        """Test server is no longer in server list"""
658

    
659
        log.info("Test if server is no longer listed")
660

    
661
        servers = self.client.list_servers()
662
        self.assertNotIn(self.serverid, [s["id"] for s in servers])
663

    
664

    
665
class NetworkTestCase(unittest.TestCase):
666
    """ Testing networking in cyclades """
667

    
668
    @classmethod
669
    def setUpClass(cls):
670
        "Initialize kamaki, get list of current networks"
671

    
672
        cls.client = CycladesClient(API, TOKEN)
673
        cls.compute = ComputeClient(API, TOKEN)
674

    
675
        cls.servername = "%s%s for %s" % (SNF_TEST_PREFIX,
676
                                          TEST_RUN_ID,
677
                                          cls.imagename)
678

    
679
        #Dictionary initialization for the vms credentials
680
        cls.serverid = dict()
681
        cls.username = dict()
682
        cls.password = dict()
683

    
684
    def _get_ipv4(self, server):
685
        """Get the public IPv4 of a server from the detailed server info"""
686

    
687
        public_addrs = filter(lambda x: x["id"] == "public",
688
                              server["addresses"]["values"])
689
        self.assertEqual(len(public_addrs), 1)
690
        ipv4_addrs = filter(lambda x: x["version"] == 4,
691
                            public_addrs[0]["values"])
692
        self.assertEqual(len(ipv4_addrs), 1)
693
        return ipv4_addrs[0]["addr"]
694

    
695
    def _connect_loginname(self, os):
696
        """Return the login name for connections based on the server OS"""
697
        if os in ("Ubuntu", "Kubuntu", "Fedora"):
698
            return "user"
699
        elif os in ("windows", "windows_alpha1"):
700
            return "Administrator"
701
        else:
702
            return "root"
703

    
704
    def _ping_once(self, ip):
705

    
706
        """Test server responds to a single IPv4 or IPv6 ping"""
707
        cmd = "ping -c 2 -w 3 %s" % (ip)
708
        ping = subprocess.Popen(cmd, shell=True,
709
                                stdout=subprocess.PIPE, stderr=subprocess.PIPE)
710
        (stdout, stderr) = ping.communicate()
711
        ret = ping.wait()
712

    
713
        return (ret == 0)
714

    
715
    def test_00001a_submit_create_server_A(self):
716
        """Test submit create server request"""
717

    
718
        log.info("Creating test server A")
719

    
720
        serverA = self.client.create_server(self.servername, self.flavorid,
721
                                            self.imageid, personality=None)
722

    
723
        self.assertEqual(serverA["name"], self.servername)
724
        self.assertEqual(serverA["flavorRef"], self.flavorid)
725
        self.assertEqual(serverA["imageRef"], self.imageid)
726
        self.assertEqual(serverA["status"], "BUILD")
727

    
728
        # Update class attributes to reflect data on building server
729
        self.serverid['A'] = serverA["id"]
730
        self.username['A'] = None
731
        self.password['A'] = serverA["adminPass"]
732

    
733
        log.info("Server A id:" + str(serverA["id"]))
734
        log.info("Server password " + (self.password['A']))
735

    
736
    def test_00001b_serverA_becomes_active(self):
737
        """Test server becomes ACTIVE"""
738

    
739
        log.info("Waiting until test server A becomes ACTIVE")
740

    
741
        fail_tmout = time.time() + self.action_timeout
742
        while True:
743
            d = self.client.get_server_details(self.serverid['A'])
744
            status = d['status']
745
            if status == 'ACTIVE':
746
                active = True
747
                break
748
            elif time.time() > fail_tmout:
749
                self.assertLess(time.time(), fail_tmout)
750
            else:
751
                time.sleep(self.query_interval)
752

    
753
        self.assertTrue(active)
754

    
755
    def test_00002a_submit_create_server_B(self):
756
        """Test submit create server request"""
757

    
758
        log.info("Creating test server B")
759

    
760
        serverB = self.client.create_server(self.servername, self.flavorid,
761
                                            self.imageid, personality=None)
762

    
763
        self.assertEqual(serverB["name"], self.servername)
764
        self.assertEqual(serverB["flavorRef"], self.flavorid)
765
        self.assertEqual(serverB["imageRef"], self.imageid)
766
        self.assertEqual(serverB["status"], "BUILD")
767

    
768
        # Update class attributes to reflect data on building server
769
        self.serverid['B'] = serverB["id"]
770
        self.username['B'] = None
771
        self.password['B'] = serverB["adminPass"]
772

    
773
        log.info("Server B id: " + str(serverB["id"]))
774
        log.info("Password " + (self.password['B']))
775

    
776
    def test_00002b_serverB_becomes_active(self):
777
        """Test server becomes ACTIVE"""
778

    
779
        log.info("Waiting until test server B becomes ACTIVE")
780

    
781
        fail_tmout = time.time() + self.action_timeout
782
        while True:
783
            d = self.client.get_server_details(self.serverid['B'])
784
            status = d['status']
785
            if status == 'ACTIVE':
786
                active = True
787
                break
788
            elif time.time() > fail_tmout:
789
                self.assertLess(time.time(), fail_tmout)
790
            else:
791
                time.sleep(self.query_interval)
792

    
793
        self.assertTrue(active)
794

    
795
    def test_001_create_network(self):
796
        """Test submit create network request"""
797

    
798
        log.info("Submit new network request")
799

    
800
        name = SNF_TEST_PREFIX + TEST_RUN_ID
801
        previous_num = len(self.client.list_networks())
802
        network = self.client.create_network(name)
803

    
804
        #Test if right name is assigned
805
        self.assertEqual(network['name'], name)
806

    
807
        # Update class attributes
808
        cls = type(self)
809
        cls.networkid = network['id']
810
        networks = self.client.list_networks()
811

    
812
        #Test if new network is created
813
        self.assertTrue(len(networks) > previous_num)
814

    
815
    def test_002_connect_to_network(self):
816
        """Test connect VMs to network"""
817

    
818
        log.info("Connect VMs to private network")
819

    
820
        self.client.connect_server(self.serverid['A'], self.networkid)
821
        self.client.connect_server(self.serverid['B'], self.networkid)
822

    
823
        #Insist on connecting until action timeout
824
        fail_tmout = time.time() + self.action_timeout
825

    
826
        while True:
827
            connected = (self.client.get_network_details(self.networkid))
828
            connections = connected['servers']['values']
829
            if (self.serverid['A'] in connections) \
830
                    and (self.serverid['B'] in connections):
831
                conn_exists = True
832
                break
833
            elif time.time() > fail_tmout:
834
                self.assertLess(time.time(), fail_tmout)
835
            else:
836
                time.sleep(self.query_interval)
837

    
838
        self.assertTrue(conn_exists)
839

    
840
    def test_002a_reboot(self):
841
        """Rebooting server A"""
842

    
843
        log.info("Rebooting server A")
844

    
845
        self.client.shutdown_server(self.serverid['A'])
846

    
847
        fail_tmout = time.time() + self.action_timeout
848
        while True:
849
            d = self.client.get_server_details(self.serverid['A'])
850
            status = d['status']
851
            if status == 'STOPPED':
852
                break
853
            elif time.time() > fail_tmout:
854
                self.assertLess(time.time(), fail_tmout)
855
            else:
856
                time.sleep(self.query_interval)
857

    
858
        self.client.start_server(self.serverid['A'])
859

    
860
        while True:
861
            d = self.client.get_server_details(self.serverid['A'])
862
            status = d['status']
863
            if status == 'ACTIVE':
864
                active = True
865
                break
866
            elif time.time() > fail_tmout:
867
                self.assertLess(time.time(), fail_tmout)
868
            else:
869
                time.sleep(self.query_interval)
870

    
871
        self.assertTrue(active)
872

    
873
    def test_002b_ping_server_A(self):
874
        "Test if server A is pingable"
875

    
876
        log.info("Testing if server A is pingable")
877

    
878
        server = self.client.get_server_details(self.serverid['A'])
879
        ip = self._get_ipv4(server)
880

    
881
        fail_tmout = time.time() + self.action_timeout
882

    
883
        s = False
884

    
885
        while True:
886

    
887
            if self._ping_once(ip):
888
                s = True
889
                break
890

    
891
            elif time.time() > fail_tmout:
892
                self.assertLess(time.time(), fail_tmout)
893

    
894
            else:
895
                time.sleep(self.query_interval)
896

    
897
        self.assertTrue(s)
898

    
899
    def test_002c_reboot(self):
900
        """Reboot server B"""
901

    
902
        log.info("Rebooting server B")
903

    
904
        self.client.shutdown_server(self.serverid['B'])
905

    
906
        fail_tmout = time.time() + self.action_timeout
907
        while True:
908
            d = self.client.get_server_details(self.serverid['B'])
909
            status = d['status']
910
            if status == 'STOPPED':
911
                break
912
            elif time.time() > fail_tmout:
913
                self.assertLess(time.time(), fail_tmout)
914
            else:
915
                time.sleep(self.query_interval)
916

    
917
        self.client.start_server(self.serverid['B'])
918

    
919
        while True:
920
            d = self.client.get_server_details(self.serverid['B'])
921
            status = d['status']
922
            if status == 'ACTIVE':
923
                active = True
924
                break
925
            elif time.time() > fail_tmout:
926
                self.assertLess(time.time(), fail_tmout)
927
            else:
928
                time.sleep(self.query_interval)
929

    
930
        self.assertTrue(active)
931

    
932
    def test_002d_ping_server_B(self):
933
        """Test if server B is pingable"""
934

    
935
        log.info("Testing if server B is pingable")
936
        server = self.client.get_server_details(self.serverid['B'])
937
        ip = self._get_ipv4(server)
938

    
939
        fail_tmout = time.time() + self.action_timeout
940

    
941
        s = False
942

    
943
        while True:
944
            if self._ping_once(ip):
945
                s = True
946
                break
947

    
948
            elif time.time() > fail_tmout:
949
                self.assertLess(time.time(), fail_tmout)
950

    
951
            else:
952
                time.sleep(self.query_interval)
953

    
954
        self.assertTrue(s)
955

    
956
    def test_003a_setup_interface_A(self):
957
        """Set up eth1 for server A"""
958

    
959
        log.info("Setting up interface eth1 for server A")
960

    
961
        server = self.client.get_server_details(self.serverid['A'])
962
        image = self.client.get_image_details(self.imageid)
963
        os = image['metadata']['values']['os']
964

    
965
        users = image["metadata"]["values"].get("users", None)
966
        userlist = users.split()
967

    
968
        if "root" in userlist:
969
            loginname = "root"
970
        elif users == None:
971
            loginname = self._connect_loginname(os)
972
        else:
973
            loginname = choice(userlist)
974

    
975
        hostip = self._get_ipv4(server)
976
        myPass = self.password['A']
977

    
978
        log.info("SSH in server A as %s/%s" % (loginname, myPass))
979

    
980
        res = False
981

    
982
        if loginname != "root":
983
            with settings(
984
                hide('warnings', 'running'),
985
                warn_only=True,
986
                host_string=hostip,
987
                user=loginname, password=myPass
988
                ):
989

    
990
                if len(sudo('ifconfig eth1 192.168.0.12')) == 0:
991
                    res = True
992

    
993
        else:
994
            with settings(
995
                hide('warnings', 'running'),
996
                warn_only=True,
997
                host_string=hostip,
998
                user=loginname, password=myPass
999
                ):
1000

    
1001
                if len(run('ifconfig eth1 192.168.0.12')) == 0:
1002
                    res = True
1003

    
1004
        self.assertTrue(res)
1005

    
1006
    def test_003b_setup_interface_B(self):
1007
        """Setup eth1 for server B"""
1008

    
1009
        log.info("Setting up interface eth1 for server B")
1010

    
1011
        server = self.client.get_server_details(self.serverid['B'])
1012
        image = self.client.get_image_details(self.imageid)
1013
        os = image['metadata']['values']['os']
1014

    
1015
        users = image["metadata"]["values"].get("users", None)
1016
        userlist = users.split()
1017

    
1018
        if "root" in userlist:
1019
            loginname = "root"
1020
        elif users == None:
1021
            loginname = self._connect_loginname(os)
1022
        else:
1023
            loginname = choice(userlist)
1024

    
1025
        hostip = self._get_ipv4(server)
1026
        myPass = self.password['B']
1027

    
1028
        log.info("SSH in server B as %s/%s" % (loginname, myPass))
1029

    
1030
        res = False
1031

    
1032
        if loginname != "root":
1033
            with settings(
1034
                hide('warnings', 'running'),
1035
                warn_only=True,
1036
                host_string=hostip,
1037
                user=loginname, password=myPass
1038
                ):
1039

    
1040
                if len(sudo('ifconfig eth1 192.168.0.13')) == 0:
1041
                    res = True
1042

    
1043
        else:
1044
            with settings(
1045
                hide('warnings', 'running'),
1046
                warn_only=True,
1047
                host_string=hostip,
1048
                user=loginname, password=myPass
1049
                ):
1050

    
1051
                if len(run('ifconfig eth1 192.168.0.13')) == 0:
1052
                    res = True
1053

    
1054
        self.assertTrue(res)
1055

    
1056
    def test_003c_test_connection_exists(self):
1057
        """Ping server B from server A to test if connection exists"""
1058

    
1059
        log.info("Testing if server A is actually connected to server B")
1060

    
1061
        server = self.client.get_server_details(self.serverid['A'])
1062
        image = self.client.get_image_details(self.imageid)
1063
        os = image['metadata']['values']['os']
1064
        hostip = self._get_ipv4(server)
1065

    
1066
        users = image["metadata"]["values"].get("users", None)
1067
        userlist = users.split()
1068

    
1069
        if "root" in userlist:
1070
            loginname = "root"
1071
        elif users == None:
1072
            loginname = self._connect_loginname(os)
1073
        else:
1074
            loginname = choice(userlist)
1075

    
1076
        myPass = self.password['A']
1077

    
1078
        try:
1079
            ssh = paramiko.SSHClient()
1080
            ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
1081
            ssh.connect(hostip, username=loginname, password=myPass)
1082
        except socket.error:
1083
            raise AssertionError
1084

    
1085
        cmd = "if ping -c 2 -w 3 192.168.0.13 >/dev/null; \
1086
               then echo \'True\'; fi;"
1087
        stdin, stdout, stderr = ssh.exec_command(cmd)
1088
        lines = stdout.readlines()
1089

    
1090
        exists = False
1091

    
1092
        if 'True\n' in lines:
1093
            exists = True
1094

    
1095
        self.assertTrue(exists)
1096

    
1097
#TODO: Test IPv6 private connectity
1098

    
1099
    def test_004_disconnect_from_network(self):
1100
        "Disconnecting server A and B from network"
1101

    
1102
        log.info("Disconnecting servers from private network")
1103

    
1104
        prev_state = self.client.get_network_details(self.networkid)
1105
        prev_conn = len(prev_state['servers']['values'])
1106

    
1107
        self.client.disconnect_server(self.serverid['A'], self.networkid)
1108
        self.client.disconnect_server(self.serverid['B'], self.networkid)
1109

    
1110
        #Insist on deleting until action timeout
1111
        fail_tmout = time.time() + self.action_timeout
1112

    
1113
        while True:
1114
            connected = (self.client.get_network_details(self.networkid))
1115
            connections = connected['servers']['values']
1116
            if ((self.serverid['A'] not in connections) and
1117
                (self.serverid['B'] not in connections)):
1118
                conn_exists = False
1119
                break
1120
            elif time.time() > fail_tmout:
1121
                self.assertLess(time.time(), fail_tmout)
1122
            else:
1123
                time.sleep(self.query_interval)
1124

    
1125
        self.assertFalse(conn_exists)
1126

    
1127
    def test_005_destroy_network(self):
1128
        """Test submit delete network request"""
1129

    
1130
        log.info("Submitting delete network request")
1131

    
1132
        self.client.delete_network(self.networkid)
1133
        networks = self.client.list_networks()
1134

    
1135
        curr_net = []
1136
        for net in networks:
1137
            curr_net.append(net['id'])
1138

    
1139
        self.assertTrue(self.networkid not in curr_net)
1140

    
1141
    def test_006_cleanup_servers(self):
1142
        """Cleanup servers created for this test"""
1143

    
1144
        log.info("Delete servers created for this test")
1145

    
1146
        self.compute.delete_server(self.serverid['A'])
1147
        self.compute.delete_server(self.serverid['B'])
1148

    
1149
        fail_tmout = time.time() + self.action_timeout
1150

    
1151
        #Ensure server gets deleted
1152
        status = dict()
1153

    
1154
        while True:
1155
            details = self.compute.get_server_details(self.serverid['A'])
1156
            status['A'] = details['status']
1157
            details = self.compute.get_server_details(self.serverid['B'])
1158
            status['B'] = details['status']
1159
            if (status['A'] == 'DELETED') and (status['B'] == 'DELETED'):
1160
                deleted = True
1161
                break
1162
            elif time.time() > fail_tmout:
1163
                self.assertLess(time.time(), fail_tmout)
1164
            else:
1165
                time.sleep(self.query_interval)
1166

    
1167
        self.assertTrue(deleted)
1168

    
1169

    
1170
class TestRunnerProcess(Process):
1171
    """A distinct process used to execute part of the tests in parallel"""
1172
    def __init__(self, **kw):
1173
        Process.__init__(self, **kw)
1174
        kwargs = kw["kwargs"]
1175
        self.testq = kwargs["testq"]
1176
        self.runner = kwargs["runner"]
1177

    
1178
    def run(self):
1179
        # Make sure this test runner process dies with the parent
1180
        # and is not left behind.
1181
        #
1182
        # WARNING: This uses the prctl(2) call and is
1183
        # Linux-specific.
1184
        prctl.set_pdeathsig(signal.SIGHUP)
1185

    
1186
        while True:
1187
            log.debug("I am process %d, GETting from queue is %s",
1188
                     os.getpid(), self.testq)
1189
            msg = self.testq.get()
1190
            log.debug("Dequeued msg: %s", msg)
1191

    
1192
            if msg == "TEST_RUNNER_TERMINATE":
1193
                raise SystemExit
1194
            elif issubclass(msg, unittest.TestCase):
1195
                # Assemble a TestSuite, and run it
1196
                suite = unittest.TestLoader().loadTestsFromTestCase(msg)
1197
                self.runner.run(suite)
1198
            else:
1199
                raise Exception("Cannot handle msg: %s" % msg)
1200

    
1201

    
1202
def _run_cases_in_parallel(cases, fanout=1, runner=None):
1203
    """Run instances of TestCase in parallel, in a number of distinct processes
1204

1205
    The cases iterable specifies the TestCases to be executed in parallel,
1206
    by test runners running in distinct processes.
1207
    The fanout parameter specifies the number of processes to spawn,
1208
    and defaults to 1.
1209
    The runner argument specifies the test runner class to use inside each
1210
    runner process.
1211

1212
    """
1213
    if runner is None:
1214
        runner = unittest.TextTestRunner(verbosity=2, failfast=True)
1215

    
1216
    # testq: The master process enqueues TestCase objects into this queue,
1217
    #        test runner processes pick them up for execution, in parallel.
1218
    testq = Queue()
1219
    runners = []
1220
    for i in xrange(0, fanout):
1221
        kwargs = dict(testq=testq, runner=runner)
1222
        runners.append(TestRunnerProcess(kwargs=kwargs))
1223

    
1224
    log.info("Spawning %d test runner processes", len(runners))
1225
    for p in runners:
1226
        p.start()
1227
    log.debug("Spawned %d test runners, PIDs are %s",
1228
              len(runners), [p.pid for p in runners])
1229

    
1230
    # Enqueue test cases
1231
    map(testq.put, cases)
1232
    map(testq.put, ["TEST_RUNNER_TERMINATE"] * len(runners))
1233

    
1234
    log.debug("Joining %d processes", len(runners))
1235
    for p in runners:
1236
        p.join()
1237
    log.debug("Done joining %d processes", len(runners))
1238

    
1239

    
1240
def _spawn_server_test_case(**kwargs):
1241
    """Construct a new unit test case class from SpawnServerTestCase"""
1242

    
1243
    name = "SpawnServerTestCase_%s" % kwargs["imageid"]
1244
    cls = type(name, (SpawnServerTestCase,), kwargs)
1245

    
1246
    # Patch extra parameters into test names by manipulating method docstrings
1247
    for (mname, m) in \
1248
        inspect.getmembers(cls, lambda x: inspect.ismethod(x)):
1249
        if hasattr(m, __doc__):
1250
            m.__func__.__doc__ = "[%s] %s" % (imagename, m.__doc__)
1251

    
1252
    # Make sure the class can be pickled, by listing it among
1253
    # the attributes of __main__. A PicklingError is raised otherwise.
1254
    setattr(__main__, name, cls)
1255
    return cls
1256

    
1257

    
1258
def _spawn_network_test_case(**kwargs):
1259
    """Construct a new unit test case class from NetworkTestCase"""
1260

    
1261
    name = "NetworkTestCase" + TEST_RUN_ID
1262
    cls = type(name, (NetworkTestCase,), kwargs)
1263

    
1264
    # Make sure the class can be pickled, by listing it among
1265
    # the attributes of __main__. A PicklingError is raised otherwise.
1266
    setattr(__main__, name, cls)
1267
    return cls
1268

    
1269

    
1270
def cleanup_servers(delete_stale=False):
1271

    
1272
    c = ComputeClient(API, TOKEN)
1273

    
1274
    servers = c.list_servers()
1275
    stale = [s for s in servers if s["name"].startswith(SNF_TEST_PREFIX)]
1276

    
1277
    if len(stale) == 0:
1278
        return
1279

    
1280
    print >> sys.stderr, "Found these stale servers from previous runs:"
1281
    print "    " + \
1282
          "\n    ".join(["%d: %s" % (s["id"], s["name"]) for s in stale])
1283

    
1284
    if delete_stale:
1285
        print >> sys.stderr, "Deleting %d stale servers:" % len(stale)
1286
        for server in stale:
1287
            c.delete_server(server["id"])
1288
        print >> sys.stderr, "    ...done"
1289
    else:
1290
        print >> sys.stderr, "Use --delete-stale to delete them."
1291

    
1292

    
1293
def cleanup_networks(delete_stale=False):
1294

    
1295
    c = CycladesClient(API, TOKEN)
1296

    
1297
    networks = c.list_networks()
1298
    stale = [n for n in networks if n["name"].startswith(SNF_TEST_PREFIX)]
1299

    
1300
    if len(stale) == 0:
1301
        return
1302

    
1303
    print >> sys.stderr, "Found these stale networks from previous runs:"
1304
    print "    " + \
1305
          "\n    ".join(["%s: %s" % (str(n["id"]), n["name"]) for n in stale])
1306

    
1307
    if delete_stale:
1308
        print >> sys.stderr, "Deleting %d stale networks:" % len(stale)
1309
        for network in stale:
1310
            c.delete_network(network["id"])
1311
        print >> sys.stderr, "    ...done"
1312
    else:
1313
        print >> sys.stderr, "Use --delete-stale to delete them."
1314

    
1315

    
1316
def parse_arguments(args):
1317
    from optparse import OptionParser
1318

    
1319
    kw = {}
1320
    kw["usage"] = "%prog [options]"
1321
    kw["description"] = \
1322
        "%prog runs a number of test scenarios on a " \
1323
        "Synnefo deployment."
1324

    
1325
    parser = OptionParser(**kw)
1326
    parser.disable_interspersed_args()
1327
    parser.add_option("--api",
1328
                      action="store", type="string", dest="api",
1329
                      help="The API URI to use to reach the Synnefo API",
1330
                      default=DEFAULT_API)
1331
    parser.add_option("--token",
1332
                      action="store", type="string", dest="token",
1333
                      help="The token to use for authentication to the API")
1334
    parser.add_option("--nofailfast",
1335
                      action="store_true", dest="nofailfast",
1336
                      help="Do not fail immediately if one of the tests " \
1337
                           "fails (EXPERIMENTAL)",
1338
                      default=False)
1339
    parser.add_option("--action-timeout",
1340
                      action="store", type="int", dest="action_timeout",
1341
                      metavar="TIMEOUT",
1342
                      help="Wait SECONDS seconds for a server action to " \
1343
                           "complete, then the test is considered failed",
1344
                      default=100)
1345
    parser.add_option("--build-warning",
1346
                      action="store", type="int", dest="build_warning",
1347
                      metavar="TIMEOUT",
1348
                      help="Warn if TIMEOUT seconds have passed and a " \
1349
                           "build operation is still pending",
1350
                      default=600)
1351
    parser.add_option("--build-fail",
1352
                      action="store", type="int", dest="build_fail",
1353
                      metavar="BUILD_TIMEOUT",
1354
                      help="Fail the test if TIMEOUT seconds have passed " \
1355
                           "and a build operation is still incomplete",
1356
                      default=900)
1357
    parser.add_option("--query-interval",
1358
                      action="store", type="int", dest="query_interval",
1359
                      metavar="INTERVAL",
1360
                      help="Query server status when requests are pending " \
1361
                           "every INTERVAL seconds",
1362
                      default=3)
1363
    parser.add_option("--fanout",
1364
                      action="store", type="int", dest="fanout",
1365
                      metavar="COUNT",
1366
                      help="Spawn up to COUNT child processes to execute " \
1367
                           "in parallel, essentially have up to COUNT " \
1368
                           "server build requests outstanding (EXPERIMENTAL)",
1369
                      default=1)
1370
    parser.add_option("--force-flavor",
1371
                      action="store", type="int", dest="force_flavorid",
1372
                      metavar="FLAVOR ID",
1373
                      help="Force all server creations to use the specified "\
1374
                           "FLAVOR ID instead of a randomly chosen one, " \
1375
                           "useful if disk space is scarce",
1376
                      default=None)
1377
    parser.add_option("--image-id",
1378
                      action="store", type="string", dest="force_imageid",
1379
                      metavar="IMAGE ID",
1380
                      help="Test the specified image id, use 'all' to test " \
1381
                           "all available images (mandatory argument)",
1382
                      default=None)
1383
    parser.add_option("--show-stale",
1384
                      action="store_true", dest="show_stale",
1385
                      help="Show stale servers from previous runs, whose "\
1386
                           "name starts with `%s'" % SNF_TEST_PREFIX,
1387
                      default=False)
1388
    parser.add_option("--delete-stale",
1389
                      action="store_true", dest="delete_stale",
1390
                      help="Delete stale servers from previous runs, whose "\
1391
                           "name starts with `%s'" % SNF_TEST_PREFIX,
1392
                      default=False)
1393
    parser.add_option("--force-personality",
1394
                      action="store", type="string", dest="personality_path",
1395
                      help="Force a personality file injection.\
1396
                            File path required. ",
1397
                      default=None)
1398
    parser.add_option("--log-folder",
1399
                      action="store", type="string", dest="log_folder",
1400
                      help="Define the absolute path where the output \
1401
                            log is stored. ",
1402
                      default="/var/log/burnin/")
1403

    
1404
    # FIXME: Change the default for build-fanout to 10
1405
    # FIXME: Allow the user to specify a specific set of Images to test
1406

    
1407
    (opts, args) = parser.parse_args(args)
1408

    
1409
    # Verify arguments
1410
    if opts.delete_stale:
1411
        opts.show_stale = True
1412

    
1413
    if not opts.show_stale:
1414
        if not opts.force_imageid:
1415
            print >>sys.stderr, "The --image-id argument is mandatory."
1416
            parser.print_help()
1417
            sys.exit(1)
1418

    
1419
        if opts.force_imageid != 'all':
1420
            try:
1421
                opts.force_imageid = str(opts.force_imageid)
1422
            except ValueError:
1423
                print >>sys.stderr, "Invalid value specified for --image-id." \
1424
                                    "Use a valid id, or `all'."
1425
                sys.exit(1)
1426

    
1427
    return (opts, args)
1428

    
1429

    
1430
def main():
1431
    """Assemble test cases into a test suite, and run it
1432

1433
    IMPORTANT: Tests have dependencies and have to be run in the specified
1434
    order inside a single test case. They communicate through attributes of the
1435
    corresponding TestCase class (shared fixtures). Distinct subclasses of
1436
    TestCase MAY SHARE NO DATA, since they are run in parallel, in distinct
1437
    test runner processes.
1438

1439
    """
1440

    
1441
    (opts, args) = parse_arguments(sys.argv[1:])
1442

    
1443
    global API, TOKEN
1444
    API = opts.api
1445
    TOKEN = opts.token
1446

    
1447
    # Cleanup stale servers from previous runs
1448
    if opts.show_stale:
1449
        cleanup_servers(delete_stale=opts.delete_stale)
1450
        cleanup_networks(delete_stale=opts.delete_stale)
1451
        return 0
1452

    
1453
    # Initialize a kamaki instance, get flavors, images
1454

    
1455
    c = ComputeClient(API, TOKEN)
1456

    
1457
    DIMAGES = c.list_images(detail=True)
1458
    DFLAVORS = c.list_flavors(detail=True)
1459

    
1460
    # FIXME: logging, log, LOG PID, TEST_RUN_ID, arguments
1461
    # Run them: FIXME: In parallel, FAILEARLY, catchbreak?
1462
    #unittest.main(verbosity=2, catchbreak=True)
1463

    
1464
    if opts.force_imageid == 'all':
1465
        test_images = DIMAGES
1466
    else:
1467
        test_images = filter(lambda x: x["id"] == opts.force_imageid, DIMAGES)
1468

    
1469
    #New folder for log per image
1470

    
1471
    if not os.path.exists(opts.log_folder):
1472
        os.mkdir(opts.log_folder)
1473

    
1474
    test_folder = os.path.join(opts.log_folder, TEST_RUN_ID)
1475
    os.mkdir(test_folder)
1476

    
1477
    for image in test_images:
1478

    
1479
        imageid = str(image["id"])
1480

    
1481
        if opts.force_flavorid:
1482
            flavorid = opts.force_flavorid
1483
        else:
1484
            flavorid = choice([f["id"] for f in DFLAVORS if f["disk"] >= 20])
1485

    
1486
        imagename = image["name"]
1487

    
1488
        #Personality dictionary for file injection test
1489
        if opts.personality_path != None:
1490
            f = open(opts.personality_path)
1491
            content = b64encode(f.read())
1492
            personality = []
1493
            st = os.stat(opts.personality_path)
1494
            personality.append({
1495
                    'path': '/root/test_inj_file',
1496
                    'owner': 'root',
1497
                    'group': 'root',
1498
                    'mode': 0x7777 & st.st_mode,
1499
                    'contents': content
1500
                    })
1501
        else:
1502
            personality = None
1503

    
1504
        servername = "%s%s for %s" % (SNF_TEST_PREFIX, TEST_RUN_ID, imagename)
1505
        is_windows = imagename.lower().find("windows") >= 0
1506

    
1507
        ServerTestCase = _spawn_server_test_case(
1508
            imageid=imageid,
1509
            flavorid=flavorid,
1510
            imagename=imagename,
1511
            personality=personality,
1512
            servername=servername,
1513
            is_windows=is_windows,
1514
            action_timeout=opts.action_timeout,
1515
            build_warning=opts.build_warning,
1516
            build_fail=opts.build_fail,
1517
            query_interval=opts.query_interval,
1518
            )
1519

    
1520
        NetworkTestCase = _spawn_network_test_case(
1521
            action_timeout=opts.action_timeout,
1522
            imageid=imageid,
1523
            flavorid=flavorid,
1524
            imagename=imagename,
1525
            query_interval=opts.query_interval,
1526
            )
1527

    
1528
        seq_cases = [UnauthorizedTestCase, ImagesTestCase, FlavorsTestCase,
1529
                     ServersTestCase, ServerTestCase, NetworkTestCase]
1530

    
1531
        #folder for each image
1532
        image_folder = os.path.join(test_folder, imageid)
1533
        os.mkdir(image_folder)
1534

    
1535
        for case in seq_cases:
1536
            log_file = os.path.join(image_folder, 'details_' +
1537
                                    (case.__name__) + "_" +
1538
                                    TEST_RUN_ID + '.log')
1539
            fail_file = os.path.join(image_folder, 'failed_' +
1540
                                     (case.__name__) + "_" +
1541
                                     TEST_RUN_ID + '.log')
1542
            error_file = os.path.join(image_folder, 'error_' +
1543
                                      (case.__name__) + "_" +
1544
                                      TEST_RUN_ID + '.log')
1545

    
1546
            f = open(log_file, "w")
1547
            fail = open(fail_file, "w")
1548
            error = open(error_file, "w")
1549

    
1550
            suite = unittest.TestLoader().loadTestsFromTestCase(case)
1551
            runner = unittest.TextTestRunner(f, verbosity=2, failfast=True)
1552
            result = runner.run(suite)
1553

    
1554
            for res in result.errors:
1555
                error.write(str(res[0]) + '\n')
1556
                error.write(str(res[0].shortDescription()) + '\n')
1557
                error.write('\n')
1558

    
1559
            for res in result.failures:
1560
                fail.write(str(res[0]) + '\n')
1561
                fail.write(str(res[0].shortDescription()) + '\n')
1562
                fail.write('\n')
1563
                if opts.nofailfast == False:
1564
                    sys.exit()
1565

    
1566
if __name__ == "__main__":
1567
    sys.exit(main())