Statistics
| Branch: | Tag: | Revision:

root / snf-cyclades-app / synnefo / tools / burnin.py @ 81e8cbf6

History | View | Annotate | Download (55.2 kB)

1
#!/usr/bin/env python
2

    
3
# Copyright 2011 GRNET S.A. All rights reserved.
4
#
5
# Redistribution and use in source and binary forms, with or
6
# without modification, are permitted provided that the following
7
# conditions are met:
8
#
9
#   1. Redistributions of source code must retain the above
10
#      copyright notice, this list of conditions and the following
11
#      disclaimer.
12
#
13
#   2. Redistributions in binary form must reproduce the above
14
#      copyright notice, this list of conditions and the following
15
#      disclaimer in the documentation and/or other materials
16
#      provided with the distribution.
17
#
18
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
19
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
20
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
21
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
22
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
25
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
26
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
28
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29
# POSSIBILITY OF SUCH DAMAGE.
30
#
31
# The views and conclusions contained in the software and
32
# documentation are those of the authors and should not be
33
# interpreted as representing official policies, either expressed
34
# or implied, of GRNET S.A.
35

    
36
"""Perform integration testing on a running Synnefo deployment"""
37

    
38
import __main__
39
import datetime
40
import inspect
41
import logging
42
import os
43
import paramiko
44
import prctl
45
import subprocess
46
import signal
47
import socket
48
import struct
49
import sys
50
import time
51
import hashlib
52
from base64 import b64encode
53
from pwd import getpwuid
54
from grp import getgrgid
55
from IPy import IP
56
from multiprocessing import Process, Queue
57
from random import choice
58

    
59
from kamaki.clients.compute import ComputeClient
60
from kamaki.clients.cyclades import CycladesClient
61
from kamaki.clients import ClientError
62

    
63
from fabric.api import *
64

    
65
from vncauthproxy.d3des import generate_response as d3des_generate_response
66

    
67
# Use backported unittest functionality if Python < 2.7
68
try:
69
    import unittest2 as unittest
70
except ImportError:
71
    if sys.version_info < (2, 7):
72
        raise Exception("The unittest2 package is required for Python < 2.7")
73
    import unittest
74

    
75

    
76
API = None
77
TOKEN = None
78
DEFAULT_API = "https://cyclades.okeanos.grnet.gr/api/v1.1"
79

    
80
# A unique id identifying this test run
81
TEST_RUN_ID = datetime.datetime.strftime(datetime.datetime.now(),
82
                                         "%Y%m%d%H%M%S")
83
SNF_TEST_PREFIX = "snf-test-"
84

    
85
# Setup logging (FIXME - verigak)
86
logging.basicConfig(format="%(message)s")
87
log = logging.getLogger("burnin")
88
log.setLevel(logging.INFO)
89

    
90
class UnauthorizedTestCase(unittest.TestCase):
91
    def test_unauthorized_access(self):
92
        """Test access without a valid token fails"""
93
        log.info("Authentication test")
94

    
95
        falseToken = '12345'
96
        c=ComputeClient(API, falseToken)
97

    
98
        with self.assertRaises(ClientError) as cm:
99
            c.list_servers()
100
            self.assertEqual(cm.exception.status, 401)
101

    
102

    
103
class ImagesTestCase(unittest.TestCase):
104
    """Test image lists for consistency"""
105
    @classmethod
106
    def setUpClass(cls):
107
        """Initialize kamaki, get (detailed) list of images"""
108
        log.info("Getting simple and detailed list of images")
109

    
110
        cls.client = ComputeClient(API, TOKEN)
111
        cls.images = cls.client.list_images()
112
        cls.dimages = cls.client.list_images(detail=True)
113

    
114
    def test_001_list_images(self):
115
        """Test image list actually returns images"""
116
        self.assertGreater(len(self.images), 0)
117
        self.assertEqual(len(self.images), -1)
118

    
119
    def test_002_list_images_detailed(self):
120
        """Test detailed image list is the same length as list"""
121
        self.assertEqual(len(self.dimages), len(self.images))
122
        self.assertEqual(len(self.dimages), 0)
123

    
124
    def test_003_same_image_names(self):
125
        """Test detailed and simple image list contain same names"""
126
        names = sorted(map(lambda x: x["name"], self.images))
127
        dnames = sorted(map(lambda x: x["name"], self.dimages))
128
        self.assertEqual(names, dnames)
129

    
130
    def test_004_unique_image_names(self):
131
        """Test images have unique names"""
132
        names = sorted(map(lambda x: x["name"], self.images))
133
        self.assertEqual(sorted(list(set(names))), names)
134

    
135
    def test_005_image_metadata(self):
136
        """Test every image has specific metadata defined"""
137
        keys = frozenset(["os", "description", "size"])
138
        for i in self.dimages:
139
            self.assertTrue(keys.issubset(i["metadata"]["values"].keys()))
140

    
141

    
142
class FlavorsTestCase(unittest.TestCase):
143
    """Test flavor lists for consistency"""
144
    @classmethod
145
    def setUpClass(cls):
146
        """Initialize kamaki, get (detailed) list of flavors"""
147
        log.info("Getting simple and detailed list of flavors")
148

    
149
        cls.client = ComputeClient(API, TOKEN)
150
        cls.flavors = cls.client.list_flavors()
151
        cls.dflavors = cls.client.list_flavors(detail=True)
152

    
153
    def test_001_list_flavors(self):
154
        """Test flavor list actually returns flavors"""
155
        self.assertGreater(len(self.flavors), 0)
156

    
157
    def test_002_list_flavors_detailed(self):
158
        """Test detailed flavor list is the same length as list"""
159
        self.assertEquals(len(self.dflavors), len(self.flavors))
160

    
161
    def test_003_same_flavor_names(self):
162
        """Test detailed and simple flavor list contain same names"""
163
        names = sorted(map(lambda x: x["name"], self.flavors))
164
        dnames = sorted(map(lambda x: x["name"], self.dflavors))
165
        self.assertEqual(names, dnames)
166

    
167
    def test_004_unique_flavor_names(self):
168
        """Test flavors have unique names"""
169
        names = sorted(map(lambda x: x["name"], self.flavors))
170
        self.assertEqual(sorted(list(set(names))), names)
171

    
172
    def test_005_well_formed_flavor_names(self):
173
        """Test flavors have names of the form CxxRyyDzz
174

175
        Where xx is vCPU count, yy is RAM in MiB, zz is Disk in GiB
176

177
        """
178
        for f in self.dflavors:
179
            self.assertEqual("C%dR%dD%d" % (f["cpu"], f["ram"], f["disk"]),
180
                             f["name"],
181
                             "Flavor %s does not match its specs." % f["name"])
182

    
183

    
184
class ServersTestCase(unittest.TestCase):
185
    """Test server lists for consistency"""
186
    @classmethod
187
    def setUpClass(cls):
188
        """Initialize kamaki, get (detailed) list of servers"""
189
        log.info("Getting simple and detailed list of servers")
190

    
191
        cls.client = ComputeClient(API, TOKEN)
192
        cls.servers = cls.client.list_servers()
193
        cls.dservers = cls.client.list_servers(detail=True)
194

    
195
    def test_001_list_servers(self):
196
        """Test server list actually returns servers"""
197
        self.assertGreater(len(self.servers), 0)
198

    
199
    def test_002_list_servers_detailed(self):
200
        """Test detailed server list is the same length as list"""
201
        self.assertEqual(len(self.dservers), len(self.servers))
202

    
203
    def test_003_same_server_names(self):
204
        """Test detailed and simple flavor list contain same names"""
205
        names = sorted(map(lambda x: x["name"], self.servers))
206
        dnames = sorted(map(lambda x: x["name"], self.dservers))
207
        self.assertEqual(names, dnames)
208

    
209

    
210
# This class gets replicated into actual TestCases dynamically
211
class SpawnServerTestCase(unittest.TestCase):
212
    """Test scenario for server of the specified image"""
213

    
214
    @classmethod
215
    def setUpClass(cls):
216
        """Initialize a kamaki instance"""
217
        log.info("Spawning server for image `%s'", cls.imagename)
218

    
219
        cls.client = ComputeClient(API, TOKEN)
220
        cls.cyclades = CycladesClient(API, TOKEN)
221

    
222
    def _get_ipv4(self, server):
223
        """Get the public IPv4 of a server from the detailed server info"""
224

    
225
        public_addrs = filter(lambda x: x["id"] == "public",
226
                              server["addresses"]["values"])
227
        self.assertEqual(len(public_addrs), 1)
228
        ipv4_addrs = filter(lambda x: x["version"] == 4,
229
                            public_addrs[0]["values"])
230
        self.assertEqual(len(ipv4_addrs), 1)
231
        return ipv4_addrs[0]["addr"]
232

    
233
    def _get_ipv6(self, server):
234
        """Get the public IPv6 of a server from the detailed server info"""
235
        public_addrs = filter(lambda x: x["id"] == "public",
236
                              server["addresses"]["values"])
237
        self.assertEqual(len(public_addrs), 1)
238
        ipv6_addrs = filter(lambda x: x["version"] == 6,
239
                            public_addrs[0]["values"])
240
        self.assertEqual(len(ipv6_addrs), 1)
241
        return ipv6_addrs[0]["addr"]
242

    
243
    def _connect_loginname(self, os):
244
        """Return the login name for connections based on the server OS"""
245
        if os in ("Ubuntu", "Kubuntu", "Fedora"):
246
            return "user"
247
        elif os in ("windows", "windows_alpha1"):
248
            return "Administrator"
249
        else:
250
            return "root"
251

    
252
    def _verify_server_status(self, current_status, new_status):
253
        """Verify a server has switched to a specified status"""
254
        server = self.client.get_server_details(self.serverid)
255
        if server["status"] not in (current_status, new_status):
256
            return None  # Do not raise exception, return so the test fails
257
        self.assertEquals(server["status"], new_status)
258

    
259
    def _get_connected_tcp_socket(self, family, host, port):
260
        """Get a connected socket from the specified family to host:port"""
261
        sock = None
262
        for res in \
263
            socket.getaddrinfo(host, port, family, socket.SOCK_STREAM, 0,
264
                               socket.AI_PASSIVE):
265
            af, socktype, proto, canonname, sa = res
266
            try:
267
                sock = socket.socket(af, socktype, proto)
268
            except socket.error as msg:
269
                sock = None
270
                continue
271
            try:
272
                sock.connect(sa)
273
            except socket.error as msg:
274
                sock.close()
275
                sock = None
276
                continue
277
        self.assertIsNotNone(sock)
278
        return sock
279

    
280
    def _ping_once(self, ipv6, ip):
281
        """Test server responds to a single IPv4 or IPv6 ping"""
282
        cmd = "ping%s -c 2 -w 3 %s" % ("6" if ipv6 else "", ip)
283
        ping = subprocess.Popen(cmd, shell=True,
284
                                stdout=subprocess.PIPE, stderr=subprocess.PIPE)
285
        (stdout, stderr) = ping.communicate()
286
        ret = ping.wait()
287
        self.assertEquals(ret, 0)
288

    
289
    def _get_hostname_over_ssh(self, hostip, username, password):
290
        ssh = paramiko.SSHClient()
291
        ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
292
        try:
293
            ssh.connect(hostip, username=username, password=password)
294
        except socket.error:
295
            raise AssertionError
296
        stdin, stdout, stderr = ssh.exec_command("hostname")
297
        lines = stdout.readlines()
298
        self.assertEqual(len(lines), 1)
299
        return lines[0]
300

    
301
    def _try_until_timeout_expires(self, warn_timeout, fail_timeout,
302
                                   opmsg, callable, *args, **kwargs):
303
        if warn_timeout == fail_timeout:
304
            warn_timeout = fail_timeout + 1
305
        warn_tmout = time.time() + warn_timeout
306
        fail_tmout = time.time() + fail_timeout
307
        while True:
308
            self.assertLess(time.time(), fail_tmout,
309
                            "operation `%s' timed out" % opmsg)
310
            if time.time() > warn_tmout:
311
                log.warning("Server %d: `%s' operation `%s' not done yet",
312
                            self.serverid, self.servername, opmsg)
313
            try:
314
                log.info("%s... " % opmsg)
315
                return callable(*args, **kwargs)
316
            except AssertionError:
317
                pass
318
            time.sleep(self.query_interval)
319

    
320
    def _insist_on_tcp_connection(self, family, host, port):
321
        familystr = {socket.AF_INET: "IPv4", socket.AF_INET6: "IPv6",
322
                     socket.AF_UNSPEC: "Unspecified-IPv4/6"}
323
        msg = "connect over %s to %s:%s" % \
324
              (familystr.get(family, "Unknown"), host, port)
325
        sock = self._try_until_timeout_expires(
326
                self.action_timeout, self.action_timeout,
327
                msg, self._get_connected_tcp_socket,
328
                family, host, port)
329
        return sock
330

    
331
    def _insist_on_status_transition(self, current_status, new_status,
332
                                    fail_timeout, warn_timeout=None):
333
        msg = "Server %d: `%s', waiting for %s -> %s" % \
334
              (self.serverid, self.servername, current_status, new_status)
335
        if warn_timeout is None:
336
            warn_timeout = fail_timeout
337
        self._try_until_timeout_expires(warn_timeout, fail_timeout,
338
                                        msg, self._verify_server_status,
339
                                        current_status, new_status)
340
        # Ensure the status is actually the expected one
341
        server = self.client.get_server_details(self.serverid)
342
        self.assertEquals(server["status"], new_status)
343

    
344
    def _insist_on_ssh_hostname(self, hostip, username, password):
345
        msg = "SSH to %s, as %s/%s" % (hostip, username, password)
346
        hostname = self._try_until_timeout_expires(
347
                self.action_timeout, self.action_timeout,
348
                msg, self._get_hostname_over_ssh,
349
                hostip, username, password)
350

    
351
        # The hostname must be of the form 'prefix-id'
352
        self.assertTrue(hostname.endswith("-%d\n" % self.serverid))
353

    
354
    def _check_file_through_ssh(self, hostip, username, password, remotepath, content):
355
        msg = "Trying file injection through SSH to %s, as %s/%s" % (hostip, username, password)
356
        log.info(msg)
357
        try:
358
            ssh = paramiko.SSHClient()
359
            ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
360
            ssh.connect(hostip, username=username, password=password)
361
        except socket.error:
362
            raise AssertionError
363
        
364
        transport = paramiko.Transport((hostip,22))
365
        transport.connect(username = username, password = password)
366

    
367
        localpath = '/tmp/'+SNF_TEST_PREFIX+'injection'
368
        sftp = paramiko.SFTPClient.from_transport(transport)
369
        sftp.get(remotepath, localpath)
370
        
371
        sftp.close()
372
        transport.close()
373

    
374
        f = open(localpath)
375
        remote_content = b64encode(f.read())
376

    
377
        # Check if files are the same
378
        return (remote_content == content)
379

    
380
    def _skipIf(self, condition, msg):
381
        if condition:
382
            self.skipTest(msg)
383

    
384
    def test_001_submit_create_server(self):
385
        """Test submit create server request"""
386

    
387
        log.info("Submit new server request")
388
        server = self.client.create_server(self.servername, self.flavorid,
389
                                           self.imageid, self.personality)
390

    
391
        log.info("Server id: " + str(server["id"]))
392
        log.info("Server password: " + server["adminPass"])
393
        
394
        self.assertEqual(server["name"], self.servername)
395
        self.assertEqual(server["flavorRef"], self.flavorid)
396
        self.assertEqual(server["imageRef"], self.imageid)
397
        self.assertEqual(server["status"], "BUILD")
398

    
399
        # Update class attributes to reflect data on building server
400
        cls = type(self)
401
        cls.serverid = server["id"]
402
        cls.username = None
403
        cls.passwd = server["adminPass"]
404

    
405
        
406

    
407
    def test_002a_server_is_building_in_list(self):
408
        """Test server is in BUILD state, in server list"""
409
        
410
        log.info("Server in BUILD state in server list")
411

    
412
        servers = self.client.list_servers(detail=True)
413
        servers = filter(lambda x: x["name"] == self.servername, servers)
414
        self.assertEqual(len(servers), 1)
415
        server = servers[0]
416
        self.assertEqual(server["name"], self.servername)
417
        self.assertEqual(server["flavorRef"], self.flavorid)
418
        self.assertEqual(server["imageRef"], self.imageid)
419
        self.assertEqual(server["status"], "BUILD")
420

    
421
    def test_002b_server_is_building_in_details(self):
422
        """Test server is in BUILD state, in details"""
423

    
424
        log.info("Server in BUILD state in details")
425

    
426
        server = self.client.get_server_details(self.serverid)
427
        self.assertEqual(server["name"], self.servername)
428
        self.assertEqual(server["flavorRef"], self.flavorid)
429
        self.assertEqual(server["imageRef"], self.imageid)
430
        self.assertEqual(server["status"], "BUILD")
431

    
432
    def test_002c_set_server_metadata(self):
433

    
434
        log.info("Creating server metadata")
435

    
436
        image = self.client.get_image_details(self.imageid)
437
        os = image["metadata"]["values"]["os"]
438
        loginname = image["metadata"]["values"].get("users", None)
439
        self.client.update_server_metadata(self.serverid, OS=os)
440

    
441
        userlist = loginname.split()
442

    
443
        # Determine the username to use for future connections
444
        # to this host
445
        cls = type(self)
446

    
447
        if "root" in userlist:
448
            cls.username = "root"
449
        elif users == None:
450
            cls.username = self._connect_loginname(os)
451
        else:
452
            cls.username = choice(userlist)
453

    
454
        self.assertIsNotNone(cls.username)
455

    
456
    def test_002d_verify_server_metadata(self):
457
        """Test server metadata keys are set based on image metadata"""
458

    
459
        log.info("Verifying image metadata")
460

    
461
        servermeta = self.client.get_server_metadata(self.serverid)
462
        imagemeta = self.client.get_image_metadata(self.imageid)
463

    
464
        self.assertEqual(servermeta["OS"], imagemeta["os"])
465

    
466
    def test_003_server_becomes_active(self):
467
        """Test server becomes ACTIVE"""
468

    
469
        log.info("Waiting for server to become ACTIVE")
470

    
471
        self._insist_on_status_transition("BUILD", "ACTIVE",
472
                                         self.build_fail, self.build_warning)
473

    
474
    def test_003a_get_server_oob_console(self):
475
        """Test getting OOB server console over VNC
476

477
        Implementation of RFB protocol follows
478
        http://www.realvnc.com/docs/rfbproto.pdf.
479

480
        """
481
        
482
        console = self.cyclades.get_server_console(self.serverid)
483
        self.assertEquals(console['type'], "vnc")
484
        sock = self._insist_on_tcp_connection(socket.AF_UNSPEC,
485
                                        console["host"], console["port"])
486

    
487
        # Step 1. ProtocolVersion message (par. 6.1.1)
488
        version = sock.recv(1024)
489
        self.assertEquals(version, 'RFB 003.008\n')
490
        sock.send(version)
491

    
492
        # Step 2. Security (par 6.1.2): Only VNC Authentication supported
493
        sec = sock.recv(1024)
494
        self.assertEquals(list(sec), ['\x01', '\x02'])
495

    
496
        # Step 3. Request VNC Authentication (par 6.1.2)
497
        sock.send('\x02')
498

    
499
        # Step 4. Receive Challenge (par 6.2.2)
500
        challenge = sock.recv(1024)
501
        self.assertEquals(len(challenge), 16)
502

    
503
        # Step 5. DES-Encrypt challenge, use password as key (par 6.2.2)
504
        response = d3des_generate_response(
505
            (console["password"] + '\0' * 8)[:8], challenge)
506
        sock.send(response)
507

    
508
        # Step 6. SecurityResult (par 6.1.3)
509
        result = sock.recv(4)
510
        self.assertEquals(list(result), ['\x00', '\x00', '\x00', '\x00'])
511
        sock.close()
512
        
513
    def test_004_server_has_ipv4(self):
514
        """Test active server has a valid IPv4 address"""
515

    
516
        log.info("Validate server's IPv4")
517

    
518
        server = self.client.get_server_details(self.serverid)
519
        ipv4 = self._get_ipv4(server)
520
        self.assertEquals(IP(ipv4).version(), 4)
521

    
522
    def test_005_server_has_ipv6(self):
523
        """Test active server has a valid IPv6 address"""
524

    
525
        log.info("Validate server's IPv6")
526

    
527
        server = self.client.get_server_details(self.serverid)
528
        ipv6 = self._get_ipv6(server)
529
        self.assertEquals(IP(ipv6).version(), 6)
530

    
531
    def test_006_server_responds_to_ping_IPv4(self):
532
        """Test server responds to ping on IPv4 address"""
533

    
534
        log.info("Testing if server responds to pings in IPv4")
535

    
536
        server = self.client.get_server_details(self.serverid)
537
        ip = self._get_ipv4(server)
538
        self._try_until_timeout_expires(self.action_timeout,
539
                                        self.action_timeout,
540
                                        "PING IPv4 to %s" % ip,
541
                                        self._ping_once,
542
                                        False, ip)
543

    
544
    def test_007_server_responds_to_ping_IPv6(self):
545
        """Test server responds to ping on IPv6 address"""
546

    
547
        log.info("Testing if server responds to pings in IPv6")
548

    
549
        server = self.client.get_server_details(self.serverid)
550
        ip = self._get_ipv6(server)
551
        self._try_until_timeout_expires(self.action_timeout,
552
                                        self.action_timeout,
553
                                        "PING IPv6 to %s" % ip,
554
                                        self._ping_once,
555
                                        True, ip)
556

    
557
    def test_008_submit_shutdown_request(self):
558
        """Test submit request to shutdown server"""
559

    
560
        log.info("Shutting down server")
561

    
562
        self.cyclades.shutdown_server(self.serverid)
563

    
564
    def test_009_server_becomes_stopped(self):
565
        """Test server becomes STOPPED"""
566

    
567
        log.info("Waiting until server becomes STOPPED")
568
        
569
        self._insist_on_status_transition("ACTIVE", "STOPPED",
570
                                         self.action_timeout,
571
                                         self.action_timeout)
572

    
573
    def test_010_submit_start_request(self):
574
        """Test submit start server request"""
575

    
576
        log.info("Starting server")
577

    
578
        self.cyclades.start_server(self.serverid)
579

    
580
    def test_011_server_becomes_active(self):
581
        """Test server becomes ACTIVE again"""
582

    
583
        log.info("Waiting until server becomes ACTIVE")
584
        
585
        self._insist_on_status_transition("STOPPED", "ACTIVE",
586
                                         self.action_timeout,
587
                                         self.action_timeout)
588

    
589
    def test_011a_server_responds_to_ping_IPv4(self):
590
        """Test server OS is actually up and running again"""
591

    
592
        log.info("Testing if server is actually up and running")
593

    
594
        self.test_006_server_responds_to_ping_IPv4()
595

    
596
    def test_012_ssh_to_server_IPv4(self):
597
        """Test SSH to server public IPv4 works, verify hostname"""
598

    
599
        self._skipIf(self.is_windows, "only valid for Linux servers")
600
        server = self.client.get_server_details(self.serverid)
601
        self._insist_on_ssh_hostname(self._get_ipv4(server),
602
                                     self.username, self.passwd)
603

    
604
    def test_013_ssh_to_server_IPv6(self):
605
        """Test SSH to server public IPv6 works, verify hostname"""
606
        self._skipIf(self.is_windows, "only valid for Linux servers")
607
        server = self.client.get_server_details(self.serverid)
608
        self._insist_on_ssh_hostname(self._get_ipv6(server),
609
                                     self.username, self.passwd)
610

    
611
    def test_014_rdp_to_server_IPv4(self):
612
        "Test RDP connection to server public IPv4 works"""
613
        self._skipIf(not self.is_windows, "only valid for Windows servers")
614
        server = self.client.get_server_details(self.serverid)
615
        ipv4 = self._get_ipv4(server)
616
        sock = _insist_on_tcp_connection(socket.AF_INET, ipv4, 3389)
617

    
618
        # No actual RDP processing done. We assume the RDP server is there
619
        # if the connection to the RDP port is successful.
620
        # FIXME: Use rdesktop, analyze exit code? see manpage [costasd]
621
        sock.close()
622

    
623
    def test_015_rdp_to_server_IPv6(self):
624
        "Test RDP connection to server public IPv6 works"""
625
        self._skipIf(not self.is_windows, "only valid for Windows servers")
626
        server = self.client.get_server_details(self.serverid)
627
        ipv6 = self._get_ipv6(server)
628
        sock = _get_tcp_connection(socket.AF_INET6, ipv6, 3389)
629

    
630
        # No actual RDP processing done. We assume the RDP server is there
631
        # if the connection to the RDP port is successful.
632
        sock.close()
633

    
634

    
635
    def test_016_personality_is_enforced(self):
636
        """Test file injection for personality enforcement"""
637
        self._skipIf(self.is_windows, "only implemented for Linux servers")
638
        self._skipIf(self.personality == None, "No personality file selected")
639

    
640
        log.info("Trying to inject file for personality enforcement")
641

    
642

    
643
        server = self.client.get_server_details(self.serverid)
644

    
645
        for inj_file in self.personality:
646
            equal_files = self._check_file_through_ssh(self._get_ipv4(server), inj_file['owner'], 
647
                                                       self.passwd, inj_file['path'], inj_file['contents'])
648
            self.assertTrue(equal_files)
649
        
650

    
651
    def test_017_submit_delete_request(self):
652
        """Test submit request to delete server"""
653

    
654
        log.info("Deleting server")
655

    
656
        self.client.delete_server(self.serverid)
657

    
658
    def test_018_server_becomes_deleted(self):
659
        """Test server becomes DELETED"""
660

    
661
        log.info("Testing if server becomes DELETED")
662

    
663
        self._insist_on_status_transition("ACTIVE", "DELETED",
664
                                         self.action_timeout,
665
                                         self.action_timeout)
666

    
667
    def test_019_server_no_longer_in_server_list(self):
668
        """Test server is no longer in server list"""
669

    
670
        log.info("Test if server is no longer listed")
671

    
672
        servers = self.client.list_servers()
673
        self.assertNotIn(self.serverid, [s["id"] for s in servers])
674

    
675

    
676
class NetworkTestCase(unittest.TestCase):
677
    """ Testing networking in cyclades """
678
  
679
    @classmethod
680
    def setUpClass(cls):
681
        "Initialize kamaki, get list of current networks"
682

    
683
        cls.client = CycladesClient(API, TOKEN)
684
        cls.compute = ComputeClient(API, TOKEN)
685

    
686
        cls.servername = "%s%s for %s" % (SNF_TEST_PREFIX, TEST_RUN_ID, cls.imagename)
687

    
688
        #Dictionary initialization for the vms credentials
689
        cls.serverid = dict()
690
        cls.username = dict()
691
        cls.password = dict()
692

    
693
    def _get_ipv4(self, server):
694
        """Get the public IPv4 of a server from the detailed server info"""
695

    
696
        public_addrs = filter(lambda x: x["id"] == "public",
697
                              server["addresses"]["values"])
698
        self.assertEqual(len(public_addrs), 1)
699
        ipv4_addrs = filter(lambda x: x["version"] == 4,
700
                            public_addrs[0]["values"])
701
        self.assertEqual(len(ipv4_addrs), 1)
702
        return ipv4_addrs[0]["addr"]
703
    
704

    
705
    def _connect_loginname(self, os):
706
        """Return the login name for connections based on the server OS"""
707
        if os in ("Ubuntu", "Kubuntu", "Fedora"):
708
            return "user"
709
        elif os in ("windows", "windows_alpha1"):
710
            return "Administrator"
711
        else:
712
            return "root"
713

    
714
    def _ping_once(self, ip):
715
    
716
        """Test server responds to a single IPv4 or IPv6 ping"""
717
        cmd = "ping -c 2 -w 3 %s" % (ip)
718
        ping = subprocess.Popen(cmd, shell=True,
719
                                stdout=subprocess.PIPE, stderr=subprocess.PIPE)
720
        (stdout, stderr) = ping.communicate()
721
        ret = ping.wait()
722
        
723
        return (ret == 0)
724

    
725

    
726
    def test_00001a_submit_create_server_A(self):
727
        """Test submit create server request"""
728

    
729
        log.info("Creating test server A")
730

    
731
        serverA = self.client.create_server(self.servername, self.flavorid,
732
                                            self.imageid, personality=None)
733

    
734
        self.assertEqual(serverA["name"], self.servername)
735
        self.assertEqual(serverA["flavorRef"], self.flavorid)
736
        self.assertEqual(serverA["imageRef"], self.imageid)
737
        self.assertEqual(serverA["status"], "BUILD")
738

    
739
        # Update class attributes to reflect data on building server
740
        self.serverid['A'] = serverA["id"]
741
        self.username['A'] = None
742
        self.password['A'] = serverA["adminPass"]
743

    
744
        log.info("Server A id:" + str(serverA["id"]))
745
        log.info("Server password " + (self.password['A']))
746
        
747

    
748

    
749
    def test_00001b_serverA_becomes_active(self):
750
        """Test server becomes ACTIVE"""
751
        
752
        log.info("Waiting until test server A becomes ACTIVE")
753

    
754
        fail_tmout = time.time()+self.action_timeout
755
        while True:
756
            d = self.client.get_server_details(self.serverid['A'])
757
            status = d['status']
758
            if status == 'ACTIVE':
759
                active = True
760
                break
761
            elif time.time() > fail_tmout:
762
                self.assertLess(time.time(), fail_tmout)
763
            else:
764
                time.sleep(self.query_interval)
765

    
766
        self.assertTrue(active)
767

    
768
    
769
    def test_00002a_submit_create_server_B(self):
770
        """Test submit create server request"""
771
        
772
        log.info("Creating test server B")
773

    
774
        serverB = self.client.create_server(self.servername, self.flavorid,
775
                                            self.imageid, personality=None)
776

    
777

    
778
        self.assertEqual(serverB["name"], self.servername)
779
        self.assertEqual(serverB["flavorRef"], self.flavorid)
780
        self.assertEqual(serverB["imageRef"], self.imageid)
781
        self.assertEqual(serverB["status"], "BUILD")
782

    
783
        # Update class attributes to reflect data on building server
784
        self.serverid['B'] = serverB["id"]
785
        self.username['B'] = None
786
        self.password['B'] = serverB["adminPass"]
787

    
788
        log.info("Server B id: " + str(serverB["id"]))
789
        log.info("Password " + (self.password['B']))
790

    
791

    
792

    
793
    def test_00002b_serverB_becomes_active(self):
794
        """Test server becomes ACTIVE"""
795

    
796
        log.info("Waiting until test server B becomes ACTIVE")
797

    
798
        fail_tmout = time.time()+self.action_timeout
799
        while True:
800
            d = self.client.get_server_details(self.serverid['B'])
801
            status = d['status']
802
            if status == 'ACTIVE':
803
                active = True
804
                break
805
            elif time.time() > fail_tmout:
806
                self.assertLess(time.time(), fail_tmout)
807
            else:
808
                time.sleep(self.query_interval)
809

    
810
        self.assertTrue(active)
811

    
812

    
813
    def test_001_create_network(self):
814
        """Test submit create network request"""
815

    
816
        log.info("Submit new network request")
817

    
818
        name = SNF_TEST_PREFIX+TEST_RUN_ID
819
        previous_num = len(self.client.list_networks())
820
        network =  self.client.create_network(name)        
821
       
822
        #Test if right name is assigned
823
        self.assertEqual(network['name'], name)
824
        
825
        # Update class attributes
826
        cls = type(self)
827
        cls.networkid = network['id']
828
        networks = self.client.list_networks()
829

    
830
        #Test if new network is created
831
        self.assertTrue(len(networks) > previous_num)
832
        
833
    
834
    def test_002_connect_to_network(self):
835
        """Test connect VMs to network"""
836

    
837
        log.info("Connect VMs to private network")
838

    
839
        self.client.connect_server(self.serverid['A'], self.networkid)
840
        self.client.connect_server(self.serverid['B'], self.networkid)
841
                
842
        #Insist on connecting until action timeout
843
        fail_tmout = time.time()+self.action_timeout
844

    
845
        while True:
846
            connected = (self.client.get_network_details(self.networkid))
847
            connections = connected['servers']['values']
848
            if (self.serverid['A'] in connections) and (self.serverid['B'] in connections):
849
                conn_exists = True
850
                break
851
            elif time.time() > fail_tmout:
852
                self.assertLess(time.time(), fail_tmout)
853
            else:
854
                time.sleep(self.query_interval)
855

    
856
        self.assertTrue(conn_exists)
857

    
858
    
859
    def test_002a_reboot(self):
860
        """Rebooting server A"""
861

    
862
        log.info("Rebooting server A")
863

    
864
        self.client.shutdown_server(self.serverid['A']) 
865
       
866
        fail_tmout = time.time()+self.action_timeout
867
        while True:
868
            d = self.client.get_server_details(self.serverid['A'])
869
            status = d['status']
870
            if status == 'STOPPED':
871
                break
872
            elif time.time() > fail_tmout:
873
                self.assertLess(time.time(), fail_tmout)
874
            else:
875
                time.sleep(self.query_interval)
876

    
877
        self.client.start_server(self.serverid['A'])
878

    
879
        while True:
880
            d = self.client.get_server_details(self.serverid['A'])
881
            status = d['status']
882
            if status == 'ACTIVE':
883
                active = True
884
                break
885
            elif time.time() > fail_tmout:
886
                self.assertLess(time.time(), fail_tmout)
887
            else:
888
                time.sleep(self.query_interval)
889

    
890

    
891
               
892
        self.assertTrue(active)
893

    
894

    
895
    def test_002b_ping_server_A(self):
896
        "Test if server A is pingable"
897

    
898
        log.info("Testing if server A is pingable")
899

    
900
        server = self.client.get_server_details(self.serverid['A'])
901
        ip = self._get_ipv4(server)
902
        
903
        fail_tmout = time.time()+self.action_timeout
904
        
905
        s = False
906

    
907
        while True:
908

    
909
            if self._ping_once(ip):
910
                s = True
911
                break
912

    
913
            elif time.time() > fail_tmout:
914
                self.assertLess(time.time(), fail_tmout)
915

    
916
            else:
917
                time.sleep(self.query_interval)
918

    
919
        self.assertTrue(s)
920

    
921

    
922
    def test_002c_reboot(self):
923
        """Reboot server B"""
924

    
925
        log.info("Rebooting server B")
926

    
927
        self.client.shutdown_server(self.serverid['B']) 
928
        
929
        fail_tmout = time.time()+self.action_timeout
930
        while True:
931
            d = self.client.get_server_details(self.serverid['B'])
932
            status = d['status']
933
            if status == 'STOPPED':
934
                break
935
            elif time.time() > fail_tmout:
936
                self.assertLess(time.time(), fail_tmout)
937
            else:
938
                time.sleep(self.query_interval)
939

    
940
        self.client.start_server(self.serverid['B'])
941

    
942
        while True:
943
            d = self.client.get_server_details(self.serverid['B'])
944
            status = d['status']
945
            if status == 'ACTIVE':
946
                active = True
947
                break
948
            elif time.time() > fail_tmout:
949
                self.assertLess(time.time(), fail_tmout)
950
            else:
951
                time.sleep(self.query_interval)
952
                
953
        self.assertTrue(active)
954
        
955

    
956
    def test_002d_ping_server_B(self):
957
        """Test if server B is pingable"""
958

    
959

    
960
        log.info("Testing if server B is pingable")
961
        server = self.client.get_server_details(self.serverid['B'])
962
        ip = self._get_ipv4(server)
963
        
964
        fail_tmout = time.time()+self.action_timeout
965
        
966
        s = False
967

    
968
        while True:
969
            if self._ping_once(ip):
970
                s = True
971
                break
972

    
973
            elif time.time() > fail_tmout:
974
                self.assertLess(time.time(), fail_tmout)
975

    
976
            else:
977
                time.sleep(self.query_interval)
978

    
979
        self.assertTrue(s)
980

    
981

    
982
    def test_003a_setup_interface_A(self):
983
        """Set up eth1 for server A"""
984

    
985
        log.info("Setting up interface eth1 for server A")
986

    
987
        server = self.client.get_server_details(self.serverid['A'])
988
        image = self.client.get_image_details(self.imageid)
989
        os = image['metadata']['values']['os']
990
        
991
        users = image["metadata"]["values"].get("users", None)
992
        userlist = users.split()
993

    
994
        if "root" in userlist:
995
            loginname = "root"
996
        elif users == None:
997
            loginname = self._connect_loginname(os)
998
        else:
999
            loginname = choice(userlist)
1000

    
1001
        hostip = self._get_ipv4(server)
1002
        myPass = self.password['A']
1003
        
1004
        log.info("SSH in server A as %s/%s" % (loginname,myPass))
1005
        
1006
        res = False
1007

    
1008
        if loginname != "root":
1009
            with settings(
1010
                hide('warnings', 'running'),
1011
                warn_only=True, 
1012
                host_string = hostip, 
1013
                user = loginname, password = myPass
1014
                ):
1015

    
1016
                if len(sudo('ifconfig eth1 192.168.0.12')) == 0: 
1017
                    res = True
1018
            
1019
        else:
1020
            with settings(
1021
                hide('warnings', 'running'),
1022
                warn_only=True, 
1023
                host_string = hostip, 
1024
                user = loginname, password = myPass
1025
                ):
1026

    
1027
                if len(run('ifconfig eth1 192.168.0.12')) == 0:
1028
                    res = True
1029

    
1030
        self.assertTrue(res)
1031

    
1032

    
1033
    def test_003b_setup_interface_B(self):
1034
        """Setup eth1 for server B"""
1035

    
1036
        log.info("Setting up interface eth1 for server B")
1037

    
1038

    
1039
        server = self.client.get_server_details(self.serverid['B'])
1040
        image = self.client.get_image_details(self.imageid)
1041
        os = image['metadata']['values']['os']
1042
        
1043
        users = image["metadata"]["values"].get("users", None)
1044
        userlist = users.split()
1045

    
1046
        if "root" in userlist:
1047
            loginname = "root"
1048
        elif users == None:
1049
            loginname = self._connect_loginname(os)
1050
        else:
1051
            loginname = choice(userlist)
1052

    
1053
        hostip = self._get_ipv4(server)
1054
        myPass = self.password['B']
1055

    
1056
        log.info("SSH in server B as %s/%s" % (loginname,myPass))
1057

    
1058
        res = False
1059

    
1060
        if loginname != "root":
1061
            with settings(
1062
                hide('warnings', 'running'),
1063
                warn_only=True, 
1064
                host_string = hostip, 
1065
                user = loginname, password = myPass
1066
                ):
1067
                
1068
                if len(sudo('ifconfig eth1 192.168.0.13'))== 0:
1069
                    res = True
1070
            
1071
        else :
1072
            with settings(
1073
                hide('warnings', 'running'),
1074
                warn_only=True, 
1075
                host_string = hostip, 
1076
                user = loginname, password = myPass
1077
                ):
1078

    
1079
                if len(run('ifconfig eth1 192.168.0.13')) == 0:
1080
                    res = True
1081

    
1082

    
1083
        self.assertTrue(res)
1084
            
1085

    
1086

    
1087
    def test_003c_test_connection_exists(self):
1088
        """Ping server B from server A to test if connection exists"""
1089

    
1090
        log.info("Testing if server A is actually connected to server B")
1091

    
1092
        server = self.client.get_server_details(self.serverid['A'])
1093
        image = self.client.get_image_details(self.imageid)
1094
        os = image['metadata']['values']['os']
1095
        hostip = self._get_ipv4(server)
1096

    
1097
        users = image["metadata"]["values"].get("users", None)
1098
        userlist = users.split()
1099

    
1100
        if "root" in userlist:
1101
            loginname = "root"
1102
        elif users == None:
1103
            loginname = self._connect_loginname(os)
1104
        else:
1105
            loginname = choice(userlist)
1106

    
1107
        myPass = self.password['A']
1108

    
1109
        try:
1110
            ssh = paramiko.SSHClient()
1111
            ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
1112
            ssh.connect(hostip, username = loginname, password = myPass)
1113
        except socket.error:
1114
            raise AssertionError
1115

    
1116
        cmd = "if ping -c 2 -w 3 192.168.0.13 >/dev/null; then echo \'True\'; fi;"
1117
        stdin, stdout, stderr = ssh.exec_command(cmd)
1118
        lines = stdout.readlines()
1119

    
1120
        exists = False
1121

    
1122
        if 'True\n' in lines:
1123
            exists = True
1124

    
1125
        self.assertTrue(exists)
1126

    
1127

    
1128
#TODO: Test IPv6 private connectity
1129

    
1130

    
1131
    def test_004_disconnect_from_network(self):
1132
        "Disconnecting server A and B from network"
1133

    
1134
        log.info("Disconnecting servers from private network")
1135

    
1136
        prev_state = self.client.get_network_details(self.networkid)
1137
        prev_conn = len(prev_state['servers']['values'])
1138

    
1139
        self.client.disconnect_server(self.serverid['A'], self.networkid)
1140
        self.client.disconnect_server(self.serverid['B'], self.networkid)
1141

    
1142
        #Insist on deleting until action timeout
1143
        fail_tmout = time.time()+self.action_timeout
1144

    
1145
        while True:
1146
            connected = (self.client.get_network_details(self.networkid))
1147
            connections = connected['servers']['values']
1148
            if (self.serverid['A'] not in connections) and (self.serverid['B'] not in connections):
1149
                conn_exists = False
1150
                break
1151
            elif time.time() > fail_tmout:
1152
                self.assertLess(time.time(), fail_tmout)
1153
            else:
1154
                time.sleep(self.query_interval)
1155

    
1156
        self.assertFalse(conn_exists)
1157

    
1158
    def test_005_destroy_network(self):
1159
        """Test submit delete network request"""
1160

    
1161
        log.info("Submitting delete network request")
1162

    
1163
        self.client.delete_network(self.networkid)        
1164
        networks = self.client.list_networks()
1165

    
1166
        curr_net = []
1167
        for net in networks:
1168
            curr_net.append(net['id'])
1169

    
1170
        self.assertTrue(self.networkid not in curr_net)
1171
        
1172
    def test_006_cleanup_servers(self):
1173
        """Cleanup servers created for this test"""
1174

    
1175
        log.info("Delete servers created for this test")
1176

    
1177
        self.compute.delete_server(self.serverid['A'])
1178
        self.compute.delete_server(self.serverid['B'])
1179

    
1180
        fail_tmout = time.time()+self.action_timeout
1181

    
1182
        #Ensure server gets deleted
1183
        status = dict() 
1184

    
1185
        while True:
1186
            details = self.compute.get_server_details(self.serverid['A'])
1187
            status['A'] = details['status']
1188
            details = self.compute.get_server_details(self.serverid['B'])
1189
            status['B'] = details['status']
1190
            if (status['A'] == 'DELETED') and (status['B'] == 'DELETED'):
1191
                deleted = True
1192
                break
1193
            elif time.time() > fail_tmout: 
1194
                self.assertLess(time.time(), fail_tmout)
1195
            else:
1196
                time.sleep(self.query_interval)
1197
                
1198
        self.assertTrue(deleted)
1199

    
1200

    
1201
class TestRunnerProcess(Process):
1202
    """A distinct process used to execute part of the tests in parallel"""
1203
    def __init__(self, **kw):
1204
        Process.__init__(self, **kw)
1205
        kwargs = kw["kwargs"]
1206
        self.testq = kwargs["testq"]
1207
        self.runner = kwargs["runner"]
1208

    
1209
    def run(self):
1210
        # Make sure this test runner process dies with the parent
1211
        # and is not left behind.
1212
        #
1213
        # WARNING: This uses the prctl(2) call and is
1214
        # Linux-specific.
1215
        prctl.set_pdeathsig(signal.SIGHUP)
1216

    
1217
        while True:
1218
            log.debug("I am process %d, GETting from queue is %s",
1219
                     os.getpid(), self.testq)
1220
            msg = self.testq.get()
1221
            log.debug("Dequeued msg: %s", msg)
1222

    
1223
            if msg == "TEST_RUNNER_TERMINATE":
1224
                raise SystemExit
1225
            elif issubclass(msg, unittest.TestCase):
1226
                # Assemble a TestSuite, and run it
1227
                suite = unittest.TestLoader().loadTestsFromTestCase(msg)
1228
                self.runner.run(suite)
1229
            else:
1230
                raise Exception("Cannot handle msg: %s" % msg)
1231

    
1232

    
1233

    
1234
def _run_cases_in_parallel(cases, fanout=1, runner=None):
1235
    """Run instances of TestCase in parallel, in a number of distinct processes
1236

1237
    The cases iterable specifies the TestCases to be executed in parallel,
1238
    by test runners running in distinct processes.
1239
    The fanout parameter specifies the number of processes to spawn,
1240
    and defaults to 1.
1241
    The runner argument specifies the test runner class to use inside each
1242
    runner process.
1243

1244
    """
1245
    if runner is None:
1246
        runner = unittest.TextTestRunner(verbosity=2, failfast=True)
1247

    
1248
    # testq: The master process enqueues TestCase objects into this queue,
1249
    #        test runner processes pick them up for execution, in parallel.
1250
    testq = Queue()
1251
    runners = []
1252
    for i in xrange(0, fanout):
1253
        kwargs = dict(testq=testq, runner=runner)
1254
        runners.append(TestRunnerProcess(kwargs=kwargs))
1255

    
1256
    log.info("Spawning %d test runner processes", len(runners))
1257
    for p in runners:
1258
        p.start()
1259
    log.debug("Spawned %d test runners, PIDs are %s",
1260
              len(runners), [p.pid for p in runners])
1261

    
1262
    # Enqueue test cases
1263
    map(testq.put, cases)
1264
    map(testq.put, ["TEST_RUNNER_TERMINATE"] * len(runners))
1265

    
1266
    log.debug("Joining %d processes", len(runners))
1267
    for p in runners:
1268
        p.join()
1269
    log.debug("Done joining %d processes", len(runners))
1270

    
1271

    
1272
def _spawn_server_test_case(**kwargs):
1273
    """Construct a new unit test case class from SpawnServerTestCase"""
1274

    
1275
    name = "SpawnServerTestCase_%s" % kwargs["imageid"]
1276
    cls = type(name, (SpawnServerTestCase,), kwargs)
1277

    
1278
    # Patch extra parameters into test names by manipulating method docstrings
1279
    for (mname, m) in \
1280
        inspect.getmembers(cls, lambda x: inspect.ismethod(x)):
1281
            if hasattr(m, __doc__):
1282
                m.__func__.__doc__ = "[%s] %s" % (imagename, m.__doc__)
1283

    
1284
    # Make sure the class can be pickled, by listing it among
1285
    # the attributes of __main__. A PicklingError is raised otherwise.
1286
    setattr(__main__, name, cls)
1287
    return cls 
1288

    
1289
def _spawn_network_test_case(**kwargs):
1290
    """Construct a new unit test case class from NetworkTestCase"""
1291

    
1292
    name = "NetworkTestCase"+TEST_RUN_ID
1293
    cls = type(name, (NetworkTestCase,), kwargs)
1294

    
1295
    # Make sure the class can be pickled, by listing it among
1296
    # the attributes of __main__. A PicklingError is raised otherwise.
1297
    setattr(__main__, name, cls)
1298
    return cls 
1299

    
1300

    
1301
def cleanup_servers(delete_stale=False):
1302

    
1303
    c = ComputeClient(API, TOKEN)
1304

    
1305
    servers = c.list_servers()
1306
    stale = [s for s in servers if s["name"].startswith(SNF_TEST_PREFIX)]
1307

    
1308
    if len(stale) == 0:
1309
        return
1310

    
1311
    print >> sys.stderr, "Found these stale servers from previous runs:"
1312
    print "    " + \
1313
          "\n    ".join(["%d: %s" % (s["id"], s["name"]) for s in stale])
1314

    
1315
    if delete_stale:
1316
        print >> sys.stderr, "Deleting %d stale servers:" % len(stale)
1317
        for server in stale:
1318
            c.delete_server(server["id"])
1319
        print >> sys.stderr, "    ...done"
1320
    else:
1321
        print >> sys.stderr, "Use --delete-stale to delete them."
1322

    
1323

    
1324
def parse_arguments(args):
1325
    from optparse import OptionParser
1326

    
1327
    kw = {}
1328
    kw["usage"] = "%prog [options]"
1329
    kw["description"] = \
1330
        "%prog runs a number of test scenarios on a " \
1331
        "Synnefo deployment."
1332

    
1333
    parser = OptionParser(**kw)
1334
    parser.disable_interspersed_args()
1335
    parser.add_option("--api",
1336
                      action="store", type="string", dest="api",
1337
                      help="The API URI to use to reach the Synnefo API",
1338
                      default=DEFAULT_API)
1339
    parser.add_option("--token",
1340
                      action="store", type="string", dest="token",
1341
                      help="The token to use for authentication to the API")
1342
    parser.add_option("--nofailfast",
1343
                      action="store_true", dest="nofailfast",
1344
                      help="Do not fail immediately if one of the tests " \
1345
                           "fails (EXPERIMENTAL)",
1346
                      default=False)
1347
    parser.add_option("--action-timeout",
1348
                      action="store", type="int", dest="action_timeout",
1349
                      metavar="TIMEOUT",
1350
                      help="Wait SECONDS seconds for a server action to " \
1351
                           "complete, then the test is considered failed",
1352
                      default=100)
1353
    parser.add_option("--build-warning",
1354
                      action="store", type="int", dest="build_warning",
1355
                      metavar="TIMEOUT",
1356
                      help="Warn if TIMEOUT seconds have passed and a " \
1357
                           "build operation is still pending",
1358
                      default=600)
1359
    parser.add_option("--build-fail",
1360
                      action="store", type="int", dest="build_fail",
1361
                      metavar="BUILD_TIMEOUT",
1362
                      help="Fail the test if TIMEOUT seconds have passed " \
1363
                           "and a build operation is still incomplete",
1364
                      default=900)
1365
    parser.add_option("--query-interval",
1366
                      action="store", type="int", dest="query_interval",
1367
                      metavar="INTERVAL",
1368
                      help="Query server status when requests are pending " \
1369
                           "every INTERVAL seconds",
1370
                      default=3)
1371
    parser.add_option("--fanout",
1372
                      action="store", type="int", dest="fanout",
1373
                      metavar="COUNT",
1374
                      help="Spawn up to COUNT child processes to execute " \
1375
                           "in parallel, essentially have up to COUNT " \
1376
                           "server build requests outstanding (EXPERIMENTAL)",
1377
                      default=1)
1378
    parser.add_option("--force-flavor",
1379
                      action="store", type="int", dest="force_flavorid",
1380
                      metavar="FLAVOR ID",
1381
                      help="Force all server creations to use the specified "\
1382
                           "FLAVOR ID instead of a randomly chosen one, " \
1383
                           "useful if disk space is scarce",
1384
                      default=None)
1385
    parser.add_option("--image-id",
1386
                      action="store", type="string", dest="force_imageid",
1387
                      metavar="IMAGE ID",
1388
                      help="Test the specified image id, use 'all' to test " \
1389
                           "all available images (mandatory argument)",
1390
                      default=None)
1391
    parser.add_option("--show-stale",
1392
                      action="store_true", dest="show_stale",
1393
                      help="Show stale servers from previous runs, whose "\
1394
                           "name starts with `%s'" % SNF_TEST_PREFIX,
1395
                      default=False)
1396
    parser.add_option("--delete-stale",
1397
                      action="store_true", dest="delete_stale",
1398
                      help="Delete stale servers from previous runs, whose "\
1399
                           "name starts with `%s'" % SNF_TEST_PREFIX,
1400
                      default=False)
1401
    parser.add_option("--force-personality",
1402
                      action="store", type="string", dest="personality_path",
1403
                      help="Force a personality file injection. File path required. ",
1404
                      default=None)
1405
    
1406

    
1407
    # FIXME: Change the default for build-fanout to 10
1408
    # FIXME: Allow the user to specify a specific set of Images to test
1409

    
1410
    (opts, args) = parser.parse_args(args)
1411

    
1412
    # Verify arguments
1413
    if opts.delete_stale:
1414
        opts.show_stale = True
1415

    
1416
    if not opts.show_stale:
1417
        if not opts.force_imageid:
1418
            print >>sys.stderr, "The --image-id argument is mandatory."
1419
            parser.print_help()
1420
            sys.exit(1)
1421

    
1422
        if opts.force_imageid != 'all':
1423
            try:
1424
                opts.force_imageid = str(opts.force_imageid)
1425
            except ValueError:
1426
                print >>sys.stderr, "Invalid value specified for --image-id." \
1427
                                    "Use a valid id, or `all'."
1428
                sys.exit(1)
1429

    
1430
    return (opts, args)
1431

    
1432

    
1433
def main():
1434
    """Assemble test cases into a test suite, and run it
1435

1436
    IMPORTANT: Tests have dependencies and have to be run in the specified
1437
    order inside a single test case. They communicate through attributes of the
1438
    corresponding TestCase class (shared fixtures). Distinct subclasses of
1439
    TestCase MAY SHARE NO DATA, since they are run in parallel, in distinct
1440
    test runner processes.
1441

1442
    """
1443

    
1444
    (opts, args) = parse_arguments(sys.argv[1:])
1445

    
1446
    global API, TOKEN
1447
    API = opts.api
1448
    TOKEN = opts.token
1449

    
1450
    # Cleanup stale servers from previous runs
1451
    if opts.show_stale:
1452
        cleanup_servers(delete_stale=opts.delete_stale)
1453
        return 0
1454

    
1455
    # Initialize a kamaki instance, get flavors, images
1456

    
1457
    c = ComputeClient(API, TOKEN)
1458

    
1459
    DIMAGES = c.list_images(detail=True)
1460
    DFLAVORS = c.list_flavors(detail=True)
1461

    
1462
    # FIXME: logging, log, LOG PID, TEST_RUN_ID, arguments
1463
    # Run them: FIXME: In parallel, FAILEARLY, catchbreak?
1464
    #unittest.main(verbosity=2, catchbreak=True)
1465

    
1466
    if opts.force_imageid == 'all':
1467
        test_images = DIMAGES
1468
    else:
1469
        test_images = filter(lambda x: x["id"] == opts.force_imageid, DIMAGES)
1470

    
1471
    
1472
    #New folder for log per image
1473
    os.mkdir(TEST_RUN_ID)
1474

    
1475
    for image in test_images:
1476
        imageid = str(image["id"])
1477
        
1478
        if opts.force_flavorid:
1479
            flavorid = opts.force_flavorid
1480
        else:
1481
            flavorid = choice([f["id"] for f in DFLAVORS if f["disk"] >= 20])
1482

    
1483
        imagename = image["name"]
1484
        
1485
        #Personality dictionary for file injection test
1486
        if opts.personality_path != None:
1487
            f = open(opts.personality_path)
1488
            content = b64encode(f.read())
1489
            personality = []
1490
            st = os.stat(opts.personality_path)
1491
            personality.append({
1492
                    'path': '/root/test_inj_file',
1493
                    'owner': 'root',
1494
                    'group': 'root',
1495
                    'mode': 0x7777 & st.st_mode,
1496
                    'contents': content
1497
                    })
1498
        else:
1499
            personality = None
1500

    
1501
        servername = "%s%s for %s" % (SNF_TEST_PREFIX, TEST_RUN_ID, imagename)
1502
        is_windows = imagename.lower().find("windows") >= 0
1503
        
1504

    
1505
        ServerTestCase = _spawn_server_test_case(imageid=imageid, flavorid=flavorid,
1506
                                                 imagename=imagename,
1507
                                                 personality=personality,
1508
                                                 servername=servername,
1509
                                                 is_windows=is_windows,
1510
                                                 action_timeout=opts.action_timeout,
1511
                                                 build_warning=opts.build_warning,
1512
                                                 build_fail=opts.build_fail,
1513
                                                 query_interval=opts.query_interval,
1514
                                                 )
1515

    
1516
    
1517
        NetworkTestCase = _spawn_network_test_case(action_timeout = opts.action_timeout,
1518
                                                   imageid = imageid,
1519
                                                   flavorid = flavorid,
1520
                                                   imagename=imagename,
1521
                                                   query_interval = opts.query_interval,
1522
                                                   )
1523
    
1524

    
1525
        # seq_cases = [UnauthorizedTestCase, ImagesTestCase, FlavorsTestCase, ServersTestCase, ServerTestCase]
1526
        seq_cases = [NetworkTestCase]
1527
        
1528
        # seq_cases = [UnauthorizedTestCase]
1529
        
1530
        #folder for each image 
1531
        image_folder = TEST_RUN_ID + '/' + imageid 
1532
        os.mkdir(image_folder)
1533

    
1534
        for case in seq_cases:
1535
            log_file = image_folder+'/'+'details_'+(case.__name__)+"_"+TEST_RUN_ID+'.log'
1536
            fail_file = image_folder+'/'+'failed_'+(case.__name__)+"_"+TEST_RUN_ID+'.log'
1537
            error_file = image_folder+'/'+'error_'+(case.__name__)+"_"+TEST_RUN_ID+'.log'
1538

    
1539
            f = open(log_file, "w")
1540
            fail = open(fail_file, "w")
1541
            error = open(error_file, "w")
1542

    
1543
            suite = unittest.TestLoader().loadTestsFromTestCase(case)
1544
            runner = unittest.TextTestRunner(f, verbosity=2)
1545
            result = runner.run(suite)
1546
        
1547
            error.write("Testcases errors: \n\n")
1548
            for res in result.errors:
1549
                error.write(str(res[0])+'\n')
1550
                error.write(str(res[0].__doc__) + '\n')
1551
                error.write('\n')
1552

    
1553
            
1554
                fail.write("Testcases failures: \n\n")
1555
                for res in result.failures:
1556
                    fail.write(str(res[0])+'\n')
1557
                    fail.write(str(res[0].__doc__) + '\n')
1558
                    fail.write('\n')
1559
        
1560

    
1561

    
1562
if __name__ == "__main__":
1563
    sys.exit(main())