Statistics
| Branch: | Tag: | Revision:

root / snf-tools / synnefo_tools / burnin.py @ 946da8b6

History | View | Annotate | Download (57 kB)

1
#!/usr/bin/env python
2

    
3
# Copyright 2011 GRNET S.A. All rights reserved.
4
#
5
# Redistribution and use in source and binary forms, with or
6
# without modification, are permitted provided that the following
7
# conditions are met:
8
#
9
#   1. Redistributions of source code must retain the above
10
#      copyright notice, this list of conditions and the following
11
#      disclaimer.
12
#
13
#   2. Redistributions in binary form must reproduce the above
14
#      copyright notice, this list of conditions and the following
15
#      disclaimer in the documentation and/or other materials
16
#      provided with the distribution.
17
#
18
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
19
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
20
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
21
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
22
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
25
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
26
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
28
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29
# POSSIBILITY OF SUCH DAMAGE.
30
#
31
# The views and conclusions contained in the software and
32
# documentation are those of the authors and should not be
33
# interpreted as representing official policies, either expressed
34
# or implied, of GRNET S.A.
35

    
36
"""Perform integration testing on a running Synnefo deployment"""
37

    
38
import __main__
39
import datetime
40
import inspect
41
import logging
42
import os
43
import paramiko
44
import prctl
45
import subprocess
46
import signal
47
import socket
48
import sys
49
import time
50
from base64 import b64encode
51
from IPy import IP
52
from multiprocessing import Process, Queue
53
from random import choice
54

    
55
from kamaki.clients.compute import ComputeClient
56
from kamaki.clients.cyclades import CycladesClient
57
from kamaki.clients.image import ImageClient
58
from kamaki.clients import ClientError
59

    
60
from fabric.api import *
61

    
62
from vncauthproxy.d3des import generate_response as d3des_generate_response
63

    
64
# Use backported unittest functionality if Python < 2.7
65
try:
66
    import unittest2 as unittest
67
except ImportError:
68
    if sys.version_info < (2, 7):
69
        raise Exception("The unittest2 package is required for Python < 2.7")
70
    import unittest
71

    
72

    
73
API = None
74
TOKEN = None
75
DEFAULT_API = "https://cyclades.okeanos.grnet.gr/api/v1.1"
76
DEFAULT_PLANKTON = "https://cyclades.okeanos.grnet.gr/plankton"
77
DEFAULT_PLANKTON_USER = "images@okeanos.grnet.gr"
78

    
79
# A unique id identifying this test run
80
TEST_RUN_ID = datetime.datetime.strftime(datetime.datetime.now(),
81
                                         "%Y%m%d%H%M%S")
82
SNF_TEST_PREFIX = "snf-test-"
83

    
84
# Setup logging (FIXME - verigak)
85
logging.basicConfig(format="%(message)s")
86
log = logging.getLogger("burnin")
87
log.setLevel(logging.INFO)
88

    
89

    
90
class UnauthorizedTestCase(unittest.TestCase):
91
    def test_unauthorized_access(self):
92
        """Test access without a valid token fails"""
93
        log.info("Authentication test")
94

    
95
        falseToken = '12345'
96
        c = ComputeClient(API, falseToken)
97

    
98
        with self.assertRaises(ClientError) as cm:
99
            c.list_servers()
100
            self.assertEqual(cm.exception.status, 401)
101

    
102

    
103
class ImagesTestCase(unittest.TestCase):
104
    """Test image lists for consistency"""
105
    @classmethod
106
    def setUpClass(cls):
107
        """Initialize kamaki, get (detailed) list of images"""
108
        log.info("Getting simple and detailed list of images")
109

    
110
        cls.plankton = ImageClient(PLANKTON, TOKEN)
111
        cls.images = cls.plankton.list_public()
112
        cls.dimages = cls.plankton.list_public(detail=True)
113

    
114
    def test_001_list_images(self):
115
        """Test image list actually returns images"""
116
        self.assertGreater(len(self.images), 0)
117

    
118
    def test_002_list_images_detailed(self):
119
        """Test detailed image list is the same length as list"""
120
        self.assertEqual(len(self.dimages), len(self.images))
121

    
122
    def test_003_same_image_names(self):
123
        """Test detailed and simple image list contain same names"""
124
        names = sorted(map(lambda x: x["name"], self.images))
125
        dnames = sorted(map(lambda x: x["name"], self.dimages))
126
        self.assertEqual(names, dnames)
127

    
128
    def test_004_unique_image_names(self):
129
        """Test system images have unique names"""
130
        sys_images = filter(lambda x: x['owner'] == PLANKTON_USER,
131
                            self.dimages)
132
        names = sorted(map(lambda x: x["name"], sys_images))
133
        self.assertEqual(sorted(list(set(names))), names)
134

    
135
    def test_005_image_metadata(self):
136
        """Test every image has specific metadata defined"""
137
        keys = frozenset(["os", "description", "size"])
138
        for i in self.dimages:
139
            self.assertTrue(keys.issubset(i["metadata"]["values"].keys()))
140

    
141

    
142
class FlavorsTestCase(unittest.TestCase):
143
    """Test flavor lists for consistency"""
144
    @classmethod
145
    def setUpClass(cls):
146
        """Initialize kamaki, get (detailed) list of flavors"""
147
        log.info("Getting simple and detailed list of flavors")
148

    
149
        cls.client = ComputeClient(API, TOKEN)
150
        cls.flavors = cls.client.list_flavors()
151
        cls.dflavors = cls.client.list_flavors(detail=True)
152

    
153
    def test_001_list_flavors(self):
154
        """Test flavor list actually returns flavors"""
155
        self.assertGreater(len(self.flavors), 0)
156

    
157
    def test_002_list_flavors_detailed(self):
158
        """Test detailed flavor list is the same length as list"""
159
        self.assertEquals(len(self.dflavors), len(self.flavors))
160

    
161
    def test_003_same_flavor_names(self):
162
        """Test detailed and simple flavor list contain same names"""
163
        names = sorted(map(lambda x: x["name"], self.flavors))
164
        dnames = sorted(map(lambda x: x["name"], self.dflavors))
165
        self.assertEqual(names, dnames)
166

    
167
    def test_004_unique_flavor_names(self):
168
        """Test flavors have unique names"""
169
        names = sorted(map(lambda x: x["name"], self.flavors))
170
        self.assertEqual(sorted(list(set(names))), names)
171

    
172
    def test_005_well_formed_flavor_names(self):
173
        """Test flavors have names of the form CxxRyyDzz
174

175
        Where xx is vCPU count, yy is RAM in MiB, zz is Disk in GiB
176

177
        """
178
        for f in self.dflavors:
179
            self.assertEqual("C%dR%dD%d" % (f["cpu"], f["ram"], f["disk"]),
180
                             f["name"],
181
                             "Flavor %s does not match its specs." % f["name"])
182

    
183

    
184
class ServersTestCase(unittest.TestCase):
185
    """Test server lists for consistency"""
186
    @classmethod
187
    def setUpClass(cls):
188
        """Initialize kamaki, get (detailed) list of servers"""
189
        log.info("Getting simple and detailed list of servers")
190

    
191
        cls.client = ComputeClient(API, TOKEN)
192
        cls.servers = cls.client.list_servers()
193
        cls.dservers = cls.client.list_servers(detail=True)
194

    
195
    # def test_001_list_servers(self):
196
    #     """Test server list actually returns servers"""
197
    #     self.assertGreater(len(self.servers), 0)
198

    
199
    def test_002_list_servers_detailed(self):
200
        """Test detailed server list is the same length as list"""
201
        self.assertEqual(len(self.dservers), len(self.servers))
202

    
203
    def test_003_same_server_names(self):
204
        """Test detailed and simple flavor list contain same names"""
205
        names = sorted(map(lambda x: x["name"], self.servers))
206
        dnames = sorted(map(lambda x: x["name"], self.dservers))
207
        self.assertEqual(names, dnames)
208

    
209

    
210
# This class gets replicated into actual TestCases dynamically
211
class SpawnServerTestCase(unittest.TestCase):
212
    """Test scenario for server of the specified image"""
213

    
214
    @classmethod
215
    def setUpClass(cls):
216
        """Initialize a kamaki instance"""
217
        log.info("Spawning server for image `%s'", cls.imagename)
218

    
219
        cls.client = ComputeClient(API, TOKEN)
220
        cls.cyclades = CycladesClient(API, TOKEN)
221

    
222
    def _get_ipv4(self, server):
223
        """Get the public IPv4 of a server from the detailed server info"""
224

    
225
        public_addrs = filter(lambda x: x["id"] == "public",
226
                              server["addresses"]["values"])
227
        self.assertEqual(len(public_addrs), 1)
228
        ipv4_addrs = filter(lambda x: x["version"] == 4,
229
                            public_addrs[0]["values"])
230
        self.assertEqual(len(ipv4_addrs), 1)
231
        return ipv4_addrs[0]["addr"]
232

    
233
    def _get_ipv6(self, server):
234
        """Get the public IPv6 of a server from the detailed server info"""
235
        public_addrs = filter(lambda x: x["id"] == "public",
236
                              server["addresses"]["values"])
237
        self.assertEqual(len(public_addrs), 1)
238
        ipv6_addrs = filter(lambda x: x["version"] == 6,
239
                            public_addrs[0]["values"])
240
        self.assertEqual(len(ipv6_addrs), 1)
241
        return ipv6_addrs[0]["addr"]
242

    
243
    def _connect_loginname(self, os):
244
        """Return the login name for connections based on the server OS"""
245
        if os in ("Ubuntu", "Kubuntu", "Fedora"):
246
            return "user"
247
        elif os in ("windows", "windows_alpha1"):
248
            return "Administrator"
249
        else:
250
            return "root"
251

    
252
    def _verify_server_status(self, current_status, new_status):
253
        """Verify a server has switched to a specified status"""
254
        server = self.client.get_server_details(self.serverid)
255
        if server["status"] not in (current_status, new_status):
256
            return None  # Do not raise exception, return so the test fails
257
        self.assertEquals(server["status"], new_status)
258

    
259
    def _get_connected_tcp_socket(self, family, host, port):
260
        """Get a connected socket from the specified family to host:port"""
261
        sock = None
262
        for res in \
263
            socket.getaddrinfo(host, port, family, socket.SOCK_STREAM, 0,
264
                               socket.AI_PASSIVE):
265
            af, socktype, proto, canonname, sa = res
266
            try:
267
                sock = socket.socket(af, socktype, proto)
268
            except socket.error as msg:
269
                sock = None
270
                continue
271
            try:
272
                sock.connect(sa)
273
            except socket.error as msg:
274
                sock.close()
275
                sock = None
276
                continue
277
        self.assertIsNotNone(sock)
278
        return sock
279

    
280
    def _ping_once(self, ipv6, ip):
281
        """Test server responds to a single IPv4 or IPv6 ping"""
282
        cmd = "ping%s -c 2 -w 3 %s" % ("6" if ipv6 else "", ip)
283
        ping = subprocess.Popen(cmd, shell=True,
284
                                stdout=subprocess.PIPE, stderr=subprocess.PIPE)
285
        (stdout, stderr) = ping.communicate()
286
        ret = ping.wait()
287
        self.assertEquals(ret, 0)
288

    
289
    def _get_hostname_over_ssh(self, hostip, username, password):
290
        ssh = paramiko.SSHClient()
291
        ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
292
        try:
293
            ssh.connect(hostip, username=username, password=password)
294
        except socket.error:
295
            raise AssertionError
296
        stdin, stdout, stderr = ssh.exec_command("hostname")
297
        lines = stdout.readlines()
298
        self.assertEqual(len(lines), 1)
299
        return lines[0]
300

    
301
    def _try_until_timeout_expires(self, warn_timeout, fail_timeout,
302
                                   opmsg, callable, *args, **kwargs):
303
        if warn_timeout == fail_timeout:
304
            warn_timeout = fail_timeout + 1
305
        warn_tmout = time.time() + warn_timeout
306
        fail_tmout = time.time() + fail_timeout
307
        while True:
308
            self.assertLess(time.time(), fail_tmout,
309
                            "operation `%s' timed out" % opmsg)
310
            if time.time() > warn_tmout:
311
                log.warning("Server %d: `%s' operation `%s' not done yet",
312
                            self.serverid, self.servername, opmsg)
313
            try:
314
                log.info("%s... " % opmsg)
315
                return callable(*args, **kwargs)
316
            except AssertionError:
317
                pass
318
            time.sleep(self.query_interval)
319

    
320
    def _insist_on_tcp_connection(self, family, host, port):
321
        familystr = {socket.AF_INET: "IPv4", socket.AF_INET6: "IPv6",
322
                     socket.AF_UNSPEC: "Unspecified-IPv4/6"}
323
        msg = "connect over %s to %s:%s" % \
324
              (familystr.get(family, "Unknown"), host, port)
325
        sock = self._try_until_timeout_expires(
326
                self.action_timeout, self.action_timeout,
327
                msg, self._get_connected_tcp_socket,
328
                family, host, port)
329
        return sock
330

    
331
    def _insist_on_status_transition(self, current_status, new_status,
332
                                    fail_timeout, warn_timeout=None):
333
        msg = "Server %d: `%s', waiting for %s -> %s" % \
334
              (self.serverid, self.servername, current_status, new_status)
335
        if warn_timeout is None:
336
            warn_timeout = fail_timeout
337
        self._try_until_timeout_expires(warn_timeout, fail_timeout,
338
                                        msg, self._verify_server_status,
339
                                        current_status, new_status)
340
        # Ensure the status is actually the expected one
341
        server = self.client.get_server_details(self.serverid)
342
        self.assertEquals(server["status"], new_status)
343

    
344
    def _insist_on_ssh_hostname(self, hostip, username, password):
345
        msg = "SSH to %s, as %s/%s" % (hostip, username, password)
346
        hostname = self._try_until_timeout_expires(
347
                self.action_timeout, self.action_timeout,
348
                msg, self._get_hostname_over_ssh,
349
                hostip, username, password)
350

    
351
        # The hostname must be of the form 'prefix-id'
352
        self.assertTrue(hostname.endswith("-%d\n" % self.serverid))
353

    
354
    def _check_file_through_ssh(self, hostip, username, password,
355
                                remotepath, content):
356
        msg = "Trying file injection through SSH to %s, as %s/%s" % \
357
            (hostip, username, password)
358
        log.info(msg)
359
        try:
360
            ssh = paramiko.SSHClient()
361
            ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
362
            ssh.connect(hostip, username=username, password=password)
363
        except socket.error:
364
            raise AssertionError
365

    
366
        transport = paramiko.Transport((hostip, 22))
367
        transport.connect(username=username, password=password)
368

    
369
        localpath = '/tmp/' + SNF_TEST_PREFIX + 'injection'
370
        sftp = paramiko.SFTPClient.from_transport(transport)
371
        sftp.get(remotepath, localpath)
372
        sftp.close()
373
        transport.close()
374

    
375
        f = open(localpath)
376
        remote_content = b64encode(f.read())
377

    
378
        # Check if files are the same
379
        return (remote_content == content)
380

    
381
    def _skipIf(self, condition, msg):
382
        if condition:
383
            self.skipTest(msg)
384

    
385
    def test_001_submit_create_server(self):
386
        """Test submit create server request"""
387

    
388
        log.info("Submit new server request")
389
        server = self.client.create_server(self.servername, self.flavorid,
390
                                           self.imageid, self.personality)
391

    
392
        log.info("Server id: " + str(server["id"]))
393
        log.info("Server password: " + server["adminPass"])
394
        self.assertEqual(server["name"], self.servername)
395
        self.assertEqual(server["flavorRef"], self.flavorid)
396
        self.assertEqual(server["imageRef"], self.imageid)
397
        self.assertEqual(server["status"], "BUILD")
398

    
399
        # Update class attributes to reflect data on building server
400
        cls = type(self)
401
        cls.serverid = server["id"]
402
        cls.username = None
403
        cls.passwd = server["adminPass"]
404

    
405
    def test_002a_server_is_building_in_list(self):
406
        """Test server is in BUILD state, in server list"""
407
        log.info("Server in BUILD state in server list")
408

    
409
        servers = self.client.list_servers(detail=True)
410
        servers = filter(lambda x: x["name"] == self.servername, servers)
411
        self.assertEqual(len(servers), 1)
412
        server = servers[0]
413
        self.assertEqual(server["name"], self.servername)
414
        self.assertEqual(server["flavorRef"], self.flavorid)
415
        self.assertEqual(server["imageRef"], self.imageid)
416
        self.assertEqual(server["status"], "BUILD")
417

    
418
    def test_002b_server_is_building_in_details(self):
419
        """Test server is in BUILD state, in details"""
420

    
421
        log.info("Server in BUILD state in details")
422

    
423
        server = self.client.get_server_details(self.serverid)
424
        self.assertEqual(server["name"], self.servername)
425
        self.assertEqual(server["flavorRef"], self.flavorid)
426
        self.assertEqual(server["imageRef"], self.imageid)
427
        self.assertEqual(server["status"], "BUILD")
428

    
429
    def test_002c_set_server_metadata(self):
430

    
431
        log.info("Creating server metadata")
432

    
433
        image = self.client.get_image_details(self.imageid)
434
        os = image["metadata"]["values"]["os"]
435
        loginname = image["metadata"]["values"].get("users", None)
436
        self.client.update_server_metadata(self.serverid, OS=os)
437

    
438
        userlist = loginname.split()
439

    
440
        # Determine the username to use for future connections
441
        # to this host
442
        cls = type(self)
443

    
444
        if "root" in userlist:
445
            cls.username = "root"
446
        elif users == None:
447
            cls.username = self._connect_loginname(os)
448
        else:
449
            cls.username = choice(userlist)
450

    
451
        self.assertIsNotNone(cls.username)
452

    
453
    def test_002d_verify_server_metadata(self):
454
        """Test server metadata keys are set based on image metadata"""
455

    
456
        log.info("Verifying image metadata")
457

    
458
        servermeta = self.client.get_server_metadata(self.serverid)
459
        imagemeta = self.client.get_image_metadata(self.imageid)
460

    
461
        self.assertEqual(servermeta["OS"], imagemeta["os"])
462

    
463
    def test_003_server_becomes_active(self):
464
        """Test server becomes ACTIVE"""
465

    
466
        log.info("Waiting for server to become ACTIVE")
467

    
468
        self._insist_on_status_transition("BUILD", "ACTIVE",
469
                                         self.build_fail, self.build_warning)
470

    
471
    def test_003a_get_server_oob_console(self):
472
        """Test getting OOB server console over VNC
473

474
        Implementation of RFB protocol follows
475
        http://www.realvnc.com/docs/rfbproto.pdf.
476

477
        """
478
        console = self.cyclades.get_server_console(self.serverid)
479
        self.assertEquals(console['type'], "vnc")
480
        sock = self._insist_on_tcp_connection(socket.AF_INET,
481
                                        console["host"], console["port"])
482

    
483
        # Step 1. ProtocolVersion message (par. 6.1.1)
484
        version = sock.recv(1024)
485
        self.assertEquals(version, 'RFB 003.008\n')
486
        sock.send(version)
487

    
488
        # Step 2. Security (par 6.1.2): Only VNC Authentication supported
489
        sec = sock.recv(1024)
490
        self.assertEquals(list(sec), ['\x01', '\x02'])
491

    
492
        # Step 3. Request VNC Authentication (par 6.1.2)
493
        sock.send('\x02')
494

    
495
        # Step 4. Receive Challenge (par 6.2.2)
496
        challenge = sock.recv(1024)
497
        self.assertEquals(len(challenge), 16)
498

    
499
        # Step 5. DES-Encrypt challenge, use password as key (par 6.2.2)
500
        response = d3des_generate_response(
501
            (console["password"] + '\0' * 8)[:8], challenge)
502
        sock.send(response)
503

    
504
        # Step 6. SecurityResult (par 6.1.3)
505
        result = sock.recv(4)
506
        self.assertEquals(list(result), ['\x00', '\x00', '\x00', '\x00'])
507
        sock.close()
508

    
509
    def test_004_server_has_ipv4(self):
510
        """Test active server has a valid IPv4 address"""
511

    
512
        log.info("Validate server's IPv4")
513

    
514
        server = self.client.get_server_details(self.serverid)
515
        ipv4 = self._get_ipv4(server)
516
        self.assertEquals(IP(ipv4).version(), 4)
517

    
518
    def test_005_server_has_ipv6(self):
519
        """Test active server has a valid IPv6 address"""
520
        self._skipIf(NO_IPV6, "--no-ipv6 flag enabled")
521

    
522
        log.info("Validate server's IPv6")
523

    
524
        server = self.client.get_server_details(self.serverid)
525
        ipv6 = self._get_ipv6(server)
526
        self.assertEquals(IP(ipv6).version(), 6)
527

    
528
    def test_006_server_responds_to_ping_IPv4(self):
529
        """Test server responds to ping on IPv4 address"""
530

    
531
        log.info("Testing if server responds to pings in IPv4")
532

    
533
        server = self.client.get_server_details(self.serverid)
534
        ip = self._get_ipv4(server)
535
        self._try_until_timeout_expires(self.action_timeout,
536
                                        self.action_timeout,
537
                                        "PING IPv4 to %s" % ip,
538
                                        self._ping_once,
539
                                        False, ip)
540

    
541
    def test_007_server_responds_to_ping_IPv6(self):
542
        """Test server responds to ping on IPv6 address"""
543
        self._skipIf(NO_IPV6, "--no-ipv6 flag enabled")
544
        log.info("Testing if server responds to pings in IPv6")
545

    
546
        server = self.client.get_server_details(self.serverid)
547
        ip = self._get_ipv6(server)
548
        self._try_until_timeout_expires(self.action_timeout,
549
                                        self.action_timeout,
550
                                        "PING IPv6 to %s" % ip,
551
                                        self._ping_once,
552
                                        True, ip)
553

    
554
    def test_008_submit_shutdown_request(self):
555
        """Test submit request to shutdown server"""
556

    
557
        log.info("Shutting down server")
558

    
559
        self.cyclades.shutdown_server(self.serverid)
560

    
561
    def test_009_server_becomes_stopped(self):
562
        """Test server becomes STOPPED"""
563

    
564
        log.info("Waiting until server becomes STOPPED")
565
        self._insist_on_status_transition("ACTIVE", "STOPPED",
566
                                         self.action_timeout,
567
                                         self.action_timeout)
568

    
569
    def test_010_submit_start_request(self):
570
        """Test submit start server request"""
571

    
572
        log.info("Starting server")
573

    
574
        self.cyclades.start_server(self.serverid)
575

    
576
    def test_011_server_becomes_active(self):
577
        """Test server becomes ACTIVE again"""
578

    
579
        log.info("Waiting until server becomes ACTIVE")
580
        self._insist_on_status_transition("STOPPED", "ACTIVE",
581
                                         self.action_timeout,
582
                                         self.action_timeout)
583

    
584
    def test_011a_server_responds_to_ping_IPv4(self):
585
        """Test server OS is actually up and running again"""
586

    
587
        log.info("Testing if server is actually up and running")
588

    
589
        self.test_006_server_responds_to_ping_IPv4()
590

    
591
    def test_012_ssh_to_server_IPv4(self):
592
        """Test SSH to server public IPv4 works, verify hostname"""
593

    
594
        self._skipIf(self.is_windows, "only valid for Linux servers")
595
        server = self.client.get_server_details(self.serverid)
596
        self._insist_on_ssh_hostname(self._get_ipv4(server),
597
                                     self.username, self.passwd)
598

    
599
    def test_013_ssh_to_server_IPv6(self):
600
        """Test SSH to server public IPv6 works, verify hostname"""
601
        self._skipIf(self.is_windows, "only valid for Linux servers")
602
        self._skipIf(NO_IPV6, "--no-ipv6 flag enabled")
603

    
604
        server = self.client.get_server_details(self.serverid)
605
        self._insist_on_ssh_hostname(self._get_ipv6(server),
606
                                     self.username, self.passwd)
607

    
608
    def test_014_rdp_to_server_IPv4(self):
609
        "Test RDP connection to server public IPv4 works"""
610
        self._skipIf(not self.is_windows, "only valid for Windows servers")
611
        server = self.client.get_server_details(self.serverid)
612
        ipv4 = self._get_ipv4(server)
613
        sock = _insist_on_tcp_connection(socket.AF_INET, ipv4, 3389)
614

    
615
        # No actual RDP processing done. We assume the RDP server is there
616
        # if the connection to the RDP port is successful.
617
        # FIXME: Use rdesktop, analyze exit code? see manpage [costasd]
618
        sock.close()
619

    
620
    def test_015_rdp_to_server_IPv6(self):
621
        "Test RDP connection to server public IPv6 works"""
622
        self._skipIf(not self.is_windows, "only valid for Windows servers")
623
        self._skipIf(NO_IPV6, "--no-ipv6 flag enabled")
624

    
625
        server = self.client.get_server_details(self.serverid)
626
        ipv6 = self._get_ipv6(server)
627
        sock = _get_tcp_connection(socket.AF_INET6, ipv6, 3389)
628

    
629
        # No actual RDP processing done. We assume the RDP server is there
630
        # if the connection to the RDP port is successful.
631
        sock.close()
632

    
633
    def test_016_personality_is_enforced(self):
634
        """Test file injection for personality enforcement"""
635
        self._skipIf(self.is_windows, "only implemented for Linux servers")
636
        self._skipIf(self.personality == None, "No personality file selected")
637

    
638
        log.info("Trying to inject file for personality enforcement")
639

    
640
        server = self.client.get_server_details(self.serverid)
641

    
642
        for inj_file in self.personality:
643
            equal_files = self._check_file_through_ssh(self._get_ipv4(server),
644
                                                       inj_file['owner'],
645
                                                       self.passwd,
646
                                                       inj_file['path'],
647
                                                       inj_file['contents'])
648
            self.assertTrue(equal_files)
649

    
650
    def test_017_submit_delete_request(self):
651
        """Test submit request to delete server"""
652

    
653
        log.info("Deleting server")
654

    
655
        self.client.delete_server(self.serverid)
656

    
657
    def test_018_server_becomes_deleted(self):
658
        """Test server becomes DELETED"""
659

    
660
        log.info("Testing if server becomes DELETED")
661

    
662
        self._insist_on_status_transition("ACTIVE", "DELETED",
663
                                         self.action_timeout,
664
                                         self.action_timeout)
665

    
666
    def test_019_server_no_longer_in_server_list(self):
667
        """Test server is no longer in server list"""
668

    
669
        log.info("Test if server is no longer listed")
670

    
671
        servers = self.client.list_servers()
672
        self.assertNotIn(self.serverid, [s["id"] for s in servers])
673

    
674

    
675
class NetworkTestCase(unittest.TestCase):
676
    """ Testing networking in cyclades """
677

    
678
    @classmethod
679
    def setUpClass(cls):
680
        "Initialize kamaki, get list of current networks"
681

    
682
        cls.client = CycladesClient(API, TOKEN)
683
        cls.compute = ComputeClient(API, TOKEN)
684

    
685
        cls.servername = "%s%s for %s" % (SNF_TEST_PREFIX,
686
                                          TEST_RUN_ID,
687
                                          cls.imagename)
688

    
689
        #Dictionary initialization for the vms credentials
690
        cls.serverid = dict()
691
        cls.username = dict()
692
        cls.password = dict()
693

    
694
    def _get_ipv4(self, server):
695
        """Get the public IPv4 of a server from the detailed server info"""
696

    
697
        public_addrs = filter(lambda x: x["id"] == "public",
698
                              server["addresses"]["values"])
699
        self.assertEqual(len(public_addrs), 1)
700
        ipv4_addrs = filter(lambda x: x["version"] == 4,
701
                            public_addrs[0]["values"])
702
        self.assertEqual(len(ipv4_addrs), 1)
703
        return ipv4_addrs[0]["addr"]
704

    
705
    def _connect_loginname(self, os):
706
        """Return the login name for connections based on the server OS"""
707
        if os in ("Ubuntu", "Kubuntu", "Fedora"):
708
            return "user"
709
        elif os in ("windows", "windows_alpha1"):
710
            return "Administrator"
711
        else:
712
            return "root"
713

    
714
    def _ping_once(self, ip):
715

    
716
        """Test server responds to a single IPv4 or IPv6 ping"""
717
        cmd = "ping -c 2 -w 3 %s" % (ip)
718
        ping = subprocess.Popen(cmd, shell=True,
719
                                stdout=subprocess.PIPE, stderr=subprocess.PIPE)
720
        (stdout, stderr) = ping.communicate()
721
        ret = ping.wait()
722

    
723
        return (ret == 0)
724

    
725
    def test_00001a_submit_create_server_A(self):
726
        """Test submit create server request"""
727

    
728
        log.info("Creating test server A")
729

    
730
        serverA = self.client.create_server(self.servername, self.flavorid,
731
                                            self.imageid, personality=None)
732

    
733
        self.assertEqual(serverA["name"], self.servername)
734
        self.assertEqual(serverA["flavorRef"], self.flavorid)
735
        self.assertEqual(serverA["imageRef"], self.imageid)
736
        self.assertEqual(serverA["status"], "BUILD")
737

    
738
        # Update class attributes to reflect data on building server
739
        self.serverid['A'] = serverA["id"]
740
        self.username['A'] = None
741
        self.password['A'] = serverA["adminPass"]
742

    
743
        log.info("Server A id:" + str(serverA["id"]))
744
        log.info("Server password " + (self.password['A']))
745

    
746
    def test_00001b_serverA_becomes_active(self):
747
        """Test server becomes ACTIVE"""
748

    
749
        log.info("Waiting until test server A becomes ACTIVE")
750

    
751
        fail_tmout = time.time() + self.action_timeout
752
        while True:
753
            d = self.client.get_server_details(self.serverid['A'])
754
            status = d['status']
755
            if status == 'ACTIVE':
756
                active = True
757
                break
758
            elif time.time() > fail_tmout:
759
                self.assertLess(time.time(), fail_tmout)
760
            else:
761
                time.sleep(self.query_interval)
762

    
763
        self.assertTrue(active)
764

    
765
    def test_00002a_submit_create_server_B(self):
766
        """Test submit create server request"""
767

    
768
        log.info("Creating test server B")
769

    
770
        serverB = self.client.create_server(self.servername, self.flavorid,
771
                                            self.imageid, personality=None)
772

    
773
        self.assertEqual(serverB["name"], self.servername)
774
        self.assertEqual(serverB["flavorRef"], self.flavorid)
775
        self.assertEqual(serverB["imageRef"], self.imageid)
776
        self.assertEqual(serverB["status"], "BUILD")
777

    
778
        # Update class attributes to reflect data on building server
779
        self.serverid['B'] = serverB["id"]
780
        self.username['B'] = None
781
        self.password['B'] = serverB["adminPass"]
782

    
783
        log.info("Server B id: " + str(serverB["id"]))
784
        log.info("Password " + (self.password['B']))
785

    
786
    def test_00002b_serverB_becomes_active(self):
787
        """Test server becomes ACTIVE"""
788

    
789
        log.info("Waiting until test server B becomes ACTIVE")
790

    
791
        fail_tmout = time.time() + self.action_timeout
792
        while True:
793
            d = self.client.get_server_details(self.serverid['B'])
794
            status = d['status']
795
            if status == 'ACTIVE':
796
                active = True
797
                break
798
            elif time.time() > fail_tmout:
799
                self.assertLess(time.time(), fail_tmout)
800
            else:
801
                time.sleep(self.query_interval)
802

    
803
        self.assertTrue(active)
804

    
805
    def test_001_create_network(self):
806
        """Test submit create network request"""
807

    
808
        log.info("Submit new network request")
809

    
810
        name = SNF_TEST_PREFIX + TEST_RUN_ID
811
        previous_num = len(self.client.list_networks())
812
        network = self.client.create_network(name)
813

    
814
        #Test if right name is assigned
815
        self.assertEqual(network['name'], name)
816

    
817
        # Update class attributes
818
        cls = type(self)
819
        cls.networkid = network['id']
820
        networks = self.client.list_networks()
821

    
822
        #Test if new network is created
823
        self.assertTrue(len(networks) > previous_num)
824

    
825
    def test_002_connect_to_network(self):
826
        """Test connect VMs to network"""
827

    
828
        log.info("Connect VMs to private network")
829

    
830
        self.client.connect_server(self.serverid['A'], self.networkid)
831
        self.client.connect_server(self.serverid['B'], self.networkid)
832

    
833
        #Insist on connecting until action timeout
834
        fail_tmout = time.time() + self.action_timeout
835

    
836
        while True:
837
            connected = (self.client.get_network_details(self.networkid))
838
            connections = connected['servers']['values']
839
            if (self.serverid['A'] in connections) \
840
                    and (self.serverid['B'] in connections):
841
                conn_exists = True
842
                break
843
            elif time.time() > fail_tmout:
844
                self.assertLess(time.time(), fail_tmout)
845
            else:
846
                time.sleep(self.query_interval)
847

    
848
        self.assertTrue(conn_exists)
849

    
850
    def test_002a_reboot(self):
851
        """Rebooting server A"""
852

    
853
        log.info("Rebooting server A")
854

    
855
        self.client.shutdown_server(self.serverid['A'])
856

    
857
        fail_tmout = time.time() + self.action_timeout
858
        while True:
859
            d = self.client.get_server_details(self.serverid['A'])
860
            status = d['status']
861
            if status == 'STOPPED':
862
                break
863
            elif time.time() > fail_tmout:
864
                self.assertLess(time.time(), fail_tmout)
865
            else:
866
                time.sleep(self.query_interval)
867

    
868
        self.client.start_server(self.serverid['A'])
869

    
870
        while True:
871
            d = self.client.get_server_details(self.serverid['A'])
872
            status = d['status']
873
            if status == 'ACTIVE':
874
                active = True
875
                break
876
            elif time.time() > fail_tmout:
877
                self.assertLess(time.time(), fail_tmout)
878
            else:
879
                time.sleep(self.query_interval)
880

    
881
        self.assertTrue(active)
882

    
883
    def test_002b_ping_server_A(self):
884
        "Test if server A is pingable"
885

    
886
        log.info("Testing if server A is pingable")
887

    
888
        server = self.client.get_server_details(self.serverid['A'])
889
        ip = self._get_ipv4(server)
890

    
891
        fail_tmout = time.time() + self.action_timeout
892

    
893
        s = False
894

    
895
        while True:
896

    
897
            if self._ping_once(ip):
898
                s = True
899
                break
900

    
901
            elif time.time() > fail_tmout:
902
                self.assertLess(time.time(), fail_tmout)
903

    
904
            else:
905
                time.sleep(self.query_interval)
906

    
907
        self.assertTrue(s)
908

    
909
    def test_002c_reboot(self):
910
        """Reboot server B"""
911

    
912
        log.info("Rebooting server B")
913

    
914
        self.client.shutdown_server(self.serverid['B'])
915

    
916
        fail_tmout = time.time() + self.action_timeout
917
        while True:
918
            d = self.client.get_server_details(self.serverid['B'])
919
            status = d['status']
920
            if status == 'STOPPED':
921
                break
922
            elif time.time() > fail_tmout:
923
                self.assertLess(time.time(), fail_tmout)
924
            else:
925
                time.sleep(self.query_interval)
926

    
927
        self.client.start_server(self.serverid['B'])
928

    
929
        while True:
930
            d = self.client.get_server_details(self.serverid['B'])
931
            status = d['status']
932
            if status == 'ACTIVE':
933
                active = True
934
                break
935
            elif time.time() > fail_tmout:
936
                self.assertLess(time.time(), fail_tmout)
937
            else:
938
                time.sleep(self.query_interval)
939

    
940
        self.assertTrue(active)
941

    
942
    def test_002d_ping_server_B(self):
943
        """Test if server B is pingable"""
944

    
945
        log.info("Testing if server B is pingable")
946
        server = self.client.get_server_details(self.serverid['B'])
947
        ip = self._get_ipv4(server)
948

    
949
        fail_tmout = time.time() + self.action_timeout
950

    
951
        s = False
952

    
953
        while True:
954
            if self._ping_once(ip):
955
                s = True
956
                break
957

    
958
            elif time.time() > fail_tmout:
959
                self.assertLess(time.time(), fail_tmout)
960

    
961
            else:
962
                time.sleep(self.query_interval)
963

    
964
        self.assertTrue(s)
965

    
966
    def test_003a_setup_interface_A(self):
967
        """Set up eth1 for server A"""
968

    
969
        log.info("Setting up interface eth1 for server A")
970

    
971
        server = self.client.get_server_details(self.serverid['A'])
972
        image = self.client.get_image_details(self.imageid)
973
        os = image['metadata']['values']['os']
974

    
975
        users = image["metadata"]["values"].get("users", None)
976
        userlist = users.split()
977

    
978
        if "root" in userlist:
979
            loginname = "root"
980
        elif users == None:
981
            loginname = self._connect_loginname(os)
982
        else:
983
            loginname = choice(userlist)
984

    
985
        hostip = self._get_ipv4(server)
986
        myPass = self.password['A']
987

    
988
        log.info("SSH in server A as %s/%s" % (loginname, myPass))
989

    
990
        res = False
991

    
992
        if loginname != "root":
993
            with settings(
994
                hide('warnings', 'running'),
995
                warn_only=True,
996
                host_string=hostip,
997
                user=loginname, password=myPass
998
                ):
999

    
1000
                if len(sudo('ifconfig eth1 192.168.0.12')) == 0:
1001
                    res = True
1002

    
1003
        else:
1004
            with settings(
1005
                hide('warnings', 'running'),
1006
                warn_only=True,
1007
                host_string=hostip,
1008
                user=loginname, password=myPass
1009
                ):
1010

    
1011
                if len(run('ifconfig eth1 192.168.0.12')) == 0:
1012
                    res = True
1013

    
1014
        self.assertTrue(res)
1015

    
1016
    def test_003b_setup_interface_B(self):
1017
        """Setup eth1 for server B"""
1018

    
1019
        log.info("Setting up interface eth1 for server B")
1020

    
1021
        server = self.client.get_server_details(self.serverid['B'])
1022
        image = self.client.get_image_details(self.imageid)
1023
        os = image['metadata']['values']['os']
1024

    
1025
        users = image["metadata"]["values"].get("users", None)
1026
        userlist = users.split()
1027

    
1028
        if "root" in userlist:
1029
            loginname = "root"
1030
        elif users == None:
1031
            loginname = self._connect_loginname(os)
1032
        else:
1033
            loginname = choice(userlist)
1034

    
1035
        hostip = self._get_ipv4(server)
1036
        myPass = self.password['B']
1037

    
1038
        log.info("SSH in server B as %s/%s" % (loginname, myPass))
1039

    
1040
        res = False
1041

    
1042
        if loginname != "root":
1043
            with settings(
1044
                hide('warnings', 'running'),
1045
                warn_only=True,
1046
                host_string=hostip,
1047
                user=loginname, password=myPass
1048
                ):
1049

    
1050
                if len(sudo('ifconfig eth1 192.168.0.13')) == 0:
1051
                    res = True
1052

    
1053
        else:
1054
            with settings(
1055
                hide('warnings', 'running'),
1056
                warn_only=True,
1057
                host_string=hostip,
1058
                user=loginname, password=myPass
1059
                ):
1060

    
1061
                if len(run('ifconfig eth1 192.168.0.13')) == 0:
1062
                    res = True
1063

    
1064
        self.assertTrue(res)
1065

    
1066
    def test_003c_test_connection_exists(self):
1067
        """Ping server B from server A to test if connection exists"""
1068

    
1069
        log.info("Testing if server A is actually connected to server B")
1070

    
1071
        server = self.client.get_server_details(self.serverid['A'])
1072
        image = self.client.get_image_details(self.imageid)
1073
        os = image['metadata']['values']['os']
1074
        hostip = self._get_ipv4(server)
1075

    
1076
        users = image["metadata"]["values"].get("users", None)
1077
        userlist = users.split()
1078

    
1079
        if "root" in userlist:
1080
            loginname = "root"
1081
        elif users == None:
1082
            loginname = self._connect_loginname(os)
1083
        else:
1084
            loginname = choice(userlist)
1085

    
1086
        myPass = self.password['A']
1087

    
1088
        try:
1089
            ssh = paramiko.SSHClient()
1090
            ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
1091
            ssh.connect(hostip, username=loginname, password=myPass)
1092
        except socket.error:
1093
            raise AssertionError
1094

    
1095
        cmd = "if ping -c 2 -w 3 192.168.0.13 >/dev/null; \
1096
               then echo \'True\'; fi;"
1097
        stdin, stdout, stderr = ssh.exec_command(cmd)
1098
        lines = stdout.readlines()
1099

    
1100
        exists = False
1101

    
1102
        if 'True\n' in lines:
1103
            exists = True
1104

    
1105
        self.assertTrue(exists)
1106

    
1107
#TODO: Test IPv6 private connectity
1108

    
1109
    def test_004_disconnect_from_network(self):
1110
        "Disconnecting server A and B from network"
1111

    
1112
        log.info("Disconnecting servers from private network")
1113

    
1114
        prev_state = self.client.get_network_details(self.networkid)
1115
        prev_conn = len(prev_state['servers']['values'])
1116

    
1117
        self.client.disconnect_server(self.serverid['A'], self.networkid)
1118
        self.client.disconnect_server(self.serverid['B'], self.networkid)
1119

    
1120
        #Insist on deleting until action timeout
1121
        fail_tmout = time.time() + self.action_timeout
1122

    
1123
        while True:
1124
            connected = (self.client.get_network_details(self.networkid))
1125
            connections = connected['servers']['values']
1126
            if ((self.serverid['A'] not in connections) and
1127
                (self.serverid['B'] not in connections)):
1128
                conn_exists = False
1129
                break
1130
            elif time.time() > fail_tmout:
1131
                self.assertLess(time.time(), fail_tmout)
1132
            else:
1133
                time.sleep(self.query_interval)
1134

    
1135
        self.assertFalse(conn_exists)
1136

    
1137
    def test_005_destroy_network(self):
1138
        """Test submit delete network request"""
1139

    
1140
        log.info("Submitting delete network request")
1141

    
1142
        self.client.delete_network(self.networkid)
1143
        networks = self.client.list_networks()
1144

    
1145
        curr_net = []
1146
        for net in networks:
1147
            curr_net.append(net['id'])
1148

    
1149
        self.assertTrue(self.networkid not in curr_net)
1150

    
1151
    def test_006_cleanup_servers(self):
1152
        """Cleanup servers created for this test"""
1153

    
1154
        log.info("Delete servers created for this test")
1155

    
1156
        self.compute.delete_server(self.serverid['A'])
1157
        self.compute.delete_server(self.serverid['B'])
1158

    
1159
        fail_tmout = time.time() + self.action_timeout
1160

    
1161
        #Ensure server gets deleted
1162
        status = dict()
1163

    
1164
        while True:
1165
            details = self.compute.get_server_details(self.serverid['A'])
1166
            status['A'] = details['status']
1167
            details = self.compute.get_server_details(self.serverid['B'])
1168
            status['B'] = details['status']
1169
            if (status['A'] == 'DELETED') and (status['B'] == 'DELETED'):
1170
                deleted = True
1171
                break
1172
            elif time.time() > fail_tmout:
1173
                self.assertLess(time.time(), fail_tmout)
1174
            else:
1175
                time.sleep(self.query_interval)
1176

    
1177
        self.assertTrue(deleted)
1178

    
1179

    
1180
class TestRunnerProcess(Process):
1181
    """A distinct process used to execute part of the tests in parallel"""
1182
    def __init__(self, **kw):
1183
        Process.__init__(self, **kw)
1184
        kwargs = kw["kwargs"]
1185
        self.testq = kwargs["testq"]
1186
        self.runner = kwargs["runner"]
1187

    
1188
    def run(self):
1189
        # Make sure this test runner process dies with the parent
1190
        # and is not left behind.
1191
        #
1192
        # WARNING: This uses the prctl(2) call and is
1193
        # Linux-specific.
1194
        prctl.set_pdeathsig(signal.SIGHUP)
1195

    
1196
        while True:
1197
            log.debug("I am process %d, GETting from queue is %s",
1198
                     os.getpid(), self.testq)
1199
            msg = self.testq.get()
1200
            log.debug("Dequeued msg: %s", msg)
1201

    
1202
            if msg == "TEST_RUNNER_TERMINATE":
1203
                raise SystemExit
1204
            elif issubclass(msg, unittest.TestCase):
1205
                # Assemble a TestSuite, and run it
1206
                suite = unittest.TestLoader().loadTestsFromTestCase(msg)
1207
                self.runner.run(suite)
1208
            else:
1209
                raise Exception("Cannot handle msg: %s" % msg)
1210

    
1211

    
1212
def _run_cases_in_parallel(cases, fanout=1, runner=None):
1213
    """Run instances of TestCase in parallel, in a number of distinct processes
1214

1215
    The cases iterable specifies the TestCases to be executed in parallel,
1216
    by test runners running in distinct processes.
1217
    The fanout parameter specifies the number of processes to spawn,
1218
    and defaults to 1.
1219
    The runner argument specifies the test runner class to use inside each
1220
    runner process.
1221

1222
    """
1223
    if runner is None:
1224
        runner = unittest.TextTestRunner(verbosity=2, failfast=True)
1225

    
1226
    # testq: The master process enqueues TestCase objects into this queue,
1227
    #        test runner processes pick them up for execution, in parallel.
1228
    testq = Queue()
1229
    runners = []
1230
    for i in xrange(0, fanout):
1231
        kwargs = dict(testq=testq, runner=runner)
1232
        runners.append(TestRunnerProcess(kwargs=kwargs))
1233

    
1234
    log.info("Spawning %d test runner processes", len(runners))
1235
    for p in runners:
1236
        p.start()
1237
    log.debug("Spawned %d test runners, PIDs are %s",
1238
              len(runners), [p.pid for p in runners])
1239

    
1240
    # Enqueue test cases
1241
    map(testq.put, cases)
1242
    map(testq.put, ["TEST_RUNNER_TERMINATE"] * len(runners))
1243

    
1244
    log.debug("Joining %d processes", len(runners))
1245
    for p in runners:
1246
        p.join()
1247
    log.debug("Done joining %d processes", len(runners))
1248

    
1249

    
1250
def _spawn_server_test_case(**kwargs):
1251
    """Construct a new unit test case class from SpawnServerTestCase"""
1252

    
1253
    name = "SpawnServerTestCase_%s" % kwargs["imageid"]
1254
    cls = type(name, (SpawnServerTestCase,), kwargs)
1255

    
1256
    # Patch extra parameters into test names by manipulating method docstrings
1257
    for (mname, m) in \
1258
        inspect.getmembers(cls, lambda x: inspect.ismethod(x)):
1259
        if hasattr(m, __doc__):
1260
            m.__func__.__doc__ = "[%s] %s" % (imagename, m.__doc__)
1261

    
1262
    # Make sure the class can be pickled, by listing it among
1263
    # the attributes of __main__. A PicklingError is raised otherwise.
1264
    setattr(__main__, name, cls)
1265
    return cls
1266

    
1267

    
1268
def _spawn_network_test_case(**kwargs):
1269
    """Construct a new unit test case class from NetworkTestCase"""
1270

    
1271
    name = "NetworkTestCase" + TEST_RUN_ID
1272
    cls = type(name, (NetworkTestCase,), kwargs)
1273

    
1274
    # Make sure the class can be pickled, by listing it among
1275
    # the attributes of __main__. A PicklingError is raised otherwise.
1276
    setattr(__main__, name, cls)
1277
    return cls
1278

    
1279

    
1280
def cleanup_servers(delete_stale=False):
1281

    
1282
    c = ComputeClient(API, TOKEN)
1283

    
1284
    servers = c.list_servers()
1285
    stale = [s for s in servers if s["name"].startswith(SNF_TEST_PREFIX)]
1286

    
1287
    if len(stale) == 0:
1288
        return
1289

    
1290
    print >> sys.stderr, "Found these stale servers from previous runs:"
1291
    print "    " + \
1292
          "\n    ".join(["%d: %s" % (s["id"], s["name"]) for s in stale])
1293

    
1294
    if delete_stale:
1295
        print >> sys.stderr, "Deleting %d stale servers:" % len(stale)
1296
        for server in stale:
1297
            c.delete_server(server["id"])
1298
        print >> sys.stderr, "    ...done"
1299
    else:
1300
        print >> sys.stderr, "Use --delete-stale to delete them."
1301

    
1302

    
1303
def cleanup_networks(delete_stale=False):
1304

    
1305
    c = CycladesClient(API, TOKEN)
1306

    
1307
    networks = c.list_networks()
1308
    stale = [n for n in networks if n["name"].startswith(SNF_TEST_PREFIX)]
1309

    
1310
    if len(stale) == 0:
1311
        return
1312

    
1313
    print >> sys.stderr, "Found these stale networks from previous runs:"
1314
    print "    " + \
1315
          "\n    ".join(["%s: %s" % (str(n["id"]), n["name"]) for n in stale])
1316

    
1317
    if delete_stale:
1318
        print >> sys.stderr, "Deleting %d stale networks:" % len(stale)
1319
        for network in stale:
1320
            c.delete_network(network["id"])
1321
        print >> sys.stderr, "    ...done"
1322
    else:
1323
        print >> sys.stderr, "Use --delete-stale to delete them."
1324

    
1325

    
1326
def parse_arguments(args):
1327
    from optparse import OptionParser
1328

    
1329
    kw = {}
1330
    kw["usage"] = "%prog [options]"
1331
    kw["description"] = \
1332
        "%prog runs a number of test scenarios on a " \
1333
        "Synnefo deployment."
1334

    
1335
    parser = OptionParser(**kw)
1336
    parser.disable_interspersed_args()
1337
    parser.add_option("--api",
1338
                      action="store", type="string", dest="api",
1339
                      help="The API URI to use to reach the Synnefo API",
1340
                      default=DEFAULT_API)
1341
    parser.add_option("--plankton",
1342
                      action="store", type="string", dest="plankton",
1343
                      help="The API URI to use to reach the Plankton API",
1344
                      default=DEFAULT_PLANKTON)
1345
    parser.add_option("--plankton-user",
1346
                      action="store", type="string", dest="plankton_user",
1347
                      help="Owner of system images",
1348
                      default=DEFAULT_PLANKTON_USER)
1349
    parser.add_option("--token",
1350
                      action="store", type="string", dest="token",
1351
                      help="The token to use for authentication to the API")
1352
    parser.add_option("--nofailfast",
1353
                      action="store_true", dest="nofailfast",
1354
                      help="Do not fail immediately if one of the tests " \
1355
                           "fails (EXPERIMENTAL)",
1356
                      default=False)
1357
    parser.add_option("--no-ipv6",
1358
                      action="store_true", dest="no_ipv6",
1359
                      help="Disables ipv6 related tests",
1360
                      default=False)
1361
    parser.add_option("--action-timeout",
1362
                      action="store", type="int", dest="action_timeout",
1363
                      metavar="TIMEOUT",
1364
                      help="Wait SECONDS seconds for a server action to " \
1365
                           "complete, then the test is considered failed",
1366
                      default=100)
1367
    parser.add_option("--build-warning",
1368
                      action="store", type="int", dest="build_warning",
1369
                      metavar="TIMEOUT",
1370
                      help="Warn if TIMEOUT seconds have passed and a " \
1371
                           "build operation is still pending",
1372
                      default=600)
1373
    parser.add_option("--build-fail",
1374
                      action="store", type="int", dest="build_fail",
1375
                      metavar="BUILD_TIMEOUT",
1376
                      help="Fail the test if TIMEOUT seconds have passed " \
1377
                           "and a build operation is still incomplete",
1378
                      default=900)
1379
    parser.add_option("--query-interval",
1380
                      action="store", type="int", dest="query_interval",
1381
                      metavar="INTERVAL",
1382
                      help="Query server status when requests are pending " \
1383
                           "every INTERVAL seconds",
1384
                      default=3)
1385
    parser.add_option("--fanout",
1386
                      action="store", type="int", dest="fanout",
1387
                      metavar="COUNT",
1388
                      help="Spawn up to COUNT child processes to execute " \
1389
                           "in parallel, essentially have up to COUNT " \
1390
                           "server build requests outstanding (EXPERIMENTAL)",
1391
                      default=1)
1392
    parser.add_option("--force-flavor",
1393
                      action="store", type="int", dest="force_flavorid",
1394
                      metavar="FLAVOR ID",
1395
                      help="Force all server creations to use the specified "\
1396
                           "FLAVOR ID instead of a randomly chosen one, " \
1397
                           "useful if disk space is scarce",
1398
                      default=None)
1399
    parser.add_option("--image-id",
1400
                      action="store", type="string", dest="force_imageid",
1401
                      metavar="IMAGE ID",
1402
                      help="Test the specified image id, use 'all' to test " \
1403
                           "all available images (mandatory argument)",
1404
                      default=None)
1405
    parser.add_option("--show-stale",
1406
                      action="store_true", dest="show_stale",
1407
                      help="Show stale servers from previous runs, whose "\
1408
                           "name starts with `%s'" % SNF_TEST_PREFIX,
1409
                      default=False)
1410
    parser.add_option("--delete-stale",
1411
                      action="store_true", dest="delete_stale",
1412
                      help="Delete stale servers from previous runs, whose "\
1413
                           "name starts with `%s'" % SNF_TEST_PREFIX,
1414
                      default=False)
1415
    parser.add_option("--force-personality",
1416
                      action="store", type="string", dest="personality_path",
1417
                      help="Force a personality file injection.\
1418
                            File path required. ",
1419
                      default=None)
1420
    parser.add_option("--log-folder",
1421
                      action="store", type="string", dest="log_folder",
1422
                      help="Define the absolute path where the output \
1423
                            log is stored. ",
1424
                      default="/var/log/burnin/")
1425

    
1426
    # FIXME: Change the default for build-fanout to 10
1427
    # FIXME: Allow the user to specify a specific set of Images to test
1428

    
1429
    (opts, args) = parser.parse_args(args)
1430

    
1431
    # Verify arguments
1432
    if opts.delete_stale:
1433
        opts.show_stale = True
1434

    
1435
    if not opts.show_stale:
1436
        if not opts.force_imageid:
1437
            print >>sys.stderr, "The --image-id argument is mandatory."
1438
            parser.print_help()
1439
            sys.exit(1)
1440

    
1441
        if opts.force_imageid != 'all':
1442
            try:
1443
                opts.force_imageid = str(opts.force_imageid)
1444
            except ValueError:
1445
                print >>sys.stderr, "Invalid value specified for --image-id." \
1446
                                    "Use a valid id, or `all'."
1447
                sys.exit(1)
1448

    
1449
    return (opts, args)
1450

    
1451

    
1452
def main():
1453
    """Assemble test cases into a test suite, and run it
1454

1455
    IMPORTANT: Tests have dependencies and have to be run in the specified
1456
    order inside a single test case. They communicate through attributes of the
1457
    corresponding TestCase class (shared fixtures). Distinct subclasses of
1458
    TestCase MAY SHARE NO DATA, since they are run in parallel, in distinct
1459
    test runner processes.
1460

1461
    """
1462

    
1463
    (opts, args) = parse_arguments(sys.argv[1:])
1464

    
1465
    global API, TOKEN, PLANKTON, PLANKTON_USER, NO_IPV6
1466
    API = opts.api
1467
    TOKEN = opts.token
1468
    PLANKTON = opts.plankton
1469
    PLANKTON_USER = opts.plankton_user
1470
    NO_IPV6 = opts.no_ipv6
1471

    
1472
    # Cleanup stale servers from previous runs
1473
    if opts.show_stale:
1474
        cleanup_servers(delete_stale=opts.delete_stale)
1475
        cleanup_networks(delete_stale=opts.delete_stale)
1476
        return 0
1477

    
1478
    # Initialize a kamaki instance, get flavors, images
1479

    
1480
    c = ComputeClient(API, TOKEN)
1481

    
1482
    DIMAGES = c.list_images(detail=True)
1483
    DFLAVORS = c.list_flavors(detail=True)
1484

    
1485
    # FIXME: logging, log, LOG PID, TEST_RUN_ID, arguments
1486
    # Run them: FIXME: In parallel, FAILEARLY, catchbreak?
1487
    #unittest.main(verbosity=2, catchbreak=True)
1488

    
1489
    if opts.force_imageid == 'all':
1490
        test_images = DIMAGES
1491
    else:
1492
        test_images = filter(lambda x: x["id"] == opts.force_imageid, DIMAGES)
1493

    
1494
    #New folder for log per image
1495

    
1496
    if not os.path.exists(opts.log_folder):
1497
        os.mkdir(opts.log_folder)
1498

    
1499
    test_folder = os.path.join(opts.log_folder, TEST_RUN_ID)
1500
    os.mkdir(test_folder)
1501

    
1502
    for image in test_images:
1503

    
1504
        imageid = str(image["id"])
1505

    
1506
        if opts.force_flavorid:
1507
            flavorid = opts.force_flavorid
1508
        else:
1509
            flavorid = choice([f["id"] for f in DFLAVORS if f["disk"] >= 20])
1510

    
1511
        imagename = image["name"]
1512

    
1513
        #Personality dictionary for file injection test
1514
        if opts.personality_path != None:
1515
            f = open(opts.personality_path)
1516
            content = b64encode(f.read())
1517
            personality = []
1518
            st = os.stat(opts.personality_path)
1519
            personality.append({
1520
                    'path': '/root/test_inj_file',
1521
                    'owner': 'root',
1522
                    'group': 'root',
1523
                    'mode': 0x7777 & st.st_mode,
1524
                    'contents': content
1525
                    })
1526
        else:
1527
            personality = None
1528

    
1529
        servername = "%s%s for %s" % (SNF_TEST_PREFIX, TEST_RUN_ID, imagename)
1530
        is_windows = imagename.lower().find("windows") >= 0
1531

    
1532
        ServerTestCase = _spawn_server_test_case(
1533
            imageid=imageid,
1534
            flavorid=flavorid,
1535
            imagename=imagename,
1536
            personality=personality,
1537
            servername=servername,
1538
            is_windows=is_windows,
1539
            action_timeout=opts.action_timeout,
1540
            build_warning=opts.build_warning,
1541
            build_fail=opts.build_fail,
1542
            query_interval=opts.query_interval,
1543
            )
1544

    
1545
        NetworkTestCase = _spawn_network_test_case(
1546
            action_timeout=opts.action_timeout,
1547
            imageid=imageid,
1548
            flavorid=flavorid,
1549
            imagename=imagename,
1550
            query_interval=opts.query_interval,
1551
            )
1552

    
1553
        seq_cases = [UnauthorizedTestCase, ImagesTestCase, FlavorsTestCase,
1554
                     ServersTestCase, ServerTestCase, NetworkTestCase]
1555

    
1556
        #folder for each image
1557
        image_folder = os.path.join(test_folder, imageid)
1558
        os.mkdir(image_folder)
1559

    
1560
        for case in seq_cases:
1561
            log_file = os.path.join(image_folder, 'details_' +
1562
                                    (case.__name__) + "_" +
1563
                                    TEST_RUN_ID + '.log')
1564
            fail_file = os.path.join(image_folder, 'failed_' +
1565
                                     (case.__name__) + "_" +
1566
                                     TEST_RUN_ID + '.log')
1567
            error_file = os.path.join(image_folder, 'error_' +
1568
                                      (case.__name__) + "_" +
1569
                                      TEST_RUN_ID + '.log')
1570

    
1571
            f = open(log_file, "w")
1572
            fail = open(fail_file, "w")
1573
            error = open(error_file, "w")
1574

    
1575
            suite = unittest.TestLoader().loadTestsFromTestCase(case)
1576
            runner = unittest.TextTestRunner(f, verbosity=2, failfast=True)
1577
            result = runner.run(suite)
1578

    
1579
            for res in result.errors:
1580
                error.write(str(res[0]) + '\n')
1581
                error.write(str(res[0].shortDescription()) + '\n')
1582
                error.write('\n')
1583

    
1584
            for res in result.failures:
1585
                fail.write(str(res[0]) + '\n')
1586
                fail.write(str(res[0].shortDescription()) + '\n')
1587
                fail.write('\n')
1588
                if opts.nofailfast == False:
1589
                    sys.exit()
1590

    
1591
if __name__ == "__main__":
1592
    sys.exit(main())