Statistics
| Branch: | Tag: | Revision:

root / snf-tools / synnefo_tools / burnin.py @ 4c9918f9

History | View | Annotate | Download (77.6 kB)

1
#!/usr/bin/env python
2

    
3
# Copyright 2011 GRNET S.A. All rights reserved.
4
#
5
# Redistribution and use in source and binary forms, with or
6
# without modification, are permitted provided that the following
7
# conditions are met:
8
#
9
#   1. Redistributions of source code must retain the above
10
#      copyright notice, this list of conditions and the following
11
#      disclaimer.
12
#
13
#   2. Redistributions in binary form must reproduce the above
14
#      copyright notice, this list of conditions and the following
15
#      disclaimer in the documentation and/or other materials
16
#      provided with the distribution.
17
#
18
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
19
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
20
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
21
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
22
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
25
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
26
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
28
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29
# POSSIBILITY OF SUCH DAMAGE.
30
#
31
# The views and conclusions contained in the software and
32
# documentation are those of the authors and should not be
33
# interpreted as representing official policies, either expressed
34
# or implied, of GRNET S.A.
35

    
36
"""Perform integration testing on a running Synnefo deployment"""
37

    
38
#import __main__
39
import datetime
40
import inspect
41
import logging
42
import os
43
import os.path
44
import paramiko
45
import prctl
46
import subprocess
47
import signal
48
import socket
49
import sys
50
import time
51
import tempfile
52
from base64 import b64encode
53
from IPy import IP
54
from multiprocessing import Process, Queue
55
from random import choice, randint
56
from optparse import OptionParser, OptionValueError
57

    
58
from kamaki.clients.compute import ComputeClient
59
from kamaki.clients.cyclades import CycladesClient
60
from kamaki.clients.image import ImageClient
61
from kamaki.clients.pithos import PithosClient
62
from kamaki.clients import ClientError
63

    
64
from vncauthproxy.d3des import generate_response as d3des_generate_response
65

    
66
# Use backported unittest functionality if Python < 2.7
67
try:
68
    import unittest2 as unittest
69
except ImportError:
70
    if sys.version_info < (2, 7):
71
        raise Exception("The unittest2 package is required for Python < 2.7")
72
    import unittest
73

    
74
# --------------------------------------------------------------------
75
# Global Variables
76
API = None
77
TOKEN = None
78
PLANKTON = None
79
PLANKTON_USER = None
80
PITHOS = None
81
PITHOS_USER = None
82
NO_IPV6 = None
83
DEFAULT_PLANKTON_USER = "images@okeanos.grnet.gr"
84
NOFAILFAST = None
85
VERBOSE = None
86

    
87
# A unique id identifying this test run
88
TEST_RUN_ID = datetime.datetime.strftime(datetime.datetime.now(),
89
                                         "%Y%m%d%H%M%S")
90
SNF_TEST_PREFIX = "snf-test-"
91

    
92
red = '\x1b[31m'
93
yellow = '\x1b[33m'
94
green = '\x1b[32m'
95
normal = '\x1b[0m'
96

    
97

    
98
# --------------------------------------------------------------------
99
# Global functions
100
def _ssh_execute(hostip, username, password, command):
101
    """Execute a command via ssh"""
102
    ssh = paramiko.SSHClient()
103
    ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
104
    try:
105
        ssh.connect(hostip, username=username, password=password)
106
    except socket.error:
107
        raise AssertionError
108
    try:
109
        stdin, stdout, stderr = ssh.exec_command(command)
110
    except paramiko.SSHException:
111
        raise AssertionError
112
    status = stdout.channel.recv_exit_status()
113
    output = stdout.readlines()
114
    ssh.close()
115
    return output, status
116

    
117

    
118
# --------------------------------------------------------------------
119
# BurninTestReulst class
120
class BurninTestResult(unittest.TextTestResult):
121
    def addSuccess(self, test):
122
        super(BurninTestResult, self).addSuccess(test)
123
        if self.showAll:
124
            if hasattr(test, 'result_dict'):
125
                run_details = test.result_dict
126

    
127
                self.stream.write("\n")
128
                for i in run_details:
129
                    self.stream.write("%s : %s \n" % (i, run_details[i]))
130
                self.stream.write("\n")
131

    
132
        elif self.dots:
133
            self.stream.write('.')
134
            self.stream.flush()
135

    
136
    def addError(self, test, err):
137
        super(BurninTestResult, self).addError(test, err)
138
        if self.showAll:
139
            self.stream.writeln("ERROR")
140
            if hasattr(test, 'result_dict'):
141
                run_details = test.result_dict
142

    
143
                self.stream.write("\n")
144
                for i in run_details:
145
                    self.stream.write("%s : %s \n" % (i, run_details[i]))
146
                self.stream.write("\n")
147

    
148
        elif self.dots:
149
            self.stream.write('E')
150
            self.stream.flush()
151

    
152
    def addFailure(self, test, err):
153
        super(BurninTestResult, self).addFailure(test, err)
154
        if self.showAll:
155
            self.stream.writeln("FAIL")
156
            if hasattr(test, 'result_dict'):
157
                run_details = test.result_dict
158

    
159
                self.stream.write("\n")
160
                for i in run_details:
161
                    self.stream.write("%s : %s \n" % (i, run_details[i]))
162
                self.stream.write("\n")
163

    
164
        elif self.dots:
165
            self.stream.write('F')
166
            self.stream.flush()
167

    
168

    
169
# --------------------------------------------------------------------
170
# Format Results
171
class burninFormatter(logging.Formatter):
172
    err_fmt = red + "ERROR: %(msg)s" + normal
173
    dbg_fmt = green + "* %(msg)s" + normal
174
    info_fmt = "%(msg)s"
175

    
176
    def __init__(self, fmt="%(levelno)s: %(msg)s"):
177
        logging.Formatter.__init__(self, fmt)
178

    
179
    def format(self, record):
180
        format_orig = self._fmt
181
        # Replace the original format with one customized by logging level
182
        if record.levelno == 10:    # DEBUG
183
            self._fmt = burninFormatter.dbg_fmt
184
        elif record.levelno == 20:  # INFO
185
            self._fmt = burninFormatter.info_fmt
186
        elif record.levelno == 40:  # ERROR
187
            self._fmt = burninFormatter.err_fmt
188
        result = logging.Formatter.format(self, record)
189
        self._fmt = format_orig
190
        return result
191

    
192
log = logging.getLogger("burnin")
193
log.setLevel(logging.DEBUG)
194
handler = logging.StreamHandler()
195
handler.setFormatter(burninFormatter())
196
log.addHandler(handler)
197

    
198

    
199
# --------------------------------------------------------------------
200
# UnauthorizedTestCase class
201
class UnauthorizedTestCase(unittest.TestCase):
202
    """Test unauthorized access"""
203
    @classmethod
204
    def setUpClass(cls):
205
        cls.result_dict = dict()
206

    
207
    def test_unauthorized_access(self):
208
        """Test access without a valid token fails"""
209
        log.info("Authentication test")
210
        falseToken = '12345'
211
        c = ComputeClient(API, falseToken)
212

    
213
        with self.assertRaises(ClientError) as cm:
214
            c.list_servers()
215
            self.assertEqual(cm.exception.status, 401)
216

    
217

    
218
# --------------------------------------------------------------------
219
# This class gest replicated into Images TestCases dynamically
220
class ImagesTestCase(unittest.TestCase):
221
    """Test image lists for consistency"""
222
    @classmethod
223
    def setUpClass(cls):
224
        """Initialize kamaki, get (detailed) list of images"""
225
        log.info("Getting simple and detailed list of images")
226
        cls.client = ComputeClient(API, TOKEN)
227
        cls.plankton = ImageClient(PLANKTON, TOKEN)
228
        cls.images = cls.plankton.list_public()
229
        cls.dimages = cls.plankton.list_public(detail=True)
230
        cls.result_dict = dict()
231
        # Create temp directory and store it inside our class
232
        # XXX: In my machine /tmp has not enough space
233
        #      so use current directory to be sure.
234
        cls.temp_dir = tempfile.mkdtemp(dir=os.getcwd())
235
        cls.temp_image_name = \
236
            SNF_TEST_PREFIX + cls.imageid + ".diskdump"
237

    
238
    @classmethod
239
    def tearDownClass(cls):
240
        """Remove local files"""
241
        try:
242
            temp_file = os.path.join(cls.temp_dir, cls.temp_image_name)
243
            os.unlink(temp_file)
244
        except:
245
            pass
246
        try:
247
            os.rmdir(cls.temp_dir)
248
        except:
249
            pass
250

    
251
    def test_001_list_images(self):
252
        """Test image list actually returns images"""
253
        self.assertGreater(len(self.images), 0)
254

    
255
    def test_002_list_images_detailed(self):
256
        """Test detailed image list is the same length as list"""
257
        self.assertEqual(len(self.dimages), len(self.images))
258

    
259
    def test_003_same_image_names(self):
260
        """Test detailed and simple image list contain same names"""
261
        names = sorted(map(lambda x: x["name"], self.images))
262
        dnames = sorted(map(lambda x: x["name"], self.dimages))
263
        self.assertEqual(names, dnames)
264

    
265
    def test_004_unique_image_names(self):
266
        """Test system images have unique names"""
267
        sys_images = filter(lambda x: x['owner'] == PLANKTON_USER,
268
                            self.dimages)
269
        names = sorted(map(lambda x: x["name"], sys_images))
270
        self.assertEqual(sorted(list(set(names))), names)
271

    
272
    def test_005_image_metadata(self):
273
        """Test every image has specific metadata defined"""
274
        keys = frozenset(["osfamily", "root_partition"])
275
        details = self.client.list_images(detail=True)
276
        for i in details:
277
            self.assertTrue(keys.issubset(i["metadata"]["values"].keys()))
278

    
279
    def test_006_download_image(self):
280
        """Download image from pithos+"""
281
        # Get image location
282
        image = filter(
283
            lambda x: x['id'] == self.imageid, self.dimages)[0]
284
        image_location = \
285
            image['location'].replace("://", " ").replace("/", " ").split()
286
        log.info("Download image, with owner %s\n\tcontainer %s, and name %s"
287
                 % (image_location[1], image_location[2], image_location[3]))
288
        pithos_client = PithosClient(PITHOS, TOKEN, image_location[1])
289
        pithos_client.container = image_location[2]
290
        temp_file = os.path.join(self.temp_dir, self.temp_image_name)
291
        with open(temp_file, "wb+") as f:
292
            pithos_client.download_object(image_location[3], f)
293

    
294
    def test_007_upload_image(self):
295
        """Upload and register image"""
296
        temp_file = os.path.join(self.temp_dir, self.temp_image_name)
297
        log.info("Upload image to pithos+")
298
        # Create container `images'
299
        pithos_client = PithosClient(PITHOS, TOKEN, PITHOS_USER)
300
        pithos_client.container = "images"
301
        pithos_client.container_put()
302
        with open(temp_file, "rb+") as f:
303
            pithos_client.upload_object(self.temp_image_name, f)
304
        log.info("Register image to plankton")
305
        location = "pithos://" + PITHOS_USER + \
306
            "/images/" + self.temp_image_name
307
        params = {'is_public': True}
308
        properties = {'OSFAMILY': "linux", 'ROOT_PARTITION': 1}
309
        self.plankton.register(self.temp_image_name, location,
310
                               params, properties)
311
        # Get image id
312
        details = self.plankton.list_public(detail=True)
313
        detail = filter(lambda x: x['location'] == location, details)
314
        self.assertEqual(len(detail), 1)
315
        cls = type(self)
316
        cls.temp_image_id = detail[0]['id']
317
        log.info("Image registered with id %s" % detail[0]['id'])
318

    
319
    def test_008_cleanup_image(self):
320
        """Cleanup image test"""
321
        log.info("Cleanup image test")
322
        # Remove image from pithos+
323
        pithos_client = PithosClient(PITHOS, TOKEN, PITHOS_USER)
324
        pithos_client.container = "images"
325
        pithos_client.del_object(self.temp_image_name)
326

    
327

    
328
# --------------------------------------------------------------------
329
# FlavorsTestCase class
330
class FlavorsTestCase(unittest.TestCase):
331
    """Test flavor lists for consistency"""
332
    @classmethod
333
    def setUpClass(cls):
334
        """Initialize kamaki, get (detailed) list of flavors"""
335
        log.info("Getting simple and detailed list of flavors")
336
        cls.client = ComputeClient(API, TOKEN)
337
        cls.flavors = cls.client.list_flavors()
338
        cls.dflavors = cls.client.list_flavors(detail=True)
339
        cls.result_dict = dict()
340

    
341
    def test_001_list_flavors(self):
342
        """Test flavor list actually returns flavors"""
343
        self.assertGreater(len(self.flavors), 0)
344

    
345
    def test_002_list_flavors_detailed(self):
346
        """Test detailed flavor list is the same length as list"""
347
        self.assertEquals(len(self.dflavors), len(self.flavors))
348

    
349
    def test_003_same_flavor_names(self):
350
        """Test detailed and simple flavor list contain same names"""
351
        names = sorted(map(lambda x: x["name"], self.flavors))
352
        dnames = sorted(map(lambda x: x["name"], self.dflavors))
353
        self.assertEqual(names, dnames)
354

    
355
    def test_004_unique_flavor_names(self):
356
        """Test flavors have unique names"""
357
        names = sorted(map(lambda x: x["name"], self.flavors))
358
        self.assertEqual(sorted(list(set(names))), names)
359

    
360
    def test_005_well_formed_flavor_names(self):
361
        """Test flavors have names of the form CxxRyyDzz
362
        Where xx is vCPU count, yy is RAM in MiB, zz is Disk in GiB
363
        """
364
        for f in self.dflavors:
365
            self.assertEqual("C%dR%dD%d" % (f["cpu"], f["ram"], f["disk"]),
366
                             f["name"],
367
                             "Flavor %s does not match its specs." % f["name"])
368

    
369

    
370
# --------------------------------------------------------------------
371
# ServersTestCase class
372
class ServersTestCase(unittest.TestCase):
373
    """Test server lists for consistency"""
374
    @classmethod
375
    def setUpClass(cls):
376
        """Initialize kamaki, get (detailed) list of servers"""
377
        log.info("Getting simple and detailed list of servers")
378

    
379
        cls.client = ComputeClient(API, TOKEN)
380
        cls.servers = cls.client.list_servers()
381
        cls.dservers = cls.client.list_servers(detail=True)
382
        cls.result_dict = dict()
383

    
384
    # def test_001_list_servers(self):
385
    #     """Test server list actually returns servers"""
386
    #     self.assertGreater(len(self.servers), 0)
387

    
388
    def test_002_list_servers_detailed(self):
389
        """Test detailed server list is the same length as list"""
390
        self.assertEqual(len(self.dservers), len(self.servers))
391

    
392
    def test_003_same_server_names(self):
393
        """Test detailed and simple flavor list contain same names"""
394
        names = sorted(map(lambda x: x["name"], self.servers))
395
        dnames = sorted(map(lambda x: x["name"], self.dservers))
396
        self.assertEqual(names, dnames)
397

    
398

    
399
# --------------------------------------------------------------------
400
# Pithos Test Cases
401
class PithosTestCase(unittest.TestCase):
402
    """Test pithos functionality"""
403
    @classmethod
404
    def setUpClass(cls):
405
        """Initialize kamaki, get list of containers"""
406
        log.info("Getting list of containers")
407

    
408
        cls.client = PithosClient(PITHOS, TOKEN, PITHOS_USER)
409
        cls.containers = cls.client.list_containers()
410
        cls.result_dict = dict()
411

    
412
    def test_001_list_containers(self):
413
        """Test container list actually returns containers"""
414
        self.assertGreater(len(self.containers), 0)
415

    
416
    def test_002_unique_containers(self):
417
        """Test if containers have unique names"""
418
        names = [n['name'] for n in self.containers]
419
        names = sorted(names)
420
        self.assertEqual(sorted(list(set(names))), names)
421

    
422
    def test_003_create_container(self):
423
        """Test create a container"""
424
        rand_num = randint(1000, 9999)
425
        rand_name = "%s%s" % (SNF_TEST_PREFIX, rand_num)
426
        names = [n['name'] for n in self.containers]
427
        while rand_name in names:
428
            rand_num = randint(1000, 9999)
429
            rand_name = "%s%s" % (SNF_TEST_PREFIX, rand_num)
430
        # Create container
431
        self.client.container = rand_name
432
        self.client.container_put()
433
        # Get list of containers
434
        new_containers = self.client.list_containers()
435
        new_container_names = [n['name'] for n in new_containers]
436
        self.assertIn(rand_name, new_container_names)
437

    
438
    def test_004_upload(self):
439
        """Test uploading something to pithos+"""
440
        # Create a tmp file
441
        with tempfile.TemporaryFile() as f:
442
            f.write("This is a temp file")
443
            f.seek(0, 0)
444
            # Where to save file
445
            self.client.upload_object("test.txt", f)
446

    
447
    def test_005_download(self):
448
        """Test download something from pithos+"""
449
        # Create tmp directory to save file
450
        tmp_dir = tempfile.mkdtemp()
451
        tmp_file = os.path.join(tmp_dir, "test.txt")
452
        with open(tmp_file, "wb+") as f:
453
            self.client.download_object("test.txt", f)
454
            # Read file
455
            f.seek(0, 0)
456
            content = f.read()
457
        # Remove files
458
        os.unlink(tmp_file)
459
        os.rmdir(tmp_dir)
460
        # Compare results
461
        self.assertEqual(content, "This is a temp file")
462

    
463
    def test_006_remove(self):
464
        """Test removing files and containers"""
465
        cont_name = self.client.container
466
        self.client.del_object("test.txt")
467
        self.client.purge_container()
468
        # List containers
469
        containers = self.client.list_containers()
470
        cont_names = [n['name'] for n in containers]
471
        self.assertNotIn(cont_name, cont_names)
472

    
473

    
474
# --------------------------------------------------------------------
475
# This class gets replicated into actual TestCases dynamically
476
class SpawnServerTestCase(unittest.TestCase):
477
    """Test scenario for server of the specified image"""
478
    @classmethod
479
    def setUpClass(cls):
480
        """Initialize a kamaki instance"""
481
        log.info("Spawning server for image `%s'" % cls.imagename)
482
        cls.client = ComputeClient(API, TOKEN)
483
        cls.cyclades = CycladesClient(API, TOKEN)
484
        cls.result_dict = dict()
485

    
486
    def _get_ipv4(self, server):
487
        """Get the public IPv4 of a server from the detailed server info"""
488

    
489
        nics = server["attachments"]["values"]
490

    
491
        for nic in nics:
492
            net_id = nic["network_id"]
493
            if self.cyclades.get_network_details(net_id)["public"]:
494
                public_addrs = nic["ipv4"]
495

    
496
        self.assertTrue(public_addrs is not None)
497

    
498
        return public_addrs
499

    
500
    def _get_ipv6(self, server):
501
        """Get the public IPv6 of a server from the detailed server info"""
502

    
503
        nics = server["attachments"]["values"]
504

    
505
        for nic in nics:
506
            net_id = nic["network_id"]
507
            if self.cyclades.get_network_details(net_id)["public"]:
508
                public_addrs = nic["ipv6"]
509

    
510
        self.assertTrue(public_addrs is not None)
511

    
512
        return public_addrs
513

    
514
    def _connect_loginname(self, os_value):
515
        """Return the login name for connections based on the server OS"""
516
        if os_value in ("Ubuntu", "Kubuntu", "Fedora"):
517
            return "user"
518
        elif os_value in ("windows", "windows_alpha1"):
519
            return "Administrator"
520
        else:
521
            return "root"
522

    
523
    def _verify_server_status(self, current_status, new_status):
524
        """Verify a server has switched to a specified status"""
525
        server = self.client.get_server_details(self.serverid)
526
        if server["status"] not in (current_status, new_status):
527
            return None  # Do not raise exception, return so the test fails
528
        self.assertEquals(server["status"], new_status)
529

    
530
    def _get_connected_tcp_socket(self, family, host, port):
531
        """Get a connected socket from the specified family to host:port"""
532
        sock = None
533
        for res in \
534
            socket.getaddrinfo(host, port, family, socket.SOCK_STREAM, 0,
535
                               socket.AI_PASSIVE):
536
            af, socktype, proto, canonname, sa = res
537
            try:
538
                sock = socket.socket(af, socktype, proto)
539
            except socket.error:
540
                sock = None
541
                continue
542
            try:
543
                sock.connect(sa)
544
            except socket.error:
545
                sock.close()
546
                sock = None
547
                continue
548
        self.assertIsNotNone(sock)
549
        return sock
550

    
551
    def _ping_once(self, ipv6, ip):
552
        """Test server responds to a single IPv4 or IPv6 ping"""
553
        cmd = "ping%s -c 2 -w 3 %s" % ("6" if ipv6 else "", ip)
554
        ping = subprocess.Popen(cmd, shell=True,
555
                                stdout=subprocess.PIPE, stderr=subprocess.PIPE)
556
        (stdout, stderr) = ping.communicate()
557
        ret = ping.wait()
558
        self.assertEquals(ret, 0)
559

    
560
    def _get_hostname_over_ssh(self, hostip, username, password):
561
        lines, status = _ssh_execute(
562
            hostip, username, password, "hostname")
563
        self.assertEqual(len(lines), 1)
564
        return lines[0]
565

    
566
    def _try_until_timeout_expires(self, warn_timeout, fail_timeout,
567
                                   opmsg, callable, *args, **kwargs):
568
        if warn_timeout == fail_timeout:
569
            warn_timeout = fail_timeout + 1
570
        warn_tmout = time.time() + warn_timeout
571
        fail_tmout = time.time() + fail_timeout
572
        while True:
573
            self.assertLess(time.time(), fail_tmout,
574
                            "operation `%s' timed out" % opmsg)
575
            if time.time() > warn_tmout:
576
                log.warning("Server %d: `%s' operation `%s' not done yet",
577
                            self.serverid, self.servername, opmsg)
578
            try:
579
                log.info("%s... " % opmsg)
580
                return callable(*args, **kwargs)
581
            except AssertionError:
582
                pass
583
            time.sleep(self.query_interval)
584

    
585
    def _insist_on_tcp_connection(self, family, host, port):
586
        familystr = {socket.AF_INET: "IPv4", socket.AF_INET6: "IPv6",
587
                     socket.AF_UNSPEC: "Unspecified-IPv4/6"}
588
        msg = "connect over %s to %s:%s" % \
589
              (familystr.get(family, "Unknown"), host, port)
590
        sock = self._try_until_timeout_expires(
591
            self.action_timeout, self.action_timeout,
592
            msg, self._get_connected_tcp_socket,
593
            family, host, port)
594
        return sock
595

    
596
    def _insist_on_status_transition(self, current_status, new_status,
597
                                     fail_timeout, warn_timeout=None):
598
        msg = "Server %d: `%s', waiting for %s -> %s" % \
599
              (self.serverid, self.servername, current_status, new_status)
600
        if warn_timeout is None:
601
            warn_timeout = fail_timeout
602
        self._try_until_timeout_expires(warn_timeout, fail_timeout,
603
                                        msg, self._verify_server_status,
604
                                        current_status, new_status)
605
        # Ensure the status is actually the expected one
606
        server = self.client.get_server_details(self.serverid)
607
        self.assertEquals(server["status"], new_status)
608

    
609
    def _insist_on_ssh_hostname(self, hostip, username, password):
610
        msg = "SSH to %s, as %s/%s" % (hostip, username, password)
611
        hostname = self._try_until_timeout_expires(
612
            self.action_timeout, self.action_timeout,
613
            msg, self._get_hostname_over_ssh,
614
            hostip, username, password)
615

    
616
        # The hostname must be of the form 'prefix-id'
617
        self.assertTrue(hostname.endswith("-%d\n" % self.serverid))
618

    
619
    def _check_file_through_ssh(self, hostip, username, password,
620
                                remotepath, content):
621
        msg = "Trying file injection through SSH to %s, as %s/%s" % \
622
            (hostip, username, password)
623
        log.info(msg)
624
        try:
625
            ssh = paramiko.SSHClient()
626
            ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
627
            ssh.connect(hostip, username=username, password=password)
628
            ssh.close()
629
        except socket.error:
630
            raise AssertionError
631

    
632
        transport = paramiko.Transport((hostip, 22))
633
        transport.connect(username=username, password=password)
634

    
635
        localpath = '/tmp/' + SNF_TEST_PREFIX + 'injection'
636
        sftp = paramiko.SFTPClient.from_transport(transport)
637
        sftp.get(remotepath, localpath)
638
        sftp.close()
639
        transport.close()
640

    
641
        f = open(localpath)
642
        remote_content = b64encode(f.read())
643

    
644
        # Check if files are the same
645
        return (remote_content == content)
646

    
647
    def _skipIf(self, condition, msg):
648
        if condition:
649
            self.skipTest(msg)
650

    
651
    def test_001_submit_create_server(self):
652
        """Test submit create server request"""
653

    
654
        log.info("Submit new server request")
655
        server = self.client.create_server(self.servername, self.flavorid,
656
                                           self.imageid, self.personality)
657

    
658
        log.info("Server id: " + str(server["id"]))
659
        log.info("Server password: " + server["adminPass"])
660
        self.assertEqual(server["name"], self.servername)
661
        self.assertEqual(server["flavorRef"], self.flavorid)
662
        self.assertEqual(server["imageRef"], self.imageid)
663
        self.assertEqual(server["status"], "BUILD")
664

    
665
        # Update class attributes to reflect data on building server
666
        cls = type(self)
667
        cls.serverid = server["id"]
668
        cls.username = None
669
        cls.passwd = server["adminPass"]
670

    
671
        self.result_dict["Server ID"] = str(server["id"])
672
        self.result_dict["Password"] = str(server["adminPass"])
673

    
674
    def test_002a_server_is_building_in_list(self):
675
        """Test server is in BUILD state, in server list"""
676
        log.info("Server in BUILD state in server list")
677

    
678
        self.result_dict.clear()
679

    
680
        servers = self.client.list_servers(detail=True)
681
        servers = filter(lambda x: x["name"] == self.servername, servers)
682

    
683
        server = servers[0]
684
        self.assertEqual(server["name"], self.servername)
685
        self.assertEqual(server["flavorRef"], self.flavorid)
686
        self.assertEqual(server["imageRef"], self.imageid)
687
        self.assertEqual(server["status"], "BUILD")
688

    
689
    def test_002b_server_is_building_in_details(self):
690
        """Test server is in BUILD state, in details"""
691

    
692
        log.info("Server in BUILD state in details")
693

    
694
        server = self.client.get_server_details(self.serverid)
695
        self.assertEqual(server["name"], self.servername)
696
        self.assertEqual(server["flavorRef"], self.flavorid)
697
        self.assertEqual(server["imageRef"], self.imageid)
698
        self.assertEqual(server["status"], "BUILD")
699

    
700
    def test_002c_set_server_metadata(self):
701

    
702
        log.info("Creating server metadata")
703

    
704
        image = self.client.get_image_details(self.imageid)
705
        os_value = image["metadata"]["values"]["os"]
706
        users = image["metadata"]["values"].get("users", None)
707
        self.client.update_server_metadata(self.serverid, OS=os_value)
708

    
709
        userlist = users.split()
710

    
711
        # Determine the username to use for future connections
712
        # to this host
713
        cls = type(self)
714

    
715
        if "root" in userlist:
716
            cls.username = "root"
717
        elif users is None:
718
            cls.username = self._connect_loginname(os_value)
719
        else:
720
            cls.username = choice(userlist)
721

    
722
        self.assertIsNotNone(cls.username)
723

    
724
    def test_002d_verify_server_metadata(self):
725
        """Test server metadata keys are set based on image metadata"""
726

    
727
        log.info("Verifying image metadata")
728

    
729
        servermeta = self.client.get_server_metadata(self.serverid)
730
        imagemeta = self.client.get_image_metadata(self.imageid)
731

    
732
        self.assertEqual(servermeta["OS"], imagemeta["os"])
733

    
734
    def test_003_server_becomes_active(self):
735
        """Test server becomes ACTIVE"""
736

    
737
        log.info("Waiting for server to become ACTIVE")
738

    
739
        self._insist_on_status_transition(
740
            "BUILD", "ACTIVE", self.build_fail, self.build_warning)
741

    
742
    def test_003a_get_server_oob_console(self):
743
        """Test getting OOB server console over VNC
744

745
        Implementation of RFB protocol follows
746
        http://www.realvnc.com/docs/rfbproto.pdf.
747

748
        """
749
        console = self.cyclades.get_server_console(self.serverid)
750
        self.assertEquals(console['type'], "vnc")
751
        sock = self._insist_on_tcp_connection(
752
            socket.AF_INET, console["host"], console["port"])
753

    
754
        # Step 1. ProtocolVersion message (par. 6.1.1)
755
        version = sock.recv(1024)
756
        self.assertEquals(version, 'RFB 003.008\n')
757
        sock.send(version)
758

    
759
        # Step 2. Security (par 6.1.2): Only VNC Authentication supported
760
        sec = sock.recv(1024)
761
        self.assertEquals(list(sec), ['\x01', '\x02'])
762

    
763
        # Step 3. Request VNC Authentication (par 6.1.2)
764
        sock.send('\x02')
765

    
766
        # Step 4. Receive Challenge (par 6.2.2)
767
        challenge = sock.recv(1024)
768
        self.assertEquals(len(challenge), 16)
769

    
770
        # Step 5. DES-Encrypt challenge, use password as key (par 6.2.2)
771
        response = d3des_generate_response(
772
            (console["password"] + '\0' * 8)[:8], challenge)
773
        sock.send(response)
774

    
775
        # Step 6. SecurityResult (par 6.1.3)
776
        result = sock.recv(4)
777
        self.assertEquals(list(result), ['\x00', '\x00', '\x00', '\x00'])
778
        sock.close()
779

    
780
    def test_004_server_has_ipv4(self):
781
        """Test active server has a valid IPv4 address"""
782

    
783
        log.info("Validate server's IPv4")
784

    
785
        server = self.client.get_server_details(self.serverid)
786
        ipv4 = self._get_ipv4(server)
787

    
788
        self.result_dict.clear()
789
        self.result_dict["IPv4"] = str(ipv4)
790

    
791
        self.assertEquals(IP(ipv4).version(), 4)
792

    
793
    def test_005_server_has_ipv6(self):
794
        """Test active server has a valid IPv6 address"""
795
        self._skipIf(NO_IPV6, "--no-ipv6 flag enabled")
796

    
797
        log.info("Validate server's IPv6")
798

    
799
        server = self.client.get_server_details(self.serverid)
800
        ipv6 = self._get_ipv6(server)
801

    
802
        self.result_dict.clear()
803
        self.result_dict["IPv6"] = str(ipv6)
804

    
805
        self.assertEquals(IP(ipv6).version(), 6)
806

    
807
    def test_006_server_responds_to_ping_IPv4(self):
808
        """Test server responds to ping on IPv4 address"""
809

    
810
        log.info("Testing if server responds to pings in IPv4")
811
        self.result_dict.clear()
812

    
813
        server = self.client.get_server_details(self.serverid)
814
        ip = self._get_ipv4(server)
815
        self._try_until_timeout_expires(self.action_timeout,
816
                                        self.action_timeout,
817
                                        "PING IPv4 to %s" % ip,
818
                                        self._ping_once,
819
                                        False, ip)
820

    
821
    def test_007_server_responds_to_ping_IPv6(self):
822
        """Test server responds to ping on IPv6 address"""
823
        self._skipIf(NO_IPV6, "--no-ipv6 flag enabled")
824
        log.info("Testing if server responds to pings in IPv6")
825

    
826
        server = self.client.get_server_details(self.serverid)
827
        ip = self._get_ipv6(server)
828
        self._try_until_timeout_expires(self.action_timeout,
829
                                        self.action_timeout,
830
                                        "PING IPv6 to %s" % ip,
831
                                        self._ping_once,
832
                                        True, ip)
833

    
834
    def test_008_submit_shutdown_request(self):
835
        """Test submit request to shutdown server"""
836

    
837
        log.info("Shutting down server")
838

    
839
        self.cyclades.shutdown_server(self.serverid)
840

    
841
    def test_009_server_becomes_stopped(self):
842
        """Test server becomes STOPPED"""
843

    
844
        log.info("Waiting until server becomes STOPPED")
845
        self._insist_on_status_transition(
846
            "ACTIVE", "STOPPED", self.action_timeout, self.action_timeout)
847

    
848
    def test_010_submit_start_request(self):
849
        """Test submit start server request"""
850

    
851
        log.info("Starting server")
852

    
853
        self.cyclades.start_server(self.serverid)
854

    
855
    def test_011_server_becomes_active(self):
856
        """Test server becomes ACTIVE again"""
857

    
858
        log.info("Waiting until server becomes ACTIVE")
859
        self._insist_on_status_transition(
860
            "STOPPED", "ACTIVE", self.action_timeout, self.action_timeout)
861

    
862
    def test_011a_server_responds_to_ping_IPv4(self):
863
        """Test server OS is actually up and running again"""
864

    
865
        log.info("Testing if server is actually up and running")
866

    
867
        self.test_006_server_responds_to_ping_IPv4()
868

    
869
    def test_012_ssh_to_server_IPv4(self):
870
        """Test SSH to server public IPv4 works, verify hostname"""
871

    
872
        self._skipIf(self.is_windows, "only valid for Linux servers")
873
        server = self.client.get_server_details(self.serverid)
874
        self._insist_on_ssh_hostname(self._get_ipv4(server),
875
                                     self.username, self.passwd)
876

    
877
    def test_013_ssh_to_server_IPv6(self):
878
        """Test SSH to server public IPv6 works, verify hostname"""
879
        self._skipIf(self.is_windows, "only valid for Linux servers")
880
        self._skipIf(NO_IPV6, "--no-ipv6 flag enabled")
881

    
882
        server = self.client.get_server_details(self.serverid)
883
        self._insist_on_ssh_hostname(self._get_ipv6(server),
884
                                     self.username, self.passwd)
885

    
886
    def test_014_rdp_to_server_IPv4(self):
887
        "Test RDP connection to server public IPv4 works"""
888
        self._skipIf(not self.is_windows, "only valid for Windows servers")
889
        server = self.client.get_server_details(self.serverid)
890
        ipv4 = self._get_ipv4(server)
891
        sock = self._insist_on_tcp_connection(socket.AF_INET, ipv4, 3389)
892

    
893
        # No actual RDP processing done. We assume the RDP server is there
894
        # if the connection to the RDP port is successful.
895
        # FIXME: Use rdesktop, analyze exit code? see manpage [costasd]
896
        sock.close()
897

    
898
    def test_015_rdp_to_server_IPv6(self):
899
        "Test RDP connection to server public IPv6 works"""
900
        self._skipIf(not self.is_windows, "only valid for Windows servers")
901
        self._skipIf(NO_IPV6, "--no-ipv6 flag enabled")
902

    
903
        server = self.client.get_server_details(self.serverid)
904
        ipv6 = self._get_ipv6(server)
905
        sock = self._get_tcp_connection(socket.AF_INET6, ipv6, 3389)
906

    
907
        # No actual RDP processing done. We assume the RDP server is there
908
        # if the connection to the RDP port is successful.
909
        sock.close()
910

    
911
    def test_016_personality_is_enforced(self):
912
        """Test file injection for personality enforcement"""
913
        self._skipIf(self.is_windows, "only implemented for Linux servers")
914
        self._skipIf(self.personality is None, "No personality file selected")
915

    
916
        log.info("Trying to inject file for personality enforcement")
917

    
918
        server = self.client.get_server_details(self.serverid)
919

    
920
        for inj_file in self.personality:
921
            equal_files = self._check_file_through_ssh(self._get_ipv4(server),
922
                                                       inj_file['owner'],
923
                                                       self.passwd,
924
                                                       inj_file['path'],
925
                                                       inj_file['contents'])
926
            self.assertTrue(equal_files)
927

    
928
    def test_017_submit_delete_request(self):
929
        """Test submit request to delete server"""
930

    
931
        log.info("Deleting server")
932

    
933
        self.client.delete_server(self.serverid)
934

    
935
    def test_018_server_becomes_deleted(self):
936
        """Test server becomes DELETED"""
937

    
938
        log.info("Testing if server becomes DELETED")
939

    
940
        self._insist_on_status_transition(
941
            "ACTIVE", "DELETED", self.action_timeout, self.action_timeout)
942

    
943
    def test_019_server_no_longer_in_server_list(self):
944
        """Test server is no longer in server list"""
945

    
946
        log.info("Test if server is no longer listed")
947

    
948
        servers = self.client.list_servers()
949
        self.assertNotIn(self.serverid, [s["id"] for s in servers])
950

    
951

    
952
class NetworkTestCase(unittest.TestCase):
953
    """ Testing networking in cyclades """
954

    
955
    @classmethod
956
    def setUpClass(cls):
957
        "Initialize kamaki, get list of current networks"
958

    
959
        cls.client = CycladesClient(API, TOKEN)
960
        cls.compute = ComputeClient(API, TOKEN)
961

    
962
        cls.servername = "%s%s for %s" % (SNF_TEST_PREFIX,
963
                                          TEST_RUN_ID,
964
                                          cls.imagename)
965

    
966
        #Dictionary initialization for the vms credentials
967
        cls.serverid = dict()
968
        cls.username = dict()
969
        cls.password = dict()
970
        cls.is_windows = cls.imagename.lower().find("windows") >= 0
971

    
972
        cls.result_dict = dict()
973

    
974
    def _skipIf(self, condition, msg):
975
        if condition:
976
            self.skipTest(msg)
977

    
978
    def _get_ipv4(self, server):
979
        """Get the public IPv4 of a server from the detailed server info"""
980

    
981
        nics = server["attachments"]["values"]
982

    
983
        for nic in nics:
984
            net_id = nic["network_id"]
985
            if self.client.get_network_details(net_id)["public"]:
986
                public_addrs = nic["ipv4"]
987

    
988
        self.assertTrue(public_addrs is not None)
989

    
990
        return public_addrs
991

    
992
    def _connect_loginname(self, os_value):
993
        """Return the login name for connections based on the server OS"""
994
        if os_value in ("Ubuntu", "Kubuntu", "Fedora"):
995
            return "user"
996
        elif os_value in ("windows", "windows_alpha1"):
997
            return "Administrator"
998
        else:
999
            return "root"
1000

    
1001
    def _ping_once(self, ip):
1002

    
1003
        """Test server responds to a single IPv4 or IPv6 ping"""
1004
        cmd = "ping -c 2 -w 3 %s" % (ip)
1005
        ping = subprocess.Popen(cmd, shell=True,
1006
                                stdout=subprocess.PIPE, stderr=subprocess.PIPE)
1007
        (stdout, stderr) = ping.communicate()
1008
        ret = ping.wait()
1009

    
1010
        return (ret == 0)
1011

    
1012
    def test_00001a_submit_create_server_A(self):
1013
        """Test submit create server request"""
1014

    
1015
        log.info("Creating test server A")
1016

    
1017
        serverA = self.client.create_server(self.servername, self.flavorid,
1018
                                            self.imageid, personality=None)
1019

    
1020
        self.assertEqual(serverA["name"], self.servername)
1021
        self.assertEqual(serverA["flavorRef"], self.flavorid)
1022
        self.assertEqual(serverA["imageRef"], self.imageid)
1023
        self.assertEqual(serverA["status"], "BUILD")
1024

    
1025
        # Update class attributes to reflect data on building server
1026
        self.serverid['A'] = serverA["id"]
1027
        self.username['A'] = None
1028
        self.password['A'] = serverA["adminPass"]
1029

    
1030
        log.info("Server A id:" + str(serverA["id"]))
1031
        log.info("Server password " + (self.password['A']))
1032

    
1033
        self.result_dict["Server A ID"] = str(serverA["id"])
1034
        self.result_dict["Server A password"] = serverA["adminPass"]
1035

    
1036
    def test_00001b_serverA_becomes_active(self):
1037
        """Test server becomes ACTIVE"""
1038

    
1039
        log.info("Waiting until test server A becomes ACTIVE")
1040
        self.result_dict.clear()
1041

    
1042
        fail_tmout = time.time() + self.action_timeout
1043
        while True:
1044
            d = self.client.get_server_details(self.serverid['A'])
1045
            status = d['status']
1046
            if status == 'ACTIVE':
1047
                active = True
1048
                break
1049
            elif time.time() > fail_tmout:
1050
                self.assertLess(time.time(), fail_tmout)
1051
            else:
1052
                time.sleep(self.query_interval)
1053

    
1054
        self.assertTrue(active)
1055

    
1056
    def test_00002a_submit_create_server_B(self):
1057
        """Test submit create server request"""
1058

    
1059
        log.info("Creating test server B")
1060

    
1061
        serverB = self.client.create_server(self.servername, self.flavorid,
1062
                                            self.imageid, personality=None)
1063

    
1064
        self.assertEqual(serverB["name"], self.servername)
1065
        self.assertEqual(serverB["flavorRef"], self.flavorid)
1066
        self.assertEqual(serverB["imageRef"], self.imageid)
1067
        self.assertEqual(serverB["status"], "BUILD")
1068

    
1069
        # Update class attributes to reflect data on building server
1070
        self.serverid['B'] = serverB["id"]
1071
        self.username['B'] = None
1072
        self.password['B'] = serverB["adminPass"]
1073

    
1074
        log.info("Server B id: " + str(serverB["id"]))
1075
        log.info("Password " + (self.password['B']))
1076

    
1077
        self.result_dict.clear()
1078
        self.result_dict["Server B ID"] = str(serverB["id"])
1079
        self.result_dict["Server B password"] = serverB["adminPass"]
1080

    
1081
    def test_00002b_serverB_becomes_active(self):
1082
        """Test server becomes ACTIVE"""
1083

    
1084
        log.info("Waiting until test server B becomes ACTIVE")
1085
        self.result_dict.clear()
1086

    
1087
        fail_tmout = time.time() + self.action_timeout
1088
        while True:
1089
            d = self.client.get_server_details(self.serverid['B'])
1090
            status = d['status']
1091
            if status == 'ACTIVE':
1092
                active = True
1093
                break
1094
            elif time.time() > fail_tmout:
1095
                self.assertLess(time.time(), fail_tmout)
1096
            else:
1097
                time.sleep(self.query_interval)
1098

    
1099
        self.assertTrue(active)
1100

    
1101
    def test_001_create_network(self):
1102
        """Test submit create network request"""
1103

    
1104
        log.info("Submit new network request")
1105
        self.result_dict.clear()
1106

    
1107
        name = SNF_TEST_PREFIX + TEST_RUN_ID
1108
        #previous_num = len(self.client.list_networks())
1109
        network = self.client.create_network(name, cidr='10.0.0.1/28')
1110

    
1111
        #Test if right name is assigned
1112
        self.assertEqual(network['name'], name)
1113

    
1114
        # Update class attributes
1115
        cls = type(self)
1116
        cls.networkid = network['id']
1117
        #networks = self.client.list_networks()
1118

    
1119
        fail_tmout = time.time() + self.action_timeout
1120

    
1121
        #Test if new network is created
1122
        while True:
1123
            d = self.client.get_network_details(network['id'])
1124
            if d['status'] == 'ACTIVE':
1125
                connected = True
1126
                break
1127
            elif time.time() > fail_tmout:
1128
                self.assertLess(time.time(), fail_tmout)
1129
            else:
1130
                log.info("Waiting for network to become ACTIVE")
1131
                time.sleep(self.query_interval)
1132

    
1133
        self.assertTrue(connected)
1134

    
1135
        self.result_dict["Private network ID"] = str(network['id'])
1136

    
1137
    def test_002_connect_to_network(self):
1138
        """Test connect VMs to network"""
1139

    
1140
        log.info("Connect VMs to private network")
1141
        self.result_dict.clear()
1142

    
1143
        self.client.connect_server(self.serverid['A'], self.networkid)
1144
        self.client.connect_server(self.serverid['B'], self.networkid)
1145

    
1146
        #Insist on connecting until action timeout
1147
        fail_tmout = time.time() + self.action_timeout
1148

    
1149
        while True:
1150

    
1151
            netsA = [x['network_id']
1152
                     for x in self.client.get_server_details(
1153
                         self.serverid['A'])['attachments']['values']]
1154
            netsB = [x['network_id']
1155
                     for x in self.client.get_server_details(
1156
                         self.serverid['B'])['attachments']['values']]
1157

    
1158
            if (self.networkid in netsA) and (self.networkid in netsB):
1159
                conn_exists = True
1160
                break
1161
            elif time.time() > fail_tmout:
1162
                self.assertLess(time.time(), fail_tmout)
1163
            else:
1164
                time.sleep(self.query_interval)
1165

    
1166
        #Adding private IPs to class attributes
1167
        cls = type(self)
1168
        cls.priv_ip = dict()
1169

    
1170
        nicsA = self.client.get_server_details(
1171
            self.serverid['A'])['attachments']['values']
1172
        nicsB = self.client.get_server_details(
1173
            self.serverid['B'])['attachments']['values']
1174

    
1175
        if conn_exists:
1176
            for nic in nicsA:
1177
                if nic["network_id"] == self.networkid:
1178
                    cls.priv_ip["A"] = nic["ipv4"]
1179

    
1180
            for nic in nicsB:
1181
                if nic["network_id"] == self.networkid:
1182
                    cls.priv_ip["B"] = nic["ipv4"]
1183

    
1184
        self.assertTrue(conn_exists)
1185

    
1186
    def test_002a_reboot(self):
1187
        """Rebooting server A"""
1188

    
1189
        log.info("Rebooting server A")
1190

    
1191
        self.client.shutdown_server(self.serverid['A'])
1192

    
1193
        fail_tmout = time.time() + self.action_timeout
1194
        while True:
1195
            d = self.client.get_server_details(self.serverid['A'])
1196
            status = d['status']
1197
            if status == 'STOPPED':
1198
                break
1199
            elif time.time() > fail_tmout:
1200
                self.assertLess(time.time(), fail_tmout)
1201
            else:
1202
                time.sleep(self.query_interval)
1203

    
1204
        self.client.start_server(self.serverid['A'])
1205

    
1206
        while True:
1207
            d = self.client.get_server_details(self.serverid['A'])
1208
            status = d['status']
1209
            if status == 'ACTIVE':
1210
                active = True
1211
                break
1212
            elif time.time() > fail_tmout:
1213
                self.assertLess(time.time(), fail_tmout)
1214
            else:
1215
                time.sleep(self.query_interval)
1216

    
1217
        self.assertTrue(active)
1218

    
1219
    def test_002b_ping_server_A(self):
1220
        "Test if server A responds to IPv4 pings"
1221

    
1222
        log.info("Testing if server A responds to IPv4 pings ")
1223
        self.result_dict.clear()
1224

    
1225
        server = self.client.get_server_details(self.serverid['A'])
1226
        ip = self._get_ipv4(server)
1227

    
1228
        fail_tmout = time.time() + self.action_timeout
1229

    
1230
        s = False
1231

    
1232
        self.result_dict["Server A public IP"] = str(ip)
1233

    
1234
        while True:
1235

    
1236
            if self._ping_once(ip):
1237
                s = True
1238
                break
1239

    
1240
            elif time.time() > fail_tmout:
1241
                self.assertLess(time.time(), fail_tmout)
1242

    
1243
            else:
1244
                time.sleep(self.query_interval)
1245

    
1246
        self.assertTrue(s)
1247

    
1248
    def test_002c_reboot(self):
1249
        """Reboot server B"""
1250

    
1251
        log.info("Rebooting server B")
1252
        self.result_dict.clear()
1253

    
1254
        self.client.shutdown_server(self.serverid['B'])
1255

    
1256
        fail_tmout = time.time() + self.action_timeout
1257
        while True:
1258
            d = self.client.get_server_details(self.serverid['B'])
1259
            status = d['status']
1260
            if status == 'STOPPED':
1261
                break
1262
            elif time.time() > fail_tmout:
1263
                self.assertLess(time.time(), fail_tmout)
1264
            else:
1265
                time.sleep(self.query_interval)
1266

    
1267
        self.client.start_server(self.serverid['B'])
1268

    
1269
        while True:
1270
            d = self.client.get_server_details(self.serverid['B'])
1271
            status = d['status']
1272
            if status == 'ACTIVE':
1273
                active = True
1274
                break
1275
            elif time.time() > fail_tmout:
1276
                self.assertLess(time.time(), fail_tmout)
1277
            else:
1278
                time.sleep(self.query_interval)
1279

    
1280
        self.assertTrue(active)
1281

    
1282
    def test_002d_ping_server_B(self):
1283
        """Test if server B responds to IPv4 pings"""
1284

    
1285
        log.info("Testing if server B responds to IPv4 pings")
1286
        self.result_dict.clear()
1287

    
1288
        server = self.client.get_server_details(self.serverid['B'])
1289
        ip = self._get_ipv4(server)
1290

    
1291
        fail_tmout = time.time() + self.action_timeout
1292

    
1293
        s = False
1294

    
1295
        self.result_dict["Server B public IP"] = str(ip)
1296

    
1297
        while True:
1298
            if self._ping_once(ip):
1299
                s = True
1300
                break
1301

    
1302
            elif time.time() > fail_tmout:
1303
                self.assertLess(time.time(), fail_tmout)
1304

    
1305
            else:
1306
                time.sleep(self.query_interval)
1307

    
1308
        self.assertTrue(s)
1309

    
1310
    def test_003a_setup_interface_A(self):
1311
        """Set up eth1 for server A"""
1312

    
1313
        self._skipIf(self.is_windows, "only valid for Linux servers")
1314

    
1315
        log.info("Setting up interface eth1 for server A")
1316
        self.result_dict.clear()
1317

    
1318
        server = self.client.get_server_details(self.serverid['A'])
1319
        image = self.client.get_image_details(self.imageid)
1320
        os_value = image['metadata']['values']['os']
1321

    
1322
        users = image["metadata"]["values"].get("users", None)
1323
        userlist = users.split()
1324

    
1325
        if "root" in userlist:
1326
            loginname = "root"
1327
        elif users is None:
1328
            loginname = self._connect_loginname(os_value)
1329
        else:
1330
            loginname = choice(userlist)
1331

    
1332
        hostip = self._get_ipv4(server)
1333
        myPass = self.password['A']
1334

    
1335
        log.info("SSH in server A as %s/%s" % (loginname, myPass))
1336
        command = "ifconfig eth1 %s" % self.priv_ip["A"]
1337
        output, status = _ssh_execute(
1338
            hostip, loginname, myPass, command)
1339

    
1340
        self.assertEquals(status, 0)
1341

    
1342
    def test_003b_setup_interface_B(self):
1343
        """Setup eth1 for server B"""
1344

    
1345
        self._skipIf(self.is_windows, "only valid for Linux servers")
1346

    
1347
        log.info("Setting up interface eth1 for server B")
1348

    
1349
        server = self.client.get_server_details(self.serverid['B'])
1350
        image = self.client.get_image_details(self.imageid)
1351
        os_value = image['metadata']['values']['os']
1352

    
1353
        users = image["metadata"]["values"].get("users", None)
1354
        userlist = users.split()
1355

    
1356
        if "root" in userlist:
1357
            loginname = "root"
1358
        elif users is None:
1359
            loginname = self._connect_loginname(os_value)
1360
        else:
1361
            loginname = choice(userlist)
1362

    
1363
        hostip = self._get_ipv4(server)
1364
        myPass = self.password['B']
1365

    
1366
        log.info("SSH in server B as %s/%s" % (loginname, myPass))
1367
        command = "ifconfig eth1 %s" % self.priv_ip["B"]
1368
        output, status = _ssh_execute(
1369
            hostip, loginname, myPass, command)
1370

    
1371
        self.assertEquals(status, 0)
1372

    
1373
    def test_003c_test_connection_exists(self):
1374
        """Ping server B from server A to test if connection exists"""
1375

    
1376
        self._skipIf(self.is_windows, "only valid for Linux servers")
1377

    
1378
        log.info("Testing if server A is actually connected to server B")
1379

    
1380
        server = self.client.get_server_details(self.serverid['A'])
1381
        image = self.client.get_image_details(self.imageid)
1382
        os_value = image['metadata']['values']['os']
1383
        hostip = self._get_ipv4(server)
1384

    
1385
        users = image["metadata"]["values"].get("users", None)
1386
        userlist = users.split()
1387

    
1388
        if "root" in userlist:
1389
            loginname = "root"
1390
        elif users is None:
1391
            loginname = self._connect_loginname(os_value)
1392
        else:
1393
            loginname = choice(userlist)
1394

    
1395
        myPass = self.password['A']
1396

    
1397
        cmd = "if ping -c 2 -w 3 %s >/dev/null; \
1398
               then echo \'True\'; fi;" % self.priv_ip["B"]
1399
        lines, status = _ssh_execute(
1400
            hostip, loginname, myPass, cmd)
1401

    
1402
        exists = False
1403

    
1404
        if 'True\n' in lines:
1405
            exists = True
1406

    
1407
        self.assertTrue(exists)
1408

    
1409
    def test_004_disconnect_from_network(self):
1410
        "Disconnecting server A and B from network"
1411

    
1412
        log.info("Disconnecting servers from private network")
1413

    
1414
        prev_state = self.client.get_network_details(self.networkid)
1415
        prev_nics = prev_state['attachments']['values']
1416
        #prev_conn = len(prev_nics)
1417

    
1418
        nicsA = [x['id']
1419
                 for x in self.client.get_server_details(
1420
                     self.serverid['A'])['attachments']['values']]
1421
        nicsB = [x['id']
1422
                 for x in self.client.get_server_details(
1423
                     self.serverid['B'])['attachments']['values']]
1424

    
1425
        for nic in prev_nics:
1426
            if nic in nicsA:
1427
                self.client.disconnect_server(self.serverid['A'], nic)
1428
            if nic in nicsB:
1429
                self.client.disconnect_server(self.serverid['B'], nic)
1430

    
1431
        #Insist on deleting until action timeout
1432
        fail_tmout = time.time() + self.action_timeout
1433

    
1434
        while True:
1435
            netsA = [x['network_id']
1436
                     for x in self.client.get_server_details(
1437
                         self.serverid['A'])['attachments']['values']]
1438
            netsB = [x['network_id']
1439
                     for x in self.client.get_server_details(
1440
                         self.serverid['B'])['attachments']['values']]
1441

    
1442
            #connected = (self.client.get_network_details(self.networkid))
1443
            #connections = connected['attachments']['values']
1444
            if (self.networkid not in netsA) and (self.networkid not in netsB):
1445
                conn_exists = False
1446
                break
1447
            elif time.time() > fail_tmout:
1448
                self.assertLess(time.time(), fail_tmout)
1449
            else:
1450
                time.sleep(self.query_interval)
1451

    
1452
        self.assertFalse(conn_exists)
1453

    
1454
    def test_005_destroy_network(self):
1455
        """Test submit delete network request"""
1456

    
1457
        log.info("Submitting delete network request")
1458

    
1459
        self.client.delete_network(self.networkid)
1460

    
1461
        fail_tmout = time.time() + self.action_timeout
1462

    
1463
        while True:
1464

    
1465
            curr_net = []
1466
            networks = self.client.list_networks()
1467

    
1468
            for net in networks:
1469
                curr_net.append(net['id'])
1470

    
1471
            if self.networkid not in curr_net:
1472
                self.assertTrue(self.networkid not in curr_net)
1473
                break
1474

    
1475
            elif time.time() > fail_tmout:
1476
                self.assertLess(time.time(), fail_tmout)
1477

    
1478
            else:
1479
                time.sleep(self.query_interval)
1480

    
1481
    def test_006_cleanup_servers(self):
1482
        """Cleanup servers created for this test"""
1483

    
1484
        log.info("Delete servers created for this test")
1485

    
1486
        self.compute.delete_server(self.serverid['A'])
1487
        self.compute.delete_server(self.serverid['B'])
1488

    
1489
        fail_tmout = time.time() + self.action_timeout
1490

    
1491
        #Ensure server gets deleted
1492
        status = dict()
1493

    
1494
        while True:
1495
            details = self.compute.get_server_details(self.serverid['A'])
1496
            status['A'] = details['status']
1497
            details = self.compute.get_server_details(self.serverid['B'])
1498
            status['B'] = details['status']
1499
            if (status['A'] == 'DELETED') and (status['B'] == 'DELETED'):
1500
                deleted = True
1501
                break
1502
            elif time.time() > fail_tmout:
1503
                self.assertLess(time.time(), fail_tmout)
1504
            else:
1505
                time.sleep(self.query_interval)
1506

    
1507
        self.assertTrue(deleted)
1508

    
1509

    
1510
class TestRunnerProcess(Process):
1511
    """A distinct process used to execute part of the tests in parallel"""
1512
    def __init__(self, **kw):
1513
        Process.__init__(self, **kw)
1514
        kwargs = kw["kwargs"]
1515
        self.testq = kwargs["testq"]
1516
        self.worker_folder = kwargs["worker_folder"]
1517

    
1518
    def run(self):
1519
        # Make sure this test runner process dies with the parent
1520
        # and is not left behind.
1521
        #
1522
        # WARNING: This uses the prctl(2) call and is
1523
        # Linux-specific.
1524

    
1525
        prctl.set_pdeathsig(signal.SIGHUP)
1526

    
1527
        multi = logging.getLogger("multiprocess")
1528

    
1529
        while True:
1530
            multi.debug("I am process %d, GETting from queue is %s" %
1531
                        (os.getpid(), self.testq))
1532
            msg = self.testq.get()
1533

    
1534
            multi.debug("Dequeued msg: %s" % msg)
1535

    
1536
            if msg == "TEST_RUNNER_TERMINATE":
1537
                raise SystemExit
1538

    
1539
            elif issubclass(msg, unittest.TestCase):
1540
                # Assemble a TestSuite, and run it
1541

    
1542
                log_file = os.path.join(self.worker_folder, 'details_' +
1543
                                        (msg.__name__) + "_" +
1544
                                        TEST_RUN_ID + '.log')
1545

    
1546
                fail_file = os.path.join(self.worker_folder, 'failed_' +
1547
                                         (msg.__name__) + "_" +
1548
                                         TEST_RUN_ID + '.log')
1549
                error_file = os.path.join(self.worker_folder, 'error_' +
1550
                                          (msg.__name__) + "_" +
1551
                                          TEST_RUN_ID + '.log')
1552

    
1553
                f = open(log_file, 'w')
1554
                fail = open(fail_file, 'w')
1555
                error = open(error_file, 'w')
1556

    
1557
                log.info(yellow + '* Starting testcase: %s' % msg + normal)
1558

    
1559
                runner = unittest.TextTestRunner(
1560
                    f, verbosity=2, failfast=True,
1561
                    resultclass=BurninTestResult)
1562
                suite = unittest.TestLoader().loadTestsFromTestCase(msg)
1563
                result = runner.run(suite)
1564

    
1565
                for res in result.errors:
1566
                    log.error("snf-burnin encountered an error in "
1567
                              "testcase: %s" % msg)
1568
                    log.error("See log for details")
1569
                    error.write(str(res[0]) + '\n')
1570
                    error.write(str(res[0].shortDescription()) + '\n')
1571
                    error.write('\n')
1572

    
1573
                for res in result.failures:
1574
                    log.error("snf-burnin failed in testcase: %s" % msg)
1575
                    log.error("See log for details")
1576
                    fail.write(str(res[0]) + '\n')
1577
                    fail.write(str(res[0].shortDescription()) + '\n')
1578
                    fail.write('\n')
1579
                    if not NOFAILFAST:
1580
                        sys.exit()
1581

    
1582
                if (len(result.failures) == 0) and (len(result.errors) == 0):
1583
                    log.debug("Passed testcase: %s" % msg)
1584

    
1585
                f.close()
1586
                fail.close()
1587
                error.close()
1588

    
1589
            else:
1590
                raise Exception("Cannot handle msg: %s" % msg)
1591

    
1592

    
1593
def _run_cases_in_series(cases, image_folder):
1594
    """Run instances of TestCase in series"""
1595

    
1596
    for case in cases:
1597

    
1598
        test = case.__name__
1599

    
1600
        log.info(yellow + '* Starting testcase: %s' % test + normal)
1601
        log_file = os.path.join(image_folder, 'details_' +
1602
                                (case.__name__) + "_" +
1603
                                TEST_RUN_ID + '.log')
1604
        fail_file = os.path.join(image_folder, 'failed_' +
1605
                                 (case.__name__) + "_" +
1606
                                 TEST_RUN_ID + '.log')
1607
        error_file = os.path.join(image_folder, 'error_' +
1608
                                  (case.__name__) + "_" +
1609
                                  TEST_RUN_ID + '.log')
1610

    
1611
        f = open(log_file, "w")
1612
        fail = open(fail_file, "w")
1613
        error = open(error_file, "w")
1614

    
1615
        suite = unittest.TestLoader().loadTestsFromTestCase(case)
1616
        runner = unittest.TextTestRunner(
1617
            f, verbosity=2, failfast=True,
1618
            resultclass=BurninTestResult)
1619
        result = runner.run(suite)
1620

    
1621
        for res in result.errors:
1622
            log.error("snf-burnin encountered an error in "
1623
                      "testcase: %s" % test)
1624
            log.error("See log for details")
1625
            error.write(str(res[0]) + '\n')
1626
            error.write(str(res[0].shortDescription()) + '\n')
1627
            error.write('\n')
1628

    
1629
        for res in result.failures:
1630
            log.error("snf-burnin failed in testcase: %s" % test)
1631
            log.error("See log for details")
1632
            fail.write(str(res[0]) + '\n')
1633
            fail.write(str(res[0].shortDescription()) + '\n')
1634
            fail.write('\n')
1635
            if not NOFAILFAST:
1636
                sys.exit()
1637

    
1638
        if (len(result.failures) == 0) and (len(result.errors) == 0):
1639
            log.debug("Passed testcase: %s" % test)
1640

    
1641

    
1642
def _run_cases_in_parallel(cases, fanout, image_folder):
1643
    """Run instances of TestCase in parallel, in a number of distinct processes
1644

1645
    The cases iterable specifies the TestCases to be executed in parallel,
1646
    by test runners running in distinct processes.
1647
    The fanout parameter specifies the number of processes to spawn,
1648
    and defaults to 1.
1649
    The runner argument specifies the test runner class to use inside each
1650
    runner process.
1651

1652
    """
1653

    
1654
    multi = logging.getLogger("multiprocess")
1655
    handler = logging.StreamHandler()
1656
    multi.addHandler(handler)
1657

    
1658
    if VERBOSE:
1659
        multi.setLevel(logging.DEBUG)
1660
    else:
1661
        multi.setLevel(logging.INFO)
1662

    
1663
    testq = []
1664
    worker_folder = []
1665
    runners = []
1666

    
1667
    for i in xrange(0, fanout):
1668
        testq.append(Queue())
1669
        worker_folder.append(os.path.join(image_folder, 'process'+str(i)))
1670
        os.mkdir(worker_folder[i])
1671

    
1672
    for i in xrange(0, fanout):
1673
        kwargs = dict(testq=testq[i], worker_folder=worker_folder[i])
1674
        runners.append(TestRunnerProcess(kwargs=kwargs))
1675

    
1676
    multi.debug("Spawning %d test runner processes" % len(runners))
1677

    
1678
    for p in runners:
1679
        p.start()
1680

    
1681
    # Enqueue test cases
1682
    for i in xrange(0, fanout):
1683
        map(testq[i].put, cases)
1684
        testq[i].put("TEST_RUNNER_TERMINATE")
1685

    
1686
    multi.debug("Spawned %d test runners, PIDs are %s" %
1687
                (len(runners), [p.pid for p in runners]))
1688

    
1689
    multi.debug("Joining %d processes" % len(runners))
1690

    
1691
    for p in runners:
1692
        p.join()
1693

    
1694
    multi.debug("Done joining %d processes" % len(runners))
1695

    
1696

    
1697
def _images_test_case(**kwargs):
1698
    """Construct a new unit test case class from ImagesTestCase"""
1699
    name = "ImagesTestCase_%s" % kwargs["imageid"]
1700
    cls = type(name, (ImagesTestCase,), kwargs)
1701

    
1702
    #Patch extra parameters into test names by manipulating method docstrings
1703
    for (mname, m) in \
1704
            inspect.getmembers(cls, lambda x: inspect.ismethod(x)):
1705
        if hasattr(m, __doc__):
1706
            m.__func__.__doc__ = "[%s] %s" % (cls.imagename, m.__doc__)
1707

    
1708
    # Make sure the class can be pickled, by listing it among
1709
    # the attributes of __main__. A PicklingError is raised otherwise.
1710
    thismodule = sys.modules[__name__]
1711
    setattr(thismodule, name, cls)
1712
    return cls
1713

    
1714

    
1715
def _spawn_server_test_case(**kwargs):
1716
    """Construct a new unit test case class from SpawnServerTestCase"""
1717

    
1718
    name = "SpawnServerTestCase_%s" % kwargs["imageid"]
1719
    cls = type(name, (SpawnServerTestCase,), kwargs)
1720

    
1721
    # Patch extra parameters into test names by manipulating method docstrings
1722
    for (mname, m) in \
1723
            inspect.getmembers(cls, lambda x: inspect.ismethod(x)):
1724
        if hasattr(m, __doc__):
1725
            m.__func__.__doc__ = "[%s] %s" % (cls.imagename, m.__doc__)
1726

    
1727
    # Make sure the class can be pickled, by listing it among
1728
    # the attributes of __main__. A PicklingError is raised otherwise.
1729

    
1730
    thismodule = sys.modules[__name__]
1731
    setattr(thismodule, name, cls)
1732
    return cls
1733

    
1734

    
1735
def _spawn_network_test_case(**kwargs):
1736
    """Construct a new unit test case class from NetworkTestCase"""
1737

    
1738
    name = "NetworkTestCase" + TEST_RUN_ID
1739
    cls = type(name, (NetworkTestCase,), kwargs)
1740

    
1741
    # Make sure the class can be pickled, by listing it among
1742
    # the attributes of __main__. A PicklingError is raised otherwise.
1743

    
1744
    thismodule = sys.modules[__name__]
1745
    setattr(thismodule, name, cls)
1746
    return cls
1747

    
1748

    
1749
# --------------------------------------------------------------------
1750
# Clean up servers/networks functions
1751
def cleanup_servers(timeout, query_interval, delete_stale=False):
1752

    
1753
    c = ComputeClient(API, TOKEN)
1754

    
1755
    servers = c.list_servers()
1756
    stale = [s for s in servers if s["name"].startswith(SNF_TEST_PREFIX)]
1757

    
1758
    if len(stale) == 0:
1759
        return
1760

    
1761
    # Show staled servers
1762
    print >>sys.stderr, yellow + \
1763
        "Found these stale servers from previous runs:" + \
1764
        normal
1765
    print >>sys.stderr, "    " + \
1766
        "\n    ".join(["%d: %s" % (s["id"], s["name"]) for s in stale])
1767

    
1768
    # Delete staled servers
1769
    if delete_stale:
1770
        print >> sys.stderr, "Deleting %d stale servers:" % len(stale)
1771
        fail_tmout = time.time() + timeout
1772
        for s in stale:
1773
            c.delete_server(s["id"])
1774
        # Wait for all servers to be deleted
1775
        while True:
1776
            servers = c.list_servers()
1777
            stale = [s for s in servers
1778
                     if s["name"].startswith(SNF_TEST_PREFIX)]
1779
            if len(stale) == 0:
1780
                print >> sys.stderr, green + "    ...done" + normal
1781
                break
1782
            elif time.time() > fail_tmout:
1783
                print >> sys.stderr, red + \
1784
                    "Not all stale servers deleted. Action timed out." + \
1785
                    normal
1786
                sys.exit(1)
1787
            else:
1788
                time.sleep(query_interval)
1789
    else:
1790
        print >> sys.stderr, "Use --delete-stale to delete them."
1791

    
1792

    
1793
def cleanup_networks(action_timeout, query_interval, delete_stale=False):
1794

    
1795
    c = CycladesClient(API, TOKEN)
1796

    
1797
    networks = c.list_networks()
1798
    stale = [n for n in networks if n["name"].startswith(SNF_TEST_PREFIX)]
1799

    
1800
    if len(stale) == 0:
1801
        return
1802

    
1803
    # Show staled networks
1804
    print >> sys.stderr, yellow + \
1805
        "Found these stale networks from previous runs:" + \
1806
        normal
1807
    print "    " + \
1808
        "\n    ".join(["%s: %s" % (str(n["id"]), n["name"]) for n in stale])
1809

    
1810
    # Delete staled networks
1811
    if delete_stale:
1812
        print >> sys.stderr, "Deleting %d stale networks:" % len(stale)
1813
        fail_tmout = time.time() + action_timeout
1814
        for n in stale:
1815
            c.delete_network(n["id"])
1816
        # Wait for all networks to be deleted
1817
        while True:
1818
            networks = c.list_networks()
1819
            stale = [n for n in networks
1820
                     if n["name"].startswith(SNF_TEST_PREFIX)]
1821
            if len(stale) == 0:
1822
                print >> sys.stderr, green + "    ...done" + normal
1823
                break
1824
            elif time.time() > fail_tmout:
1825
                print >> sys.stderr, red + \
1826
                    "Not all stale networks deleted. Action timed out." + \
1827
                    normal
1828
                sys.exit(1)
1829
            else:
1830
                time.sleep(query_interval)
1831
    else:
1832
        print >> sys.stderr, "Use --delete-stale to delete them."
1833

    
1834

    
1835
# --------------------------------------------------------------------
1836
# Parse arguments functions
1837
def parse_comma(option, opt, value, parser):
1838
    tests = set(['all', 'auth', 'images', 'flavors',
1839
                 'pithos', 'servers', 'server_spawn',
1840
                 'network_spawn'])
1841
    parse_input = value.split(',')
1842

    
1843
    if not (set(parse_input)).issubset(tests):
1844
        raise OptionValueError("The selected set of tests is invalid")
1845

    
1846
    setattr(parser.values, option.dest, value.split(','))
1847

    
1848

    
1849
def parse_arguments(args):
1850

    
1851
    kw = {}
1852
    kw["usage"] = "%prog [options]"
1853
    kw["description"] = \
1854
        "%prog runs a number of test scenarios on a " \
1855
        "Synnefo deployment."
1856

    
1857
    parser = OptionParser(**kw)
1858
    parser.disable_interspersed_args()
1859

    
1860
    parser.add_option("--api",
1861
                      action="store", type="string", dest="api",
1862
                      help="The API URI to use to reach the Synnefo API",
1863
                      default=None)
1864
    parser.add_option("--plankton",
1865
                      action="store", type="string", dest="plankton",
1866
                      help="The API URI to use to reach the Plankton API",
1867
                      default=None)
1868
    parser.add_option("--plankton-user",
1869
                      action="store", type="string", dest="plankton_user",
1870
                      help="Owner of system images",
1871
                      default=DEFAULT_PLANKTON_USER)
1872
    parser.add_option("--pithos",
1873
                      action="store", type="string", dest="pithos",
1874
                      help="The API URI to use to reach the Pithos API",
1875
                      default=None)
1876
    parser.add_option("--pithos_user",
1877
                      action="store", type="string", dest="pithos_user",
1878
                      help="Owner of the pithos account",
1879
                      default=None)
1880
    parser.add_option("--token",
1881
                      action="store", type="string", dest="token",
1882
                      help="The token to use for authentication to the API")
1883
    parser.add_option("--nofailfast",
1884
                      action="store_true", dest="nofailfast",
1885
                      help="Do not fail immediately if one of the tests "
1886
                           "fails (EXPERIMENTAL)",
1887
                      default=False)
1888
    parser.add_option("--no-ipv6",
1889
                      action="store_true", dest="no_ipv6",
1890
                      help="Disables ipv6 related tests",
1891
                      default=False)
1892
    parser.add_option("--action-timeout",
1893
                      action="store", type="int", dest="action_timeout",
1894
                      metavar="TIMEOUT",
1895
                      help="Wait SECONDS seconds for a server action to "
1896
                           "complete, then the test is considered failed",
1897
                      default=100)
1898
    parser.add_option("--build-warning",
1899
                      action="store", type="int", dest="build_warning",
1900
                      metavar="TIMEOUT",
1901
                      help="Warn if TIMEOUT seconds have passed and a "
1902
                           "build operation is still pending",
1903
                      default=600)
1904
    parser.add_option("--build-fail",
1905
                      action="store", type="int", dest="build_fail",
1906
                      metavar="BUILD_TIMEOUT",
1907
                      help="Fail the test if TIMEOUT seconds have passed "
1908
                           "and a build operation is still incomplete",
1909
                      default=900)
1910
    parser.add_option("--query-interval",
1911
                      action="store", type="int", dest="query_interval",
1912
                      metavar="INTERVAL",
1913
                      help="Query server status when requests are pending "
1914
                           "every INTERVAL seconds",
1915
                      default=3)
1916
    parser.add_option("--fanout",
1917
                      action="store", type="int", dest="fanout",
1918
                      metavar="COUNT",
1919
                      help="Spawn up to COUNT child processes to execute "
1920
                           "in parallel, essentially have up to COUNT "
1921
                           "server build requests outstanding (EXPERIMENTAL)",
1922
                      default=1)
1923
    parser.add_option("--force-flavor",
1924
                      action="store", type="int", dest="force_flavorid",
1925
                      metavar="FLAVOR ID",
1926
                      help="Force all server creations to use the specified "
1927
                           "FLAVOR ID instead of a randomly chosen one, "
1928
                           "useful if disk space is scarce",
1929
                      default=None)
1930
    parser.add_option("--image-id",
1931
                      action="store", type="string", dest="force_imageid",
1932
                      metavar="IMAGE ID",
1933
                      help="Test the specified image id, use 'all' to test "
1934
                           "all available images (mandatory argument)",
1935
                      default=None)
1936
    parser.add_option("--show-stale",
1937
                      action="store_true", dest="show_stale",
1938
                      help="Show stale servers from previous runs, whose "
1939
                           "name starts with `%s'" % SNF_TEST_PREFIX,
1940
                      default=False)
1941
    parser.add_option("--delete-stale",
1942
                      action="store_true", dest="delete_stale",
1943
                      help="Delete stale servers from previous runs, whose "
1944
                           "name starts with `%s'" % SNF_TEST_PREFIX,
1945
                      default=False)
1946
    parser.add_option("--force-personality",
1947
                      action="store", type="string", dest="personality_path",
1948
                      help="Force a personality file injection.\
1949
                            File path required. ",
1950
                      default=None)
1951
    parser.add_option("--log-folder",
1952
                      action="store", type="string", dest="log_folder",
1953
                      help="Define the absolute path where the output \
1954
                            log is stored. ",
1955
                      default="/var/log/burnin/")
1956
    parser.add_option("--verbose", "-V",
1957
                      action="store_true", dest="verbose",
1958
                      help="Print detailed output about multiple "
1959
                           "processes spawning",
1960
                      default=False)
1961
    parser.add_option("--set-tests",
1962
                      action="callback",
1963
                      dest="tests",
1964
                      type="string",
1965
                      help='Set comma seperated tests for this run. \
1966
                            Available tests: auth, images, flavors, \
1967
                                             servers, server_spawn, \
1968
                                             network_spawn, pithos. \
1969
                            Default = all',
1970
                      default='all',
1971
                      callback=parse_comma)
1972

    
1973
    (opts, args) = parser.parse_args(args)
1974

    
1975
    # -----------------------
1976
    # Verify arguments
1977

    
1978
    # `delete_stale' implies `show_stale'
1979
    if opts.delete_stale:
1980
        opts.show_stale = True
1981

    
1982
    # `token' is mandatory
1983
    _mandatory_argument(opts.token, "--token")
1984
    # `api' is mandatory
1985
    _mandatory_argument(opts.api, "--api")
1986

    
1987
    if not opts.show_stale:
1988
        # `image-id' is mandatory
1989
        _mandatory_argument(opts.force_imageid, "--image_id")
1990
        if opts.force_imageid != 'all':
1991
            try:
1992
                opts.force_imageid = str(opts.force_imageid)
1993
            except ValueError:
1994
                print >>sys.stderr, red + \
1995
                    "Invalid value specified for" + \
1996
                    "--image-id. Use a valid id, or `all'." + \
1997
                    normal
1998
                sys.exit(1)
1999
        # `pithos' is mandatory
2000
        _mandatory_argument(opts.pithos, "--pithos")
2001
        # `pithos_user' is mandatory
2002
        _mandatory_argument(opts.pithos_user, "--pithos_user")
2003
        # `plankton' is mandatory
2004
        _mandatory_argument(opts.plankton, "--plankton")
2005

    
2006
    return (opts, args)
2007

    
2008

    
2009
def _mandatory_argument(Arg, Str):
2010
    if not Arg:
2011
        print >>sys.stderr, red + \
2012
            "The " + Str + " argument is mandatory.\n" + \
2013
            normal
2014
        sys.exit(1)
2015

    
2016

    
2017
# --------------------------------------------------------------------
2018
# Burnin main function
2019
def main():
2020
    """Assemble test cases into a test suite, and run it
2021

2022
    IMPORTANT: Tests have dependencies and have to be run in the specified
2023
    order inside a single test case. They communicate through attributes of the
2024
    corresponding TestCase class (shared fixtures). Distinct subclasses of
2025
    TestCase MAY SHARE NO DATA, since they are run in parallel, in distinct
2026
    test runner processes.
2027

2028
    """
2029

    
2030
    # Parse arguments using `optparse'
2031
    (opts, args) = parse_arguments(sys.argv[1:])
2032

    
2033
    # Some global variables
2034
    global API, TOKEN, PLANKTON, PLANKTON_USER
2035
    global PITHOS, PITHOS_USER, NO_IPV6, VERBOSE, NOFAILFAST
2036
    API = opts.api
2037
    TOKEN = opts.token
2038
    PLANKTON = opts.plankton
2039
    PLANKTON_USER = opts.plankton_user
2040
    PITHOS = opts.pithos
2041
    PITHOS_USER = opts.pithos_user
2042
    NO_IPV6 = opts.no_ipv6
2043
    VERBOSE = opts.verbose
2044
    NOFAILFAST = opts.nofailfast
2045

    
2046
    # If `show_stale', cleanup stale servers
2047
    # from previous runs and exit
2048
    if opts.show_stale:
2049
        # We must clean the servers first
2050
        cleanup_servers(opts.action_timeout, opts.query_interval,
2051
                        delete_stale=opts.delete_stale)
2052
        cleanup_networks(opts.action_timeout, opts.query_interval,
2053
                         delete_stale=opts.delete_stale)
2054
        return 0
2055

    
2056
    # Initialize a kamaki instance, get flavors, images
2057
    c = ComputeClient(API, TOKEN)
2058
    DIMAGES = c.list_images(detail=True)
2059
    DFLAVORS = c.list_flavors(detail=True)
2060

    
2061
    # FIXME: logging, log, LOG PID, TEST_RUN_ID, arguments
2062
    # Run them: FIXME: In parallel, FAILEARLY, catchbreak?
2063
    #unittest.main(verbosity=2, catchbreak=True)
2064

    
2065
    # Get a list of images we are going to test
2066
    if opts.force_imageid == 'all':
2067
        test_images = DIMAGES
2068
    else:
2069
        test_images = filter(lambda x: x["id"] == opts.force_imageid, DIMAGES)
2070

    
2071
    # Create output (logging) folder
2072
    if not os.path.exists(opts.log_folder):
2073
        os.mkdir(opts.log_folder)
2074
    test_folder = os.path.join(opts.log_folder, TEST_RUN_ID)
2075
    os.mkdir(test_folder)
2076

    
2077
    for image in test_images:
2078
        imageid = str(image["id"])
2079
        imagename = image["name"]
2080
        # Choose a flavor (given from user or random)
2081
        if opts.force_flavorid:
2082
            flavorid = opts.force_flavorid
2083
        else:
2084
            flavorid = choice([f["id"] for f in DFLAVORS if f["disk"] >= 20])
2085
        # Personality dictionary for file injection test
2086
        if opts.personality_path is not None:
2087
            f = open(opts.personality_path)
2088
            content = b64encode(f.read())
2089
            personality = []
2090
            st = os.stat(opts.personality_path)
2091
            personality.append({
2092
                'path': '/root/test_inj_file',
2093
                'owner': 'root',
2094
                'group': 'root',
2095
                'mode': 0x7777 & st.st_mode,
2096
                'contents': content})
2097
        else:
2098
            personality = None
2099
        # Give a name to our test servers
2100
        servername = "%s%s for %s" % (SNF_TEST_PREFIX, TEST_RUN_ID, imagename)
2101
        is_windows = imagename.lower().find("windows") >= 0
2102

    
2103
        # Create Server TestCases
2104
        ServerTestCase = _spawn_server_test_case(
2105
            imageid=imageid,
2106
            flavorid=flavorid,
2107
            imagename=imagename,
2108
            personality=personality,
2109
            servername=servername,
2110
            is_windows=is_windows,
2111
            action_timeout=opts.action_timeout,
2112
            build_warning=opts.build_warning,
2113
            build_fail=opts.build_fail,
2114
            query_interval=opts.query_interval)
2115
        # Create Network TestCases
2116
        NetworkTestCase = _spawn_network_test_case(
2117
            action_timeout=opts.action_timeout,
2118
            imageid=imageid,
2119
            flavorid=flavorid,
2120
            imagename=imagename,
2121
            query_interval=opts.query_interval)
2122
        # Create Images TestCase
2123
        CImagesTestCase = _images_test_case(
2124
            action_timeout=opts.action_timeout,
2125
            imageid=imageid,
2126
            flavorid=flavorid,
2127
            imagename=imagename,
2128
            query_interval=opts.query_interval)
2129

    
2130
        # Choose the tests we are going to run
2131
        test_dict = {'auth': UnauthorizedTestCase,
2132
                     'images': CImagesTestCase,
2133
                     'flavors': FlavorsTestCase,
2134
                     'servers': ServersTestCase,
2135
                     'pithos': PithosTestCase,
2136
                     'server_spawn': ServerTestCase,
2137
                     'network_spawn': NetworkTestCase}
2138
        seq_cases = []
2139
        if 'all' in opts.tests:
2140
            seq_cases = [UnauthorizedTestCase, CImagesTestCase,
2141
                         FlavorsTestCase, ServersTestCase,
2142
                         PithosTestCase, ServerTestCase,
2143
                         NetworkTestCase]
2144
        else:
2145
            for test in opts.tests:
2146
                seq_cases.append(test_dict[test])
2147

    
2148
        # Folder for each image
2149
        image_folder = os.path.join(test_folder, imageid)
2150
        os.mkdir(image_folder)
2151

    
2152
        # Run each test
2153
        if opts.fanout > 1:
2154
            _run_cases_in_parallel(seq_cases, opts.fanout, image_folder)
2155
        else:
2156
            _run_cases_in_series(seq_cases, image_folder)
2157

    
2158

    
2159
# --------------------------------------------------------------------
2160
# Call main
2161
if __name__ == "__main__":
2162
    sys.exit(main())