Statistics
| Branch: | Tag: | Revision:

root / snf-tools / synnefo_tools / burnin.py @ d9647bf1

History | View | Annotate | Download (80.6 kB)

1
#!/usr/bin/env python
2

    
3
# Copyright 2011 GRNET S.A. All rights reserved.
4
#
5
# Redistribution and use in source and binary forms, with or
6
# without modification, are permitted provided that the following
7
# conditions are met:
8
#
9
#   1. Redistributions of source code must retain the above
10
#      copyright notice, this list of conditions and the following
11
#      disclaimer.
12
#
13
#   2. Redistributions in binary form must reproduce the above
14
#      copyright notice, this list of conditions and the following
15
#      disclaimer in the documentation and/or other materials
16
#      provided with the distribution.
17
#
18
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
19
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
20
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
21
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
22
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
25
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
26
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
27
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
28
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29
# POSSIBILITY OF SUCH DAMAGE.
30
#
31
# The views and conclusions contained in the software and
32
# documentation are those of the authors and should not be
33
# interpreted as representing official policies, either expressed
34
# or implied, of GRNET S.A.
35

    
36
"""Perform integration testing on a running Synnefo deployment"""
37

    
38
#import __main__
39
import datetime
40
import inspect
41
import logging
42
import os
43
import os.path
44
import paramiko
45
import prctl
46
import subprocess
47
import signal
48
import socket
49
import sys
50
import time
51
import tempfile
52
from base64 import b64encode
53
from IPy import IP
54
from multiprocessing import Process, Queue
55
from random import choice, randint
56
from optparse import OptionParser, OptionValueError
57

    
58
from kamaki.clients.compute import ComputeClient
59
from kamaki.clients.cyclades import CycladesClient
60
from kamaki.clients.image import ImageClient
61
from kamaki.clients.pithos import PithosClient
62
from kamaki.clients.astakos import AstakosClient
63
from kamaki.clients import ClientError
64

    
65
from vncauthproxy.d3des import generate_response as d3des_generate_response
66

    
67
# Use backported unittest functionality if Python < 2.7
68
try:
69
    import unittest2 as unittest
70
except ImportError:
71
    if sys.version_info < (2, 7):
72
        raise Exception("The unittest2 package is required for Python < 2.7")
73
    import unittest
74

    
75
# --------------------------------------------------------------------
76
# Global Variables
77
AUTH_URL = None
78
TOKEN = None
79
PLANKTON_USER = None
80
NO_IPV6 = None
81
DEFAULT_PLANKTON_USER = "images@okeanos.grnet.gr"
82
NOFAILFAST = None
83
VERBOSE = None
84

    
85
# A unique id identifying this test run
86
TEST_RUN_ID = datetime.datetime.strftime(datetime.datetime.now(),
87
                                         "%Y%m%d%H%M%S")
88
SNF_TEST_PREFIX = "snf-test-"
89

    
90
red = '\x1b[31m'
91
yellow = '\x1b[33m'
92
green = '\x1b[32m'
93
normal = '\x1b[0m'
94

    
95

    
96
# --------------------------------------------------------------------
97
# Global functions
98
def _ssh_execute(hostip, username, password, command):
99
    """Execute a command via ssh"""
100
    ssh = paramiko.SSHClient()
101
    ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
102
    try:
103
        ssh.connect(hostip, username=username, password=password)
104
    except socket.error, err:
105
        raise AssertionError(err)
106
    try:
107
        stdin, stdout, stderr = ssh.exec_command(command)
108
    except paramiko.SSHException, err:
109
        raise AssertionError(err)
110
    status = stdout.channel.recv_exit_status()
111
    output = stdout.readlines()
112
    ssh.close()
113
    return output, status
114

    
115

    
116
def _get_user_id():
117
    """Authenticate to astakos and get unique users id"""
118
    astakos = AstakosClient(AUTH_URL, TOKEN)
119
    authenticate = astakos.authenticate()
120
    return authenticate['access']['user']['id']
121

    
122

    
123
# --------------------------------------------------------------------
124
# BurninTestReulst class
125
class BurninTestResult(unittest.TextTestResult):
126
    def addSuccess(self, test):
127
        super(BurninTestResult, self).addSuccess(test)
128
        if self.showAll:
129
            if hasattr(test, 'result_dict'):
130
                run_details = test.result_dict
131

    
132
                self.stream.write("\n")
133
                for i in run_details:
134
                    self.stream.write("%s : %s \n" % (i, run_details[i]))
135
                self.stream.write("\n")
136

    
137
        elif self.dots:
138
            self.stream.write('.')
139
            self.stream.flush()
140

    
141
    def addError(self, test, err):
142
        super(BurninTestResult, self).addError(test, err)
143
        if self.showAll:
144
            self.stream.writeln("ERROR")
145
            if hasattr(test, 'result_dict'):
146
                run_details = test.result_dict
147

    
148
                self.stream.write("\n")
149
                for i in run_details:
150
                    self.stream.write("%s : %s \n" % (i, run_details[i]))
151
                self.stream.write("\n")
152

    
153
        elif self.dots:
154
            self.stream.write('E')
155
            self.stream.flush()
156

    
157
    def addFailure(self, test, err):
158
        super(BurninTestResult, self).addFailure(test, err)
159
        if self.showAll:
160
            self.stream.writeln("FAIL")
161
            if hasattr(test, 'result_dict'):
162
                run_details = test.result_dict
163

    
164
                self.stream.write("\n")
165
                for i in run_details:
166
                    self.stream.write("%s : %s \n" % (i, run_details[i]))
167
                self.stream.write("\n")
168

    
169
        elif self.dots:
170
            self.stream.write('F')
171
            self.stream.flush()
172

    
173

    
174
# --------------------------------------------------------------------
175
# Format Results
176
class burninFormatter(logging.Formatter):
177
    err_fmt = red + "ERROR: %(msg)s" + normal
178
    dbg_fmt = green + "* %(msg)s" + normal
179
    info_fmt = "%(msg)s"
180

    
181
    def __init__(self, fmt="%(levelno)s: %(msg)s"):
182
        logging.Formatter.__init__(self, fmt)
183

    
184
    def format(self, record):
185
        format_orig = self._fmt
186
        # Replace the original format with one customized by logging level
187
        if record.levelno == 10:    # DEBUG
188
            self._fmt = burninFormatter.dbg_fmt
189
        elif record.levelno == 20:  # INFO
190
            self._fmt = burninFormatter.info_fmt
191
        elif record.levelno == 40:  # ERROR
192
            self._fmt = burninFormatter.err_fmt
193
        result = logging.Formatter.format(self, record)
194
        self._fmt = format_orig
195
        return result
196

    
197
log = logging.getLogger("burnin")
198
log.setLevel(logging.DEBUG)
199
handler = logging.StreamHandler()
200
handler.setFormatter(burninFormatter())
201
log.addHandler(handler)
202

    
203

    
204
# --------------------------------------------------------------------
205
# UnauthorizedTestCase class
206
class UnauthorizedTestCase(unittest.TestCase):
207
    """Test unauthorized access"""
208
    @classmethod
209
    def setUpClass(cls):
210
        cls.astakos = AstakosClient(AUTH_URL, TOKEN)
211
        cls.compute_url = \
212
            cls.astakos.get_service_endpoints('compute')['publicURL']
213
        cls.result_dict = dict()
214

    
215
    def test_unauthorized_access(self):
216
        """Test access without a valid token fails"""
217
        log.info("Authentication test")
218
        falseToken = '12345'
219
        c = ComputeClient(self.compute_url, falseToken)
220

    
221
        with self.assertRaises(ClientError) as cm:
222
            c.list_servers()
223
            self.assertEqual(cm.exception.status, 401)
224

    
225

    
226
# --------------------------------------------------------------------
227
# This class gest replicated into Images TestCases dynamically
228
class ImagesTestCase(unittest.TestCase):
229
    """Test image lists for consistency"""
230
    @classmethod
231
    def setUpClass(cls):
232
        """Initialize kamaki, get (detailed) list of images"""
233
        log.info("Getting simple and detailed list of images")
234
        cls.astakos_client = AstakosClient(AUTH_URL, TOKEN)
235
        # Compute Client
236
        compute_url = \
237
            cls.astakos_client.get_service_endpoints('compute')['publicURL']
238
        cls.compute_client = ComputeClient(compute_url, TOKEN)
239
        # Image Client
240
        image_url = \
241
            cls.astakos_client.get_service_endpoints('image')['publicURL']
242
        cls.image_client = ImageClient(image_url, TOKEN)
243
        # Pithos Client
244
        pithos_url = cls.astakos_client.\
245
            get_service_endpoints('object-store')['publicURL']
246
        cls.pithos_client = PithosClient(pithos_url, TOKEN)
247

    
248
        # Get images
249
        cls.images = \
250
            filter(lambda x: not x['name'].startswith(SNF_TEST_PREFIX),
251
                   cls.image_client.list_public())
252
        cls.dimages = \
253
            filter(lambda x: not x['name'].startswith(SNF_TEST_PREFIX),
254
                   cls.image_client.list_public(detail=True))
255
        cls.result_dict = dict()
256
        # Get uniq user id
257
        cls.uuid = _get_user_id()
258
        log.info("Uniq user id = %s" % cls.uuid)
259
        # Create temp directory and store it inside our class
260
        # XXX: In my machine /tmp has not enough space
261
        #      so use current directory to be sure.
262
        cls.temp_dir = tempfile.mkdtemp(dir=os.getcwd())
263
        cls.temp_image_name = \
264
            SNF_TEST_PREFIX + cls.imageid + ".diskdump"
265

    
266
    @classmethod
267
    def tearDownClass(cls):
268
        """Remove local files"""
269
        try:
270
            temp_file = os.path.join(cls.temp_dir, cls.temp_image_name)
271
            os.unlink(temp_file)
272
        except:
273
            pass
274
        try:
275
            os.rmdir(cls.temp_dir)
276
        except:
277
            pass
278

    
279
    def test_001_list_images(self):
280
        """Test image list actually returns images"""
281
        self.assertGreater(len(self.images), 0)
282

    
283
    def test_002_list_images_detailed(self):
284
        """Test detailed image list is the same length as list"""
285
        self.assertEqual(len(self.dimages), len(self.images))
286

    
287
    def test_003_same_image_names(self):
288
        """Test detailed and simple image list contain same names"""
289
        names = sorted(map(lambda x: x["name"], self.images))
290
        dnames = sorted(map(lambda x: x["name"], self.dimages))
291
        self.assertEqual(names, dnames)
292

    
293
# XXX: Find a way to resolve owner's uuid to username.
294
#      (maybe use astakosclient)
295
#    def test_004_unique_image_names(self):
296
#        """Test system images have unique names"""
297
#        sys_images = filter(lambda x: x['owner'] == PLANKTON_USER,
298
#                            self.dimages)
299
#        names = sorted(map(lambda x: x["name"], sys_images))
300
#        self.assertEqual(sorted(list(set(names))), names)
301

    
302
    def test_005_image_metadata(self):
303
        """Test every image has specific metadata defined"""
304
        keys = frozenset(["osfamily", "root_partition"])
305
        details = self.compute_client.list_images(detail=True)
306
        for i in details:
307
            self.assertTrue(keys.issubset(i["metadata"].keys()))
308

    
309
    def test_006_download_image(self):
310
        """Download image from pithos+"""
311
        # Get image location
312
        image = filter(
313
            lambda x: x['id'] == self.imageid, self.dimages)[0]
314
        image_location = \
315
            image['location'].replace("://", " ").replace("/", " ").split()
316
        log.info("Download image, with owner %s\n\tcontainer %s, and name %s"
317
                 % (image_location[1], image_location[2], image_location[3]))
318
        self.pithos_client.account = image_location[1]
319
        self.pithos_client.container = image_location[2]
320
        temp_file = os.path.join(self.temp_dir, self.temp_image_name)
321
        with open(temp_file, "wb+") as f:
322
            self.pithos_client.download_object(image_location[3], f)
323

    
324
    def test_007_upload_image(self):
325
        """Upload and register image"""
326
        temp_file = os.path.join(self.temp_dir, self.temp_image_name)
327
        log.info("Upload image to pithos+")
328
        # Create container `images'
329
        self.pithos_client.account = self.uuid
330
        self.pithos_client.container = "images"
331
        self.pithos_client.container_put()
332
        with open(temp_file, "rb+") as f:
333
            self.pithos_client.upload_object(self.temp_image_name, f)
334
        log.info("Register image to plankton")
335
        location = "pithos://" + self.uuid + \
336
            "/images/" + self.temp_image_name
337
        params = {'is_public': True}
338
        properties = {'OSFAMILY': "linux", 'ROOT_PARTITION': 1}
339
        self.image_client.register(
340
            self.temp_image_name, location, params, properties)
341
        # Get image id
342
        details = self.image_client.list_public(detail=True)
343
        detail = filter(lambda x: x['location'] == location, details)
344
        self.assertEqual(len(detail), 1)
345
        cls = type(self)
346
        cls.temp_image_id = detail[0]['id']
347
        log.info("Image registered with id %s" % detail[0]['id'])
348

    
349
    def test_008_cleanup_image(self):
350
        """Cleanup image test"""
351
        log.info("Cleanup image test")
352
        # Remove image from pithos+
353
        self.pithos_client.account = self.uuid
354
        self.pithos_client.container = "images"
355
        self.pithos_client.del_object(self.temp_image_name)
356

    
357

    
358
# --------------------------------------------------------------------
359
# FlavorsTestCase class
360
class FlavorsTestCase(unittest.TestCase):
361
    """Test flavor lists for consistency"""
362
    @classmethod
363
    def setUpClass(cls):
364
        """Initialize kamaki, get (detailed) list of flavors"""
365
        log.info("Getting simple and detailed list of flavors")
366
        cls.astakos_client = AstakosClient(AUTH_URL, TOKEN)
367
        # Compute Client
368
        compute_url = \
369
            cls.astakos_client.get_service_endpoints('compute')['publicURL']
370
        cls.compute_client = ComputeClient(compute_url, TOKEN)
371
        cls.flavors = cls.compute_client.list_flavors()
372
        cls.dflavors = cls.compute_client.list_flavors(detail=True)
373
        cls.result_dict = dict()
374

    
375
    def test_001_list_flavors(self):
376
        """Test flavor list actually returns flavors"""
377
        self.assertGreater(len(self.flavors), 0)
378

    
379
    def test_002_list_flavors_detailed(self):
380
        """Test detailed flavor list is the same length as list"""
381
        self.assertEquals(len(self.dflavors), len(self.flavors))
382

    
383
    def test_003_same_flavor_names(self):
384
        """Test detailed and simple flavor list contain same names"""
385
        names = sorted(map(lambda x: x["name"], self.flavors))
386
        dnames = sorted(map(lambda x: x["name"], self.dflavors))
387
        self.assertEqual(names, dnames)
388

    
389
    def test_004_unique_flavor_names(self):
390
        """Test flavors have unique names"""
391
        names = sorted(map(lambda x: x["name"], self.flavors))
392
        self.assertEqual(sorted(list(set(names))), names)
393

    
394
    def test_005_well_formed_flavor_names(self):
395
        """Test flavors have names of the form CxxRyyDzz
396
        Where xx is vCPU count, yy is RAM in MiB, zz is Disk in GiB
397
        """
398
        for f in self.dflavors:
399
            flavor = (f["vcpus"], f["ram"], f["disk"], f["SNF:disk_template"])
400
            self.assertEqual("C%dR%dD%d%s" % flavor,
401
                             f["name"],
402
                             "Flavor %s does not match its specs." % f["name"])
403

    
404

    
405
# --------------------------------------------------------------------
406
# ServersTestCase class
407
class ServersTestCase(unittest.TestCase):
408
    """Test server lists for consistency"""
409
    @classmethod
410
    def setUpClass(cls):
411
        """Initialize kamaki, get (detailed) list of servers"""
412
        log.info("Getting simple and detailed list of servers")
413

    
414
        cls.astakos_client = AstakosClient(AUTH_URL, TOKEN)
415
        # Compute Client
416
        compute_url = \
417
            cls.astakos_client.get_service_endpoints('compute')['publicURL']
418
        cls.compute_client = ComputeClient(compute_url, TOKEN)
419
        cls.servers = cls.compute_client.list_servers()
420
        cls.dservers = cls.compute_client.list_servers(detail=True)
421
        cls.result_dict = dict()
422

    
423
    # def test_001_list_servers(self):
424
    #     """Test server list actually returns servers"""
425
    #     self.assertGreater(len(self.servers), 0)
426

    
427
    def test_002_list_servers_detailed(self):
428
        """Test detailed server list is the same length as list"""
429
        self.assertEqual(len(self.dservers), len(self.servers))
430

    
431
    def test_003_same_server_names(self):
432
        """Test detailed and simple flavor list contain same names"""
433
        names = sorted(map(lambda x: x["name"], self.servers))
434
        dnames = sorted(map(lambda x: x["name"], self.dservers))
435
        self.assertEqual(names, dnames)
436

    
437

    
438
# --------------------------------------------------------------------
439
# Pithos Test Cases
440
class PithosTestCase(unittest.TestCase):
441
    """Test pithos functionality"""
442
    @classmethod
443
    def setUpClass(cls):
444
        """Initialize kamaki, get list of containers"""
445
        # Get uniq user id
446
        cls.uuid = _get_user_id()
447
        log.info("Uniq user id = %s" % cls.uuid)
448
        log.info("Getting list of containers")
449

    
450
        cls.astakos_client = AstakosClient(AUTH_URL, TOKEN)
451
        # Pithos Client
452
        pithos_url = cls.astakos_client.\
453
            get_service_endpoints('object-store')['publicURL']
454
        cls.pithos_client = PithosClient(pithos_url, TOKEN, cls.uuid)
455

    
456
        cls.containers = cls.pithos_client.list_containers()
457
        cls.result_dict = dict()
458

    
459
    def test_001_list_containers(self):
460
        """Test container list actually returns containers"""
461
        self.assertGreater(len(self.containers), 0)
462

    
463
    def test_002_unique_containers(self):
464
        """Test if containers have unique names"""
465
        names = [n['name'] for n in self.containers]
466
        names = sorted(names)
467
        self.assertEqual(sorted(list(set(names))), names)
468

    
469
    def test_003_create_container(self):
470
        """Test create a container"""
471
        rand_num = randint(1000, 9999)
472
        rand_name = "%s%s" % (SNF_TEST_PREFIX, rand_num)
473
        names = [n['name'] for n in self.containers]
474
        while rand_name in names:
475
            rand_num = randint(1000, 9999)
476
            rand_name = "%s%s" % (SNF_TEST_PREFIX, rand_num)
477
        # Create container
478
        self.pithos_client.container = rand_name
479
        self.pithos_client.container_put()
480
        # Get list of containers
481
        new_containers = self.pithos_client.list_containers()
482
        new_container_names = [n['name'] for n in new_containers]
483
        self.assertIn(rand_name, new_container_names)
484

    
485
    def test_004_upload(self):
486
        """Test uploading something to pithos+"""
487
        # Create a tmp file
488
        with tempfile.TemporaryFile() as f:
489
            f.write("This is a temp file")
490
            f.seek(0, 0)
491
            # Where to save file
492
            self.pithos_client.upload_object("test.txt", f)
493

    
494
    def test_005_download(self):
495
        """Test download something from pithos+"""
496
        # Create tmp directory to save file
497
        tmp_dir = tempfile.mkdtemp()
498
        tmp_file = os.path.join(tmp_dir, "test.txt")
499
        with open(tmp_file, "wb+") as f:
500
            self.pithos_client.download_object("test.txt", f)
501
            # Read file
502
            f.seek(0, 0)
503
            content = f.read()
504
        # Remove files
505
        os.unlink(tmp_file)
506
        os.rmdir(tmp_dir)
507
        # Compare results
508
        self.assertEqual(content, "This is a temp file")
509

    
510
    def test_006_remove(self):
511
        """Test removing files and containers"""
512
        cont_name = self.pithos_client.container
513
        self.pithos_client.del_object("test.txt")
514
        self.pithos_client.purge_container()
515
        # List containers
516
        containers = self.pithos_client.list_containers()
517
        cont_names = [n['name'] for n in containers]
518
        self.assertNotIn(cont_name, cont_names)
519

    
520

    
521
# --------------------------------------------------------------------
522
# This class gets replicated into actual TestCases dynamically
523
class SpawnServerTestCase(unittest.TestCase):
524
    """Test scenario for server of the specified image"""
525
    @classmethod
526
    def setUpClass(cls):
527
        """Initialize a kamaki instance"""
528
        log.info("Spawning server for image `%s'" % cls.imagename)
529

    
530
        cls.astakos_client = AstakosClient(AUTH_URL, TOKEN)
531
        # Cyclades Client
532
        compute_url = \
533
            cls.astakos_client.get_service_endpoints('compute')['publicURL']
534
        cls.cyclades_client = CycladesClient(compute_url, TOKEN)
535

    
536
        cls.result_dict = dict()
537

    
538
    def _get_ipv4(self, server):
539
        """Get the public IPv4 of a server from the detailed server info"""
540

    
541
        nics = server["attachments"]
542

    
543
        for nic in nics:
544
            net_id = nic["network_id"]
545
            if self.cyclades_client.get_network_details(net_id)["public"]:
546
                public_addrs = nic["ipv4"]
547

    
548
        self.assertTrue(public_addrs is not None)
549

    
550
        return public_addrs
551

    
552
    def _get_ipv6(self, server):
553
        """Get the public IPv6 of a server from the detailed server info"""
554

    
555
        nics = server["attachments"]
556

    
557
        for nic in nics:
558
            net_id = nic["network_id"]
559
            if self.cyclades_client.get_network_details(net_id)["public"]:
560
                public_addrs = nic["ipv6"]
561

    
562
        self.assertTrue(public_addrs is not None)
563

    
564
        return public_addrs
565

    
566
    def _connect_loginname(self, os_value):
567
        """Return the login name for connections based on the server OS"""
568
        if os_value in ("Ubuntu", "Kubuntu", "Fedora"):
569
            return "user"
570
        elif os_value in ("windows", "windows_alpha1"):
571
            return "Administrator"
572
        else:
573
            return "root"
574

    
575
    def _verify_server_status(self, current_status, new_status):
576
        """Verify a server has switched to a specified status"""
577
        server = self.cyclades_client.get_server_details(self.serverid)
578
        if server["status"] not in (current_status, new_status):
579
            return None  # Do not raise exception, return so the test fails
580
        self.assertEquals(server["status"], new_status)
581

    
582
    def _get_connected_tcp_socket(self, family, host, port):
583
        """Get a connected socket from the specified family to host:port"""
584
        sock = None
585
        for res in \
586
            socket.getaddrinfo(host, port, family, socket.SOCK_STREAM, 0,
587
                               socket.AI_PASSIVE):
588
            af, socktype, proto, canonname, sa = res
589
            try:
590
                sock = socket.socket(af, socktype, proto)
591
            except socket.error:
592
                sock = None
593
                continue
594
            try:
595
                sock.connect(sa)
596
            except socket.error:
597
                sock.close()
598
                sock = None
599
                continue
600
        self.assertIsNotNone(sock)
601
        return sock
602

    
603
    def _ping_once(self, ipv6, ip):
604
        """Test server responds to a single IPv4 or IPv6 ping"""
605
        cmd = "ping%s -c 2 -w 3 %s" % ("6" if ipv6 else "", ip)
606
        ping = subprocess.Popen(cmd, shell=True,
607
                                stdout=subprocess.PIPE, stderr=subprocess.PIPE)
608
        (stdout, stderr) = ping.communicate()
609
        ret = ping.wait()
610
        self.assertEquals(ret, 0)
611

    
612
    def _get_hostname_over_ssh(self, hostip, username, password):
613
        lines, status = _ssh_execute(
614
            hostip, username, password, "hostname")
615
        self.assertEqual(len(lines), 1)
616
        return lines[0]
617

    
618
    def _try_until_timeout_expires(self, warn_timeout, fail_timeout,
619
                                   opmsg, callable, *args, **kwargs):
620
        if warn_timeout == fail_timeout:
621
            warn_timeout = fail_timeout + 1
622
        warn_tmout = time.time() + warn_timeout
623
        fail_tmout = time.time() + fail_timeout
624
        while True:
625
            self.assertLess(time.time(), fail_tmout,
626
                            "operation `%s' timed out" % opmsg)
627
            if time.time() > warn_tmout:
628
                log.warning("Server %d: `%s' operation `%s' not done yet",
629
                            self.serverid, self.servername, opmsg)
630
            try:
631
                log.info("%s... " % opmsg)
632
                return callable(*args, **kwargs)
633
            except AssertionError:
634
                pass
635
            time.sleep(self.query_interval)
636

    
637
    def _insist_on_tcp_connection(self, family, host, port):
638
        familystr = {socket.AF_INET: "IPv4", socket.AF_INET6: "IPv6",
639
                     socket.AF_UNSPEC: "Unspecified-IPv4/6"}
640
        msg = "connect over %s to %s:%s" % \
641
              (familystr.get(family, "Unknown"), host, port)
642
        sock = self._try_until_timeout_expires(
643
            self.action_timeout, self.action_timeout,
644
            msg, self._get_connected_tcp_socket,
645
            family, host, port)
646
        return sock
647

    
648
    def _insist_on_status_transition(self, current_status, new_status,
649
                                     fail_timeout, warn_timeout=None):
650
        msg = "Server %d: `%s', waiting for %s -> %s" % \
651
              (self.serverid, self.servername, current_status, new_status)
652
        if warn_timeout is None:
653
            warn_timeout = fail_timeout
654
        self._try_until_timeout_expires(warn_timeout, fail_timeout,
655
                                        msg, self._verify_server_status,
656
                                        current_status, new_status)
657
        # Ensure the status is actually the expected one
658
        server = self.cyclades_client.get_server_details(self.serverid)
659
        self.assertEquals(server["status"], new_status)
660

    
661
    def _insist_on_ssh_hostname(self, hostip, username, password):
662
        msg = "SSH to %s, as %s/%s" % (hostip, username, password)
663
        hostname = self._try_until_timeout_expires(
664
            self.action_timeout, self.action_timeout,
665
            msg, self._get_hostname_over_ssh,
666
            hostip, username, password)
667

    
668
        # The hostname must be of the form 'prefix-id'
669
        self.assertTrue(hostname.endswith("-%d\n" % self.serverid))
670

    
671
    def _check_file_through_ssh(self, hostip, username, password,
672
                                remotepath, content):
673
        msg = "Trying file injection through SSH to %s, as %s/%s" % \
674
            (hostip, username, password)
675
        log.info(msg)
676
        try:
677
            ssh = paramiko.SSHClient()
678
            ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
679
            ssh.connect(hostip, username=username, password=password)
680
            ssh.close()
681
        except socket.error, err:
682
            raise AssertionError(err)
683

    
684
        transport = paramiko.Transport((hostip, 22))
685
        transport.connect(username=username, password=password)
686

    
687
        localpath = '/tmp/' + SNF_TEST_PREFIX + 'injection'
688
        sftp = paramiko.SFTPClient.from_transport(transport)
689
        sftp.get(remotepath, localpath)
690
        sftp.close()
691
        transport.close()
692

    
693
        f = open(localpath)
694
        remote_content = b64encode(f.read())
695

    
696
        # Check if files are the same
697
        return (remote_content == content)
698

    
699
    def _skipIf(self, condition, msg):
700
        if condition:
701
            self.skipTest(msg)
702

    
703
    def test_001_submit_create_server(self):
704
        """Test submit create server request"""
705

    
706
        log.info("Submit new server request")
707
        server = self.cyclades_client.create_server(
708
            self.servername, self.flavorid, self.imageid, self.personality)
709

    
710
        log.info("Server id: " + str(server["id"]))
711
        log.info("Server password: " + server["adminPass"])
712
        self.assertEqual(server["name"], self.servername)
713
        self.assertEqual(server["flavor"]["id"], self.flavorid)
714
        self.assertEqual(server["image"]["id"], self.imageid)
715
        self.assertEqual(server["status"], "BUILD")
716

    
717
        # Update class attributes to reflect data on building server
718
        cls = type(self)
719
        cls.serverid = server["id"]
720
        cls.username = None
721
        cls.passwd = server["adminPass"]
722

    
723
        self.result_dict["Server ID"] = str(server["id"])
724
        self.result_dict["Password"] = str(server["adminPass"])
725

    
726
    def test_002a_server_is_building_in_list(self):
727
        """Test server is in BUILD state, in server list"""
728
        log.info("Server in BUILD state in server list")
729

    
730
        self.result_dict.clear()
731

    
732
        servers = self.cyclades_client.list_servers(detail=True)
733
        servers = filter(lambda x: x["name"] == self.servername, servers)
734

    
735
        server = servers[0]
736
        self.assertEqual(server["name"], self.servername)
737
        self.assertEqual(server["flavor"]["id"], self.flavorid)
738
        self.assertEqual(server["image"]["id"], self.imageid)
739
        self.assertEqual(server["status"], "BUILD")
740

    
741
    def test_002b_server_is_building_in_details(self):
742
        """Test server is in BUILD state, in details"""
743

    
744
        log.info("Server in BUILD state in details")
745

    
746
        server = self.cyclades_client.get_server_details(self.serverid)
747
        self.assertEqual(server["name"], self.servername)
748
        self.assertEqual(server["flavor"]["id"], self.flavorid)
749
        self.assertEqual(server["image"]["id"], self.imageid)
750
        self.assertEqual(server["status"], "BUILD")
751

    
752
    def test_002c_set_server_metadata(self):
753

    
754
        log.info("Creating server metadata")
755

    
756
        image = self.cyclades_client.get_image_details(self.imageid)
757
        os_value = image["metadata"]["os"]
758
        users = image["metadata"].get("users", None)
759
        self.cyclades_client.update_server_metadata(self.serverid, OS=os_value)
760

    
761
        userlist = users.split()
762

    
763
        # Determine the username to use for future connections
764
        # to this host
765
        cls = type(self)
766

    
767
        if "root" in userlist:
768
            cls.username = "root"
769
        elif users is None:
770
            cls.username = self._connect_loginname(os_value)
771
        else:
772
            cls.username = choice(userlist)
773

    
774
        self.assertIsNotNone(cls.username)
775

    
776
    def test_002d_verify_server_metadata(self):
777
        """Test server metadata keys are set based on image metadata"""
778

    
779
        log.info("Verifying image metadata")
780

    
781
        servermeta = self.cyclades_client.get_server_metadata(self.serverid)
782
        imagemeta = self.cyclades_client.get_image_metadata(self.imageid)
783

    
784
        self.assertEqual(servermeta["OS"], imagemeta["os"])
785

    
786
    def test_003_server_becomes_active(self):
787
        """Test server becomes ACTIVE"""
788

    
789
        log.info("Waiting for server to become ACTIVE")
790

    
791
        self._insist_on_status_transition(
792
            "BUILD", "ACTIVE", self.build_fail, self.build_warning)
793

    
794
    def test_003a_get_server_oob_console(self):
795
        """Test getting OOB server console over VNC
796

797
        Implementation of RFB protocol follows
798
        http://www.realvnc.com/docs/rfbproto.pdf.
799

800
        """
801
        console = self.cyclades_client.get_server_console(self.serverid)
802
        self.assertEquals(console['type'], "vnc")
803
        sock = self._insist_on_tcp_connection(
804
            socket.AF_INET, console["host"], console["port"])
805

    
806
        # Step 1. ProtocolVersion message (par. 6.1.1)
807
        version = sock.recv(1024)
808
        self.assertEquals(version, 'RFB 003.008\n')
809
        sock.send(version)
810

    
811
        # Step 2. Security (par 6.1.2): Only VNC Authentication supported
812
        sec = sock.recv(1024)
813
        self.assertEquals(list(sec), ['\x01', '\x02'])
814

    
815
        # Step 3. Request VNC Authentication (par 6.1.2)
816
        sock.send('\x02')
817

    
818
        # Step 4. Receive Challenge (par 6.2.2)
819
        challenge = sock.recv(1024)
820
        self.assertEquals(len(challenge), 16)
821

    
822
        # Step 5. DES-Encrypt challenge, use password as key (par 6.2.2)
823
        response = d3des_generate_response(
824
            (console["password"] + '\0' * 8)[:8], challenge)
825
        sock.send(response)
826

    
827
        # Step 6. SecurityResult (par 6.1.3)
828
        result = sock.recv(4)
829
        self.assertEquals(list(result), ['\x00', '\x00', '\x00', '\x00'])
830
        sock.close()
831

    
832
    def test_004_server_has_ipv4(self):
833
        """Test active server has a valid IPv4 address"""
834

    
835
        log.info("Validate server's IPv4")
836

    
837
        server = self.cyclades_client.get_server_details(self.serverid)
838
        ipv4 = self._get_ipv4(server)
839

    
840
        self.result_dict.clear()
841
        self.result_dict["IPv4"] = str(ipv4)
842

    
843
        self.assertEquals(IP(ipv4).version(), 4)
844

    
845
    def test_005_server_has_ipv6(self):
846
        """Test active server has a valid IPv6 address"""
847
        self._skipIf(NO_IPV6, "--no-ipv6 flag enabled")
848

    
849
        log.info("Validate server's IPv6")
850

    
851
        server = self.cyclades_client.get_server_details(self.serverid)
852
        ipv6 = self._get_ipv6(server)
853

    
854
        self.result_dict.clear()
855
        self.result_dict["IPv6"] = str(ipv6)
856

    
857
        self.assertEquals(IP(ipv6).version(), 6)
858

    
859
    def test_006_server_responds_to_ping_IPv4(self):
860
        """Test server responds to ping on IPv4 address"""
861

    
862
        log.info("Testing if server responds to pings in IPv4")
863
        self.result_dict.clear()
864

    
865
        server = self.cyclades_client.get_server_details(self.serverid)
866
        ip = self._get_ipv4(server)
867
        self._try_until_timeout_expires(self.action_timeout,
868
                                        self.action_timeout,
869
                                        "PING IPv4 to %s" % ip,
870
                                        self._ping_once,
871
                                        False, ip)
872

    
873
    def test_007_server_responds_to_ping_IPv6(self):
874
        """Test server responds to ping on IPv6 address"""
875
        self._skipIf(NO_IPV6, "--no-ipv6 flag enabled")
876
        log.info("Testing if server responds to pings in IPv6")
877

    
878
        server = self.cyclades_client.get_server_details(self.serverid)
879
        ip = self._get_ipv6(server)
880
        self._try_until_timeout_expires(self.action_timeout,
881
                                        self.action_timeout,
882
                                        "PING IPv6 to %s" % ip,
883
                                        self._ping_once,
884
                                        True, ip)
885

    
886
    def test_008_submit_shutdown_request(self):
887
        """Test submit request to shutdown server"""
888

    
889
        log.info("Shutting down server")
890

    
891
        self.cyclades_client.shutdown_server(self.serverid)
892

    
893
    def test_009_server_becomes_stopped(self):
894
        """Test server becomes STOPPED"""
895

    
896
        log.info("Waiting until server becomes STOPPED")
897
        self._insist_on_status_transition(
898
            "ACTIVE", "STOPPED", self.action_timeout, self.action_timeout)
899

    
900
    def test_010_submit_start_request(self):
901
        """Test submit start server request"""
902

    
903
        log.info("Starting server")
904

    
905
        self.cyclades_client.start_server(self.serverid)
906

    
907
    def test_011_server_becomes_active(self):
908
        """Test server becomes ACTIVE again"""
909

    
910
        log.info("Waiting until server becomes ACTIVE")
911
        self._insist_on_status_transition(
912
            "STOPPED", "ACTIVE", self.action_timeout, self.action_timeout)
913

    
914
    def test_011a_server_responds_to_ping_IPv4(self):
915
        """Test server OS is actually up and running again"""
916

    
917
        log.info("Testing if server is actually up and running")
918

    
919
        self.test_006_server_responds_to_ping_IPv4()
920

    
921
    def test_012_ssh_to_server_IPv4(self):
922
        """Test SSH to server public IPv4 works, verify hostname"""
923

    
924
        self._skipIf(self.is_windows, "only valid for Linux servers")
925
        server = self.cyclades_client.get_server_details(self.serverid)
926
        self._insist_on_ssh_hostname(self._get_ipv4(server),
927
                                     self.username, self.passwd)
928

    
929
    def test_013_ssh_to_server_IPv6(self):
930
        """Test SSH to server public IPv6 works, verify hostname"""
931
        self._skipIf(self.is_windows, "only valid for Linux servers")
932
        self._skipIf(NO_IPV6, "--no-ipv6 flag enabled")
933

    
934
        server = self.cyclades_client.get_server_details(self.serverid)
935
        self._insist_on_ssh_hostname(self._get_ipv6(server),
936
                                     self.username, self.passwd)
937

    
938
    def test_014_rdp_to_server_IPv4(self):
939
        "Test RDP connection to server public IPv4 works"""
940
        self._skipIf(not self.is_windows, "only valid for Windows servers")
941
        server = self.cyclades_client.get_server_details(self.serverid)
942
        ipv4 = self._get_ipv4(server)
943
        sock = self._insist_on_tcp_connection(socket.AF_INET, ipv4, 3389)
944

    
945
        # No actual RDP processing done. We assume the RDP server is there
946
        # if the connection to the RDP port is successful.
947
        # FIXME: Use rdesktop, analyze exit code? see manpage [costasd]
948
        sock.close()
949

    
950
    def test_015_rdp_to_server_IPv6(self):
951
        "Test RDP connection to server public IPv6 works"""
952
        self._skipIf(not self.is_windows, "only valid for Windows servers")
953
        self._skipIf(NO_IPV6, "--no-ipv6 flag enabled")
954

    
955
        server = self.cyclades_client.get_server_details(self.serverid)
956
        ipv6 = self._get_ipv6(server)
957
        sock = self._get_tcp_connection(socket.AF_INET6, ipv6, 3389)
958

    
959
        # No actual RDP processing done. We assume the RDP server is there
960
        # if the connection to the RDP port is successful.
961
        sock.close()
962

    
963
    def test_016_personality_is_enforced(self):
964
        """Test file injection for personality enforcement"""
965
        self._skipIf(self.is_windows, "only implemented for Linux servers")
966
        self._skipIf(self.personality is None, "No personality file selected")
967

    
968
        log.info("Trying to inject file for personality enforcement")
969

    
970
        server = self.cyclades_client.get_server_details(self.serverid)
971

    
972
        for inj_file in self.personality:
973
            equal_files = self._check_file_through_ssh(self._get_ipv4(server),
974
                                                       inj_file['owner'],
975
                                                       self.passwd,
976
                                                       inj_file['path'],
977
                                                       inj_file['contents'])
978
            self.assertTrue(equal_files)
979

    
980
    def test_017_submit_delete_request(self):
981
        """Test submit request to delete server"""
982

    
983
        log.info("Deleting server")
984

    
985
        self.cyclades_client.delete_server(self.serverid)
986

    
987
    def test_018_server_becomes_deleted(self):
988
        """Test server becomes DELETED"""
989

    
990
        log.info("Testing if server becomes DELETED")
991

    
992
        self._insist_on_status_transition(
993
            "ACTIVE", "DELETED", self.action_timeout, self.action_timeout)
994

    
995
    def test_019_server_no_longer_in_server_list(self):
996
        """Test server is no longer in server list"""
997

    
998
        log.info("Test if server is no longer listed")
999

    
1000
        servers = self.cyclades_client.list_servers()
1001
        self.assertNotIn(self.serverid, [s["id"] for s in servers])
1002

    
1003

    
1004
class NetworkTestCase(unittest.TestCase):
1005
    """ Testing networking in cyclades """
1006

    
1007
    @classmethod
1008
    def setUpClass(cls):
1009
        "Initialize kamaki, get list of current networks"
1010

    
1011
        cls.astakos_client = AstakosClient(AUTH_URL, TOKEN)
1012
        # Cyclades Client
1013
        compute_url = \
1014
            cls.astakos_client.get_service_endpoints('compute')['publicURL']
1015
        cls.cyclades_client = CycladesClient(compute_url, TOKEN)
1016

    
1017
        cls.servername = "%s%s for %s" % (SNF_TEST_PREFIX,
1018
                                          TEST_RUN_ID,
1019
                                          cls.imagename)
1020

    
1021
        #Dictionary initialization for the vms credentials
1022
        cls.serverid = dict()
1023
        cls.username = dict()
1024
        cls.password = dict()
1025
        cls.is_windows = cls.imagename.lower().find("windows") >= 0
1026

    
1027
        cls.result_dict = dict()
1028

    
1029
    def _skipIf(self, condition, msg):
1030
        if condition:
1031
            self.skipTest(msg)
1032

    
1033
    def _get_ipv4(self, server):
1034
        """Get the public IPv4 of a server from the detailed server info"""
1035

    
1036
        nics = server["attachments"]
1037

    
1038
        for nic in nics:
1039
            net_id = nic["network_id"]
1040
            if self.cyclades_client.get_network_details(net_id)["public"]:
1041
                public_addrs = nic["ipv4"]
1042

    
1043
        self.assertTrue(public_addrs is not None)
1044

    
1045
        return public_addrs
1046

    
1047
    def _connect_loginname(self, os_value):
1048
        """Return the login name for connections based on the server OS"""
1049
        if os_value in ("Ubuntu", "Kubuntu", "Fedora"):
1050
            return "user"
1051
        elif os_value in ("windows", "windows_alpha1"):
1052
            return "Administrator"
1053
        else:
1054
            return "root"
1055

    
1056
    def _ping_once(self, ip):
1057

    
1058
        """Test server responds to a single IPv4 or IPv6 ping"""
1059
        cmd = "ping -c 2 -w 3 %s" % (ip)
1060
        ping = subprocess.Popen(cmd, shell=True,
1061
                                stdout=subprocess.PIPE, stderr=subprocess.PIPE)
1062
        (stdout, stderr) = ping.communicate()
1063
        ret = ping.wait()
1064

    
1065
        return (ret == 0)
1066

    
1067
    def test_00001a_submit_create_server_A(self):
1068
        """Test submit create server request"""
1069

    
1070
        log.info("Creating test server A")
1071

    
1072
        serverA = self.cyclades_client.create_server(
1073
            self.servername, self.flavorid, self.imageid, personality=None)
1074

    
1075
        self.assertEqual(serverA["name"], self.servername)
1076
        self.assertEqual(serverA["flavor"]["id"], self.flavorid)
1077
        self.assertEqual(serverA["image"]["id"], self.imageid)
1078
        self.assertEqual(serverA["status"], "BUILD")
1079

    
1080
        # Update class attributes to reflect data on building server
1081
        self.serverid['A'] = serverA["id"]
1082
        self.username['A'] = None
1083
        self.password['A'] = serverA["adminPass"]
1084

    
1085
        log.info("Server A id:" + str(serverA["id"]))
1086
        log.info("Server password " + (self.password['A']))
1087

    
1088
        self.result_dict["Server A ID"] = str(serverA["id"])
1089
        self.result_dict["Server A password"] = serverA["adminPass"]
1090

    
1091
    def test_00001b_serverA_becomes_active(self):
1092
        """Test server becomes ACTIVE"""
1093

    
1094
        log.info("Waiting until test server A becomes ACTIVE")
1095
        self.result_dict.clear()
1096

    
1097
        fail_tmout = time.time() + self.action_timeout
1098
        while True:
1099
            d = self.cyclades_client.get_server_details(self.serverid['A'])
1100
            status = d['status']
1101
            if status == 'ACTIVE':
1102
                active = True
1103
                break
1104
            elif time.time() > fail_tmout:
1105
                self.assertLess(time.time(), fail_tmout)
1106
            else:
1107
                time.sleep(self.query_interval)
1108

    
1109
        self.assertTrue(active)
1110

    
1111
    def test_00002a_submit_create_server_B(self):
1112
        """Test submit create server request"""
1113

    
1114
        log.info("Creating test server B")
1115

    
1116
        serverB = self.cyclades_client.create_server(
1117
            self.servername, self.flavorid, self.imageid, personality=None)
1118

    
1119
        self.assertEqual(serverB["name"], self.servername)
1120
        self.assertEqual(serverB["flavor"]["id"], self.flavorid)
1121
        self.assertEqual(serverB["image"]["id"], self.imageid)
1122
        self.assertEqual(serverB["status"], "BUILD")
1123

    
1124
        # Update class attributes to reflect data on building server
1125
        self.serverid['B'] = serverB["id"]
1126
        self.username['B'] = None
1127
        self.password['B'] = serverB["adminPass"]
1128

    
1129
        log.info("Server B id: " + str(serverB["id"]))
1130
        log.info("Password " + (self.password['B']))
1131

    
1132
        self.result_dict.clear()
1133
        self.result_dict["Server B ID"] = str(serverB["id"])
1134
        self.result_dict["Server B password"] = serverB["adminPass"]
1135

    
1136
    def test_00002b_serverB_becomes_active(self):
1137
        """Test server becomes ACTIVE"""
1138

    
1139
        log.info("Waiting until test server B becomes ACTIVE")
1140
        self.result_dict.clear()
1141

    
1142
        fail_tmout = time.time() + self.action_timeout
1143
        while True:
1144
            d = self.cyclades_client.get_server_details(self.serverid['B'])
1145
            status = d['status']
1146
            if status == 'ACTIVE':
1147
                active = True
1148
                break
1149
            elif time.time() > fail_tmout:
1150
                self.assertLess(time.time(), fail_tmout)
1151
            else:
1152
                time.sleep(self.query_interval)
1153

    
1154
        self.assertTrue(active)
1155

    
1156
    def test_001_create_network(self):
1157
        """Test submit create network request"""
1158

    
1159
        log.info("Submit new network request")
1160
        self.result_dict.clear()
1161

    
1162
        name = SNF_TEST_PREFIX + TEST_RUN_ID
1163
        #previous_num = len(self.client.list_networks())
1164
        network = self.cyclades_client.create_network(
1165
            name, cidr='10.0.1.0/28', dhcp=True)
1166

    
1167
        #Test if right name is assigned
1168
        self.assertEqual(network['name'], name)
1169

    
1170
        # Update class attributes
1171
        cls = type(self)
1172
        cls.networkid = network['id']
1173
        #networks = self.client.list_networks()
1174

    
1175
        fail_tmout = time.time() + self.action_timeout
1176

    
1177
        #Test if new network is created
1178
        while True:
1179
            d = self.cyclades_client.get_network_details(network['id'])
1180
            if d['status'] == 'ACTIVE':
1181
                connected = True
1182
                break
1183
            elif time.time() > fail_tmout:
1184
                self.assertLess(time.time(), fail_tmout)
1185
            else:
1186
                log.info("Waiting for network to become ACTIVE")
1187
                time.sleep(self.query_interval)
1188

    
1189
        self.assertTrue(connected)
1190

    
1191
        self.result_dict["Private network ID"] = str(network['id'])
1192

    
1193
    def test_002_connect_to_network(self):
1194
        """Test connect VMs to network"""
1195

    
1196
        log.info("Connect VMs to private network")
1197
        self.result_dict.clear()
1198

    
1199
        self.cyclades_client.connect_server(self.serverid['A'], self.networkid)
1200
        self.cyclades_client.connect_server(self.serverid['B'], self.networkid)
1201

    
1202
        #Insist on connecting until action timeout
1203
        fail_tmout = time.time() + self.action_timeout
1204

    
1205
        while True:
1206

    
1207
            netsA = [x['network_id']
1208
                     for x in self.cyclades_client.get_server_details(
1209
                         self.serverid['A'])['attachments']]
1210
            netsB = [x['network_id']
1211
                     for x in self.cyclades_client.get_server_details(
1212
                         self.serverid['B'])['attachments']]
1213

    
1214
            if (self.networkid in netsA) and (self.networkid in netsB):
1215
                conn_exists = True
1216
                break
1217
            elif time.time() > fail_tmout:
1218
                self.assertLess(time.time(), fail_tmout)
1219
            else:
1220
                time.sleep(self.query_interval)
1221

    
1222
        #Adding private IPs to class attributes
1223
        cls = type(self)
1224
        cls.priv_ip = dict()
1225

    
1226
        nicsA = self.cyclades_client.get_server_details(
1227
            self.serverid['A'])['attachments']
1228
        nicsB = self.cyclades_client.get_server_details(
1229
            self.serverid['B'])['attachments']
1230

    
1231
        if conn_exists:
1232
            for nic in nicsA:
1233
                if nic["network_id"] == self.networkid:
1234
                    cls.priv_ip["A"] = nic["ipv4"]
1235
            self.result_dict["Server A private IP"] = str(cls.priv_ip["A"])
1236

    
1237
            for nic in nicsB:
1238
                if nic["network_id"] == self.networkid:
1239
                    cls.priv_ip["B"] = nic["ipv4"]
1240
            self.result_dict["Server B private IP"] = str(cls.priv_ip["B"])
1241

    
1242
        self.assertTrue(conn_exists)
1243
        self.assertIsNot(cls.priv_ip["A"], None)
1244
        self.assertIsNot(cls.priv_ip["B"], None)
1245

    
1246
    def test_002a_reboot(self):
1247
        """Rebooting server A"""
1248

    
1249
        log.info("Rebooting server A")
1250

    
1251
        self.cyclades_client.shutdown_server(self.serverid['A'])
1252

    
1253
        fail_tmout = time.time() + self.action_timeout
1254
        while True:
1255
            d = self.cyclades_client.get_server_details(self.serverid['A'])
1256
            status = d['status']
1257
            if status == 'STOPPED':
1258
                break
1259
            elif time.time() > fail_tmout:
1260
                self.assertLess(time.time(), fail_tmout)
1261
            else:
1262
                time.sleep(self.query_interval)
1263

    
1264
        self.cyclades_client.start_server(self.serverid['A'])
1265

    
1266
        while True:
1267
            d = self.cyclades_client.get_server_details(self.serverid['A'])
1268
            status = d['status']
1269
            if status == 'ACTIVE':
1270
                active = True
1271
                break
1272
            elif time.time() > fail_tmout:
1273
                self.assertLess(time.time(), fail_tmout)
1274
            else:
1275
                time.sleep(self.query_interval)
1276

    
1277
        self.assertTrue(active)
1278

    
1279
    def test_002b_ping_server_A(self):
1280
        "Test if server A responds to IPv4 pings"
1281

    
1282
        log.info("Testing if server A responds to IPv4 pings ")
1283
        self.result_dict.clear()
1284

    
1285
        server = self.cyclades_client.get_server_details(self.serverid['A'])
1286
        ip = self._get_ipv4(server)
1287

    
1288
        fail_tmout = time.time() + self.action_timeout
1289

    
1290
        s = False
1291

    
1292
        self.result_dict["Server A public IP"] = str(ip)
1293

    
1294
        while True:
1295

    
1296
            if self._ping_once(ip):
1297
                s = True
1298
                break
1299

    
1300
            elif time.time() > fail_tmout:
1301
                self.assertLess(time.time(), fail_tmout)
1302

    
1303
            else:
1304
                time.sleep(self.query_interval)
1305

    
1306
        self.assertTrue(s)
1307

    
1308
    def test_002c_reboot(self):
1309
        """Reboot server B"""
1310

    
1311
        log.info("Rebooting server B")
1312
        self.result_dict.clear()
1313

    
1314
        self.cyclades_client.shutdown_server(self.serverid['B'])
1315

    
1316
        fail_tmout = time.time() + self.action_timeout
1317
        while True:
1318
            d = self.cyclades_client.get_server_details(self.serverid['B'])
1319
            status = d['status']
1320
            if status == 'STOPPED':
1321
                break
1322
            elif time.time() > fail_tmout:
1323
                self.assertLess(time.time(), fail_tmout)
1324
            else:
1325
                time.sleep(self.query_interval)
1326

    
1327
        self.cyclades_client.start_server(self.serverid['B'])
1328

    
1329
        while True:
1330
            d = self.cyclades_client.get_server_details(self.serverid['B'])
1331
            status = d['status']
1332
            if status == 'ACTIVE':
1333
                active = True
1334
                break
1335
            elif time.time() > fail_tmout:
1336
                self.assertLess(time.time(), fail_tmout)
1337
            else:
1338
                time.sleep(self.query_interval)
1339

    
1340
        self.assertTrue(active)
1341

    
1342
    def test_002d_ping_server_B(self):
1343
        """Test if server B responds to IPv4 pings"""
1344

    
1345
        log.info("Testing if server B responds to IPv4 pings")
1346
        self.result_dict.clear()
1347

    
1348
        server = self.cyclades_client.get_server_details(self.serverid['B'])
1349
        ip = self._get_ipv4(server)
1350

    
1351
        fail_tmout = time.time() + self.action_timeout
1352

    
1353
        s = False
1354

    
1355
        self.result_dict["Server B public IP"] = str(ip)
1356

    
1357
        while True:
1358
            if self._ping_once(ip):
1359
                s = True
1360
                break
1361

    
1362
            elif time.time() > fail_tmout:
1363
                self.assertLess(time.time(), fail_tmout)
1364

    
1365
            else:
1366
                time.sleep(self.query_interval)
1367

    
1368
        self.assertTrue(s)
1369

    
1370
    def test_003a_setup_interface_A(self):
1371
        """Setup eth1 for server A"""
1372

    
1373
        self._skipIf(self.is_windows, "only valid for Linux servers")
1374

    
1375
        log.info("Setting up interface eth1 for server A")
1376
        self.result_dict.clear()
1377

    
1378
        server = self.cyclades_client.get_server_details(self.serverid['A'])
1379
        image = self.cyclades_client.get_image_details(self.imageid)
1380
        os_value = image['metadata']['os']
1381

    
1382
        users = image["metadata"].get("users", None)
1383
        userlist = users.split()
1384

    
1385
        if "root" in userlist:
1386
            loginname = "root"
1387
        elif users is None:
1388
            loginname = self._connect_loginname(os_value)
1389
        else:
1390
            loginname = choice(userlist)
1391

    
1392
        hostip = self._get_ipv4(server)
1393
        myPass = self.password['A']
1394

    
1395
        log.info("SSH in server A as %s/%s" % (loginname, myPass))
1396
        command = "ifconfig eth1 %s && ifconfig eth1 | " \
1397
                  "grep 'inet addr:' | cut -d: -f2 | awk '{ print $1}'" \
1398
                  % self.priv_ip["A"]
1399
        output, status = _ssh_execute(
1400
            hostip, loginname, myPass, command)
1401

    
1402
        self.assertEquals(status, 0)
1403
        self.assertEquals(output[0].strip(), self.priv_ip["A"])
1404

    
1405
    def test_003b_setup_interface_B(self):
1406
        """Setup eth1 for server B"""
1407

    
1408
        self._skipIf(self.is_windows, "only valid for Linux servers")
1409

    
1410
        log.info("Setting up interface eth1 for server B")
1411

    
1412
        server = self.cyclades_client.get_server_details(self.serverid['B'])
1413
        image = self.cyclades_client.get_image_details(self.imageid)
1414
        os_value = image['metadata']['os']
1415

    
1416
        users = image["metadata"].get("users", None)
1417
        userlist = users.split()
1418

    
1419
        if "root" in userlist:
1420
            loginname = "root"
1421
        elif users is None:
1422
            loginname = self._connect_loginname(os_value)
1423
        else:
1424
            loginname = choice(userlist)
1425

    
1426
        hostip = self._get_ipv4(server)
1427
        myPass = self.password['B']
1428

    
1429
        log.info("SSH in server B as %s/%s" % (loginname, myPass))
1430
        command = "ifconfig eth1 %s && ifconfig eth1 | " \
1431
                  "grep 'inet addr:' | cut -d: -f2 | awk '{ print $1}'" \
1432
                  % self.priv_ip["B"]
1433
        output, status = _ssh_execute(
1434
            hostip, loginname, myPass, command)
1435

    
1436
        self.assertEquals(status, 0)
1437
        self.assertEquals(output[0].strip(), self.priv_ip["B"])
1438

    
1439
    def test_003c_test_connection_exists(self):
1440
        """Ping server B from server A to test if connection exists"""
1441

    
1442
        self._skipIf(self.is_windows, "only valid for Linux servers")
1443

    
1444
        log.info("Testing if server A is actually connected to server B")
1445

    
1446
        server = self.cyclades_client.get_server_details(self.serverid['A'])
1447
        image = self.cyclades_client.get_image_details(self.imageid)
1448
        os_value = image['metadata']['os']
1449
        hostip = self._get_ipv4(server)
1450

    
1451
        users = image["metadata"].get("users", None)
1452
        userlist = users.split()
1453

    
1454
        if "root" in userlist:
1455
            loginname = "root"
1456
        elif users is None:
1457
            loginname = self._connect_loginname(os_value)
1458
        else:
1459
            loginname = choice(userlist)
1460

    
1461
        myPass = self.password['A']
1462

    
1463
        cmd = "if ping -c 2 -w 3 %s >/dev/null; \
1464
               then echo \'True\'; fi;" % self.priv_ip["B"]
1465
        lines, status = _ssh_execute(
1466
            hostip, loginname, myPass, cmd)
1467

    
1468
        exists = False
1469

    
1470
        if 'True\n' in lines:
1471
            exists = True
1472

    
1473
        self.assertTrue(exists)
1474

    
1475
    def test_004_disconnect_from_network(self):
1476
        "Disconnecting server A and B from network"
1477

    
1478
        log.info("Disconnecting servers from private network")
1479

    
1480
        prev_state = self.cyclades_client.get_network_details(self.networkid)
1481
        prev_nics = prev_state['attachments']
1482
        #prev_conn = len(prev_nics)
1483

    
1484
        nicsA = [x['id']
1485
                 for x in self.cyclades_client.get_server_details(
1486
                     self.serverid['A'])['attachments']]
1487
        nicsB = [x['id']
1488
                 for x in self.cyclades_client.get_server_details(
1489
                     self.serverid['B'])['attachments']]
1490

    
1491
        for nic in prev_nics:
1492
            if nic in nicsA:
1493
                self.cyclades_client.disconnect_server(self.serverid['A'], nic)
1494
            if nic in nicsB:
1495
                self.cyclades_client.disconnect_server(self.serverid['B'], nic)
1496

    
1497
        #Insist on deleting until action timeout
1498
        fail_tmout = time.time() + self.action_timeout
1499

    
1500
        while True:
1501
            netsA = [x['network_id']
1502
                     for x in self.cyclades_client.get_server_details(
1503
                         self.serverid['A'])['attachments']]
1504
            netsB = [x['network_id']
1505
                     for x in self.cyclades_client.get_server_details(
1506
                         self.serverid['B'])['attachments']]
1507

    
1508
            #connected = (self.client.get_network_details(self.networkid))
1509
            #connections = connected['attachments']
1510
            if (self.networkid not in netsA) and (self.networkid not in netsB):
1511
                conn_exists = False
1512
                break
1513
            elif time.time() > fail_tmout:
1514
                self.assertLess(time.time(), fail_tmout)
1515
            else:
1516
                time.sleep(self.query_interval)
1517

    
1518
        self.assertFalse(conn_exists)
1519

    
1520
    def test_005_destroy_network(self):
1521
        """Test submit delete network request"""
1522

    
1523
        log.info("Submitting delete network request")
1524

    
1525
        self.cyclades_client.delete_network(self.networkid)
1526

    
1527
        fail_tmout = time.time() + self.action_timeout
1528

    
1529
        while True:
1530

    
1531
            curr_net = []
1532
            networks = self.cyclades_client.list_networks()
1533

    
1534
            for net in networks:
1535
                curr_net.append(net['id'])
1536

    
1537
            if self.networkid not in curr_net:
1538
                self.assertTrue(self.networkid not in curr_net)
1539
                break
1540

    
1541
            elif time.time() > fail_tmout:
1542
                self.assertLess(time.time(), fail_tmout)
1543

    
1544
            else:
1545
                time.sleep(self.query_interval)
1546

    
1547
    def test_006_cleanup_servers(self):
1548
        """Cleanup servers created for this test"""
1549

    
1550
        log.info("Delete servers created for this test")
1551

    
1552
        self.cyclades_client.delete_server(self.serverid['A'])
1553
        self.cyclades_client.delete_server(self.serverid['B'])
1554

    
1555
        fail_tmout = time.time() + self.action_timeout
1556

    
1557
        #Ensure server gets deleted
1558
        status = dict()
1559

    
1560
        while True:
1561
            details = \
1562
                self.cyclades_client.get_server_details(self.serverid['A'])
1563
            status['A'] = details['status']
1564
            details = \
1565
                self.cyclades_client.get_server_details(self.serverid['B'])
1566
            status['B'] = details['status']
1567
            if (status['A'] == 'DELETED') and (status['B'] == 'DELETED'):
1568
                deleted = True
1569
                break
1570
            elif time.time() > fail_tmout:
1571
                self.assertLess(time.time(), fail_tmout)
1572
            else:
1573
                time.sleep(self.query_interval)
1574

    
1575
        self.assertTrue(deleted)
1576

    
1577

    
1578
class TestRunnerProcess(Process):
1579
    """A distinct process used to execute part of the tests in parallel"""
1580
    def __init__(self, **kw):
1581
        Process.__init__(self, **kw)
1582
        kwargs = kw["kwargs"]
1583
        self.testq = kwargs["testq"]
1584
        self.worker_folder = kwargs["worker_folder"]
1585

    
1586
    def run(self):
1587
        # Make sure this test runner process dies with the parent
1588
        # and is not left behind.
1589
        #
1590
        # WARNING: This uses the prctl(2) call and is
1591
        # Linux-specific.
1592

    
1593
        prctl.set_pdeathsig(signal.SIGHUP)
1594

    
1595
        multi = logging.getLogger("multiprocess")
1596

    
1597
        while True:
1598
            multi.debug("I am process %d, GETting from queue is %s" %
1599
                        (os.getpid(), self.testq))
1600
            msg = self.testq.get()
1601

    
1602
            multi.debug("Dequeued msg: %s" % msg)
1603

    
1604
            if msg == "TEST_RUNNER_TERMINATE":
1605
                raise SystemExit
1606

    
1607
            elif issubclass(msg, unittest.TestCase):
1608
                # Assemble a TestSuite, and run it
1609

    
1610
                log_file = os.path.join(self.worker_folder, 'details_' +
1611
                                        (msg.__name__) + "_" +
1612
                                        TEST_RUN_ID + '.log')
1613

    
1614
                fail_file = os.path.join(self.worker_folder, 'failed_' +
1615
                                         (msg.__name__) + "_" +
1616
                                         TEST_RUN_ID + '.log')
1617
                error_file = os.path.join(self.worker_folder, 'error_' +
1618
                                          (msg.__name__) + "_" +
1619
                                          TEST_RUN_ID + '.log')
1620

    
1621
                f = open(log_file, 'w')
1622
                fail = open(fail_file, 'w')
1623
                error = open(error_file, 'w')
1624

    
1625
                log.info(yellow + '* Starting testcase: %s' % msg + normal)
1626

    
1627
                runner = unittest.TextTestRunner(
1628
                    f, verbosity=2, failfast=True,
1629
                    resultclass=BurninTestResult)
1630
                suite = unittest.TestLoader().loadTestsFromTestCase(msg)
1631
                result = runner.run(suite)
1632

    
1633
                for res in result.errors:
1634
                    log.error("snf-burnin encountered an error in "
1635
                              "testcase: %s" % msg)
1636
                    log.error("See log for details")
1637
                    error.write(str(res[0]) + '\n')
1638
                    error.write(str(res[0].shortDescription()) + '\n')
1639
                    error.write('\n')
1640

    
1641
                for res in result.failures:
1642
                    log.error("snf-burnin failed in testcase: %s" % msg)
1643
                    log.error("See log for details")
1644
                    fail.write(str(res[0]) + '\n')
1645
                    fail.write(str(res[0].shortDescription()) + '\n')
1646
                    fail.write('\n')
1647
                    if not NOFAILFAST:
1648
                        sys.exit()
1649

    
1650
                if (len(result.failures) == 0) and (len(result.errors) == 0):
1651
                    log.debug("Passed testcase: %s" % msg)
1652

    
1653
                f.close()
1654
                fail.close()
1655
                error.close()
1656

    
1657
            else:
1658
                raise Exception("Cannot handle msg: %s" % msg)
1659

    
1660

    
1661
def _run_cases_in_series(cases, image_folder):
1662
    """Run instances of TestCase in series"""
1663

    
1664
    for case in cases:
1665

    
1666
        test = case.__name__
1667

    
1668
        log.info(yellow + '* Starting testcase: %s' % test + normal)
1669
        log_file = os.path.join(image_folder, 'details_' +
1670
                                (case.__name__) + "_" +
1671
                                TEST_RUN_ID + '.log')
1672
        fail_file = os.path.join(image_folder, 'failed_' +
1673
                                 (case.__name__) + "_" +
1674
                                 TEST_RUN_ID + '.log')
1675
        error_file = os.path.join(image_folder, 'error_' +
1676
                                  (case.__name__) + "_" +
1677
                                  TEST_RUN_ID + '.log')
1678

    
1679
        f = open(log_file, "w")
1680
        fail = open(fail_file, "w")
1681
        error = open(error_file, "w")
1682

    
1683
        suite = unittest.TestLoader().loadTestsFromTestCase(case)
1684
        runner = unittest.TextTestRunner(
1685
            f, verbosity=2, failfast=True,
1686
            resultclass=BurninTestResult)
1687
        result = runner.run(suite)
1688

    
1689
        for res in result.errors:
1690
            log.error("snf-burnin encountered an error in "
1691
                      "testcase: %s" % test)
1692
            log.error("See log for details")
1693
            error.write(str(res[0]) + '\n')
1694
            error.write(str(res[0].shortDescription()) + '\n')
1695
            error.write('\n')
1696

    
1697
        for res in result.failures:
1698
            log.error("snf-burnin failed in testcase: %s" % test)
1699
            log.error("See log for details")
1700
            fail.write(str(res[0]) + '\n')
1701
            fail.write(str(res[0].shortDescription()) + '\n')
1702
            fail.write('\n')
1703
            if not NOFAILFAST:
1704
                sys.exit()
1705

    
1706
        if (len(result.failures) == 0) and (len(result.errors) == 0):
1707
            log.debug("Passed testcase: %s" % test)
1708

    
1709

    
1710
def _run_cases_in_parallel(cases, fanout, image_folder):
1711
    """Run instances of TestCase in parallel, in a number of distinct processes
1712

1713
    The cases iterable specifies the TestCases to be executed in parallel,
1714
    by test runners running in distinct processes.
1715
    The fanout parameter specifies the number of processes to spawn,
1716
    and defaults to 1.
1717
    The runner argument specifies the test runner class to use inside each
1718
    runner process.
1719

1720
    """
1721

    
1722
    multi = logging.getLogger("multiprocess")
1723
    handler = logging.StreamHandler()
1724
    multi.addHandler(handler)
1725

    
1726
    if VERBOSE:
1727
        multi.setLevel(logging.DEBUG)
1728
    else:
1729
        multi.setLevel(logging.INFO)
1730

    
1731
    testq = []
1732
    worker_folder = []
1733
    runners = []
1734

    
1735
    for i in xrange(0, fanout):
1736
        testq.append(Queue())
1737
        worker_folder.append(os.path.join(image_folder, 'process'+str(i)))
1738
        os.mkdir(worker_folder[i])
1739

    
1740
    for i in xrange(0, fanout):
1741
        kwargs = dict(testq=testq[i], worker_folder=worker_folder[i])
1742
        runners.append(TestRunnerProcess(kwargs=kwargs))
1743

    
1744
    multi.debug("Spawning %d test runner processes" % len(runners))
1745

    
1746
    for p in runners:
1747
        p.start()
1748

    
1749
    # Enqueue test cases
1750
    for i in xrange(0, fanout):
1751
        map(testq[i].put, cases)
1752
        testq[i].put("TEST_RUNNER_TERMINATE")
1753

    
1754
    multi.debug("Spawned %d test runners, PIDs are %s" %
1755
                (len(runners), [p.pid for p in runners]))
1756

    
1757
    multi.debug("Joining %d processes" % len(runners))
1758

    
1759
    for p in runners:
1760
        p.join()
1761

    
1762
    multi.debug("Done joining %d processes" % len(runners))
1763

    
1764

    
1765
def _images_test_case(**kwargs):
1766
    """Construct a new unit test case class from ImagesTestCase"""
1767
    name = "ImagesTestCase_%s" % kwargs["imageid"]
1768
    cls = type(name, (ImagesTestCase,), kwargs)
1769

    
1770
    #Patch extra parameters into test names by manipulating method docstrings
1771
    for (mname, m) in \
1772
            inspect.getmembers(cls, lambda x: inspect.ismethod(x)):
1773
        if hasattr(m, __doc__):
1774
            m.__func__.__doc__ = "[%s] %s" % (cls.imagename, m.__doc__)
1775

    
1776
    # Make sure the class can be pickled, by listing it among
1777
    # the attributes of __main__. A PicklingError is raised otherwise.
1778
    thismodule = sys.modules[__name__]
1779
    setattr(thismodule, name, cls)
1780
    return cls
1781

    
1782

    
1783
def _spawn_server_test_case(**kwargs):
1784
    """Construct a new unit test case class from SpawnServerTestCase"""
1785

    
1786
    name = "SpawnServerTestCase_%s" % kwargs["imageid"]
1787
    cls = type(name, (SpawnServerTestCase,), kwargs)
1788

    
1789
    # Patch extra parameters into test names by manipulating method docstrings
1790
    for (mname, m) in \
1791
            inspect.getmembers(cls, lambda x: inspect.ismethod(x)):
1792
        if hasattr(m, __doc__):
1793
            m.__func__.__doc__ = "[%s] %s" % (cls.imagename, m.__doc__)
1794

    
1795
    # Make sure the class can be pickled, by listing it among
1796
    # the attributes of __main__. A PicklingError is raised otherwise.
1797

    
1798
    thismodule = sys.modules[__name__]
1799
    setattr(thismodule, name, cls)
1800
    return cls
1801

    
1802

    
1803
def _spawn_network_test_case(**kwargs):
1804
    """Construct a new unit test case class from NetworkTestCase"""
1805

    
1806
    name = "NetworkTestCase" + TEST_RUN_ID
1807
    cls = type(name, (NetworkTestCase,), kwargs)
1808

    
1809
    # Make sure the class can be pickled, by listing it among
1810
    # the attributes of __main__. A PicklingError is raised otherwise.
1811

    
1812
    thismodule = sys.modules[__name__]
1813
    setattr(thismodule, name, cls)
1814
    return cls
1815

    
1816

    
1817
# --------------------------------------------------------------------
1818
# Clean up servers/networks functions
1819
def cleanup_servers(timeout, query_interval, delete_stale=False):
1820

    
1821
    astakos_client = AstakosClient(AUTH_URL, TOKEN)
1822
    # Compute Client
1823
    compute_url = astakos_client.get_service_endpoints('compute')['publicURL']
1824
    compute_client = ComputeClient(compute_url, TOKEN)
1825

    
1826
    servers = compute_client.list_servers()
1827
    stale = [s for s in servers if s["name"].startswith(SNF_TEST_PREFIX)]
1828

    
1829
    if len(stale) == 0:
1830
        return
1831

    
1832
    # Show staled servers
1833
    print >>sys.stderr, yellow + \
1834
        "Found these stale servers from previous runs:" + \
1835
        normal
1836
    print >>sys.stderr, "    " + \
1837
        "\n    ".join(["%d: %s" % (s["id"], s["name"]) for s in stale])
1838

    
1839
    # Delete staled servers
1840
    if delete_stale:
1841
        print >> sys.stderr, "Deleting %d stale servers:" % len(stale)
1842
        fail_tmout = time.time() + timeout
1843
        for s in stale:
1844
            compute_client.delete_server(s["id"])
1845
        # Wait for all servers to be deleted
1846
        while True:
1847
            servers = compute_client.list_servers()
1848
            stale = [s for s in servers
1849
                     if s["name"].startswith(SNF_TEST_PREFIX)]
1850
            if len(stale) == 0:
1851
                print >> sys.stderr, green + "    ...done" + normal
1852
                break
1853
            elif time.time() > fail_tmout:
1854
                print >> sys.stderr, red + \
1855
                    "Not all stale servers deleted. Action timed out." + \
1856
                    normal
1857
                sys.exit(1)
1858
            else:
1859
                time.sleep(query_interval)
1860
    else:
1861
        print >> sys.stderr, "Use --delete-stale to delete them."
1862

    
1863

    
1864
def cleanup_networks(action_timeout, query_interval, delete_stale=False):
1865

    
1866
    astakos_client = AstakosClient(AUTH_URL, TOKEN)
1867
    # Cyclades Client
1868
    compute_url = astakos_client.get_service_endpoints('compute')['publicURL']
1869
    cyclades_client = CycladesClient(compute_url, TOKEN)
1870

    
1871
    networks = cyclades_client.list_networks()
1872
    stale = [n for n in networks if n["name"].startswith(SNF_TEST_PREFIX)]
1873

    
1874
    if len(stale) == 0:
1875
        return
1876

    
1877
    # Show staled networks
1878
    print >> sys.stderr, yellow + \
1879
        "Found these stale networks from previous runs:" + \
1880
        normal
1881
    print "    " + \
1882
        "\n    ".join(["%s: %s" % (str(n["id"]), n["name"]) for n in stale])
1883

    
1884
    # Delete staled networks
1885
    if delete_stale:
1886
        print >> sys.stderr, "Deleting %d stale networks:" % len(stale)
1887
        fail_tmout = time.time() + action_timeout
1888
        for n in stale:
1889
            cyclades_client.delete_network(n["id"])
1890
        # Wait for all networks to be deleted
1891
        while True:
1892
            networks = cyclades_client.list_networks()
1893
            stale = [n for n in networks
1894
                     if n["name"].startswith(SNF_TEST_PREFIX)]
1895
            if len(stale) == 0:
1896
                print >> sys.stderr, green + "    ...done" + normal
1897
                break
1898
            elif time.time() > fail_tmout:
1899
                print >> sys.stderr, red + \
1900
                    "Not all stale networks deleted. Action timed out." + \
1901
                    normal
1902
                sys.exit(1)
1903
            else:
1904
                time.sleep(query_interval)
1905
    else:
1906
        print >> sys.stderr, "Use --delete-stale to delete them."
1907

    
1908

    
1909
# --------------------------------------------------------------------
1910
# Parse arguments functions
1911
def parse_comma(option, opt, value, parser):
1912
    tests = set(['all', 'auth', 'images', 'flavors',
1913
                 'pithos', 'servers', 'server_spawn',
1914
                 'network_spawn'])
1915
    parse_input = value.split(',')
1916

    
1917
    if not (set(parse_input)).issubset(tests):
1918
        raise OptionValueError("The selected set of tests is invalid")
1919

    
1920
    setattr(parser.values, option.dest, value.split(','))
1921

    
1922

    
1923
def parse_arguments(args):
1924

    
1925
    kw = {}
1926
    kw["usage"] = "%prog [options]"
1927
    kw["description"] = \
1928
        "%prog runs a number of test scenarios on a " \
1929
        "Synnefo deployment."
1930

    
1931
    parser = OptionParser(**kw)
1932
    parser.disable_interspersed_args()
1933

    
1934
    parser.add_option("--auth-url",
1935
                      action="store", type="string", dest="auth_url",
1936
                      help="The AUTH URI to use to reach the Synnefo API",
1937
                      default=None)
1938
    parser.add_option("--plankton-user",
1939
                      action="store", type="string", dest="plankton_user",
1940
                      help="Owner of system images",
1941
                      default=DEFAULT_PLANKTON_USER)
1942
    parser.add_option("--token",
1943
                      action="store", type="string", dest="token",
1944
                      help="The token to use for authentication to the API")
1945
    parser.add_option("--nofailfast",
1946
                      action="store_true", dest="nofailfast",
1947
                      help="Do not fail immediately if one of the tests "
1948
                           "fails (EXPERIMENTAL)",
1949
                      default=False)
1950
    parser.add_option("--no-ipv6",
1951
                      action="store_true", dest="no_ipv6",
1952
                      help="Disables ipv6 related tests",
1953
                      default=False)
1954
    parser.add_option("--action-timeout",
1955
                      action="store", type="int", dest="action_timeout",
1956
                      metavar="TIMEOUT",
1957
                      help="Wait SECONDS seconds for a server action to "
1958
                           "complete, then the test is considered failed",
1959
                      default=100)
1960
    parser.add_option("--build-warning",
1961
                      action="store", type="int", dest="build_warning",
1962
                      metavar="TIMEOUT",
1963
                      help="Warn if TIMEOUT seconds have passed and a "
1964
                           "build operation is still pending",
1965
                      default=600)
1966
    parser.add_option("--build-fail",
1967
                      action="store", type="int", dest="build_fail",
1968
                      metavar="BUILD_TIMEOUT",
1969
                      help="Fail the test if TIMEOUT seconds have passed "
1970
                           "and a build operation is still incomplete",
1971
                      default=900)
1972
    parser.add_option("--query-interval",
1973
                      action="store", type="int", dest="query_interval",
1974
                      metavar="INTERVAL",
1975
                      help="Query server status when requests are pending "
1976
                           "every INTERVAL seconds",
1977
                      default=3)
1978
    parser.add_option("--fanout",
1979
                      action="store", type="int", dest="fanout",
1980
                      metavar="COUNT",
1981
                      help="Spawn up to COUNT child processes to execute "
1982
                           "in parallel, essentially have up to COUNT "
1983
                           "server build requests outstanding (EXPERIMENTAL)",
1984
                      default=1)
1985
    parser.add_option("--force-flavor",
1986
                      action="store", type="int", dest="force_flavorid",
1987
                      metavar="FLAVOR ID",
1988
                      help="Force all server creations to use the specified "
1989
                           "FLAVOR ID instead of a randomly chosen one, "
1990
                           "useful if disk space is scarce",
1991
                      default=None)
1992
    parser.add_option("--image-id",
1993
                      action="store", type="string", dest="force_imageid",
1994
                      metavar="IMAGE ID",
1995
                      help="Test the specified image id, use 'all' to test "
1996
                           "all available images (mandatory argument)",
1997
                      default=None)
1998
    parser.add_option("--show-stale",
1999
                      action="store_true", dest="show_stale",
2000
                      help="Show stale servers from previous runs, whose "
2001
                           "name starts with `%s'" % SNF_TEST_PREFIX,
2002
                      default=False)
2003
    parser.add_option("--delete-stale",
2004
                      action="store_true", dest="delete_stale",
2005
                      help="Delete stale servers from previous runs, whose "
2006
                           "name starts with `%s'" % SNF_TEST_PREFIX,
2007
                      default=False)
2008
    parser.add_option("--force-personality",
2009
                      action="store", type="string", dest="personality_path",
2010
                      help="Force a personality file injection.\
2011
                            File path required. ",
2012
                      default=None)
2013
    parser.add_option("--log-folder",
2014
                      action="store", type="string", dest="log_folder",
2015
                      help="Define the absolute path where the output \
2016
                            log is stored. ",
2017
                      default="/var/log/burnin/")
2018
    parser.add_option("--verbose", "-V",
2019
                      action="store_true", dest="verbose",
2020
                      help="Print detailed output about multiple "
2021
                           "processes spawning",
2022
                      default=False)
2023
    parser.add_option("--set-tests",
2024
                      action="callback",
2025
                      dest="tests",
2026
                      type="string",
2027
                      help='Set comma seperated tests for this run. \
2028
                            Available tests: auth, images, flavors, \
2029
                                             servers, server_spawn, \
2030
                                             network_spawn, pithos. \
2031
                            Default = all',
2032
                      default='all',
2033
                      callback=parse_comma)
2034

    
2035
    (opts, args) = parser.parse_args(args)
2036

    
2037
    # -----------------------
2038
    # Verify arguments
2039

    
2040
    # `delete_stale' implies `show_stale'
2041
    if opts.delete_stale:
2042
        opts.show_stale = True
2043

    
2044
    # `token' is mandatory
2045
    _mandatory_argument(opts.token, "--token")
2046
    # `auth_url' is mandatory
2047
    _mandatory_argument(opts.auth_url, "--auth-url")
2048

    
2049
    if not opts.show_stale:
2050
        # `image-id' is mandatory
2051
        _mandatory_argument(opts.force_imageid, "--image-id")
2052
        if opts.force_imageid != 'all':
2053
            try:
2054
                opts.force_imageid = str(opts.force_imageid)
2055
            except ValueError:
2056
                print >>sys.stderr, red + \
2057
                    "Invalid value specified for" + \
2058
                    "--image-id. Use a valid id, or `all'." + \
2059
                    normal
2060
                sys.exit(1)
2061

    
2062
    return (opts, args)
2063

    
2064

    
2065
def _mandatory_argument(Arg, Str):
2066
    if (Arg is None) or (Arg == ""):
2067
        print >>sys.stderr, red + \
2068
            "The " + Str + " argument is mandatory.\n" + \
2069
            normal
2070
        sys.exit(1)
2071

    
2072

    
2073
# --------------------------------------------------------------------
2074
# Burnin main function
2075
def main():
2076
    """Assemble test cases into a test suite, and run it
2077

2078
    IMPORTANT: Tests have dependencies and have to be run in the specified
2079
    order inside a single test case. They communicate through attributes of the
2080
    corresponding TestCase class (shared fixtures). Distinct subclasses of
2081
    TestCase MAY SHARE NO DATA, since they are run in parallel, in distinct
2082
    test runner processes.
2083

2084
    """
2085

    
2086
    # Parse arguments using `optparse'
2087
    (opts, args) = parse_arguments(sys.argv[1:])
2088

    
2089
    # Some global variables
2090
    global AUTH_URL, TOKEN, PLANKTON_USER
2091
    global NO_IPV6, VERBOSE, NOFAILFAST
2092
    AUTH_URL = opts.auth_url
2093
    TOKEN = opts.token
2094
    PLANKTON_USER = opts.plankton_user
2095
    NO_IPV6 = opts.no_ipv6
2096
    VERBOSE = opts.verbose
2097
    NOFAILFAST = opts.nofailfast
2098

    
2099
    # If `show_stale', cleanup stale servers
2100
    # from previous runs and exit
2101
    if opts.show_stale:
2102
        # We must clean the servers first
2103
        cleanup_servers(opts.action_timeout, opts.query_interval,
2104
                        delete_stale=opts.delete_stale)
2105
        cleanup_networks(opts.action_timeout, opts.query_interval,
2106
                         delete_stale=opts.delete_stale)
2107
        return 0
2108

    
2109
    # Initialize a kamaki instance, get flavors, images
2110
    astakos_client = AstakosClient(AUTH_URL, TOKEN)
2111
    # Compute Client
2112
    compute_url = astakos_client.get_service_endpoints('compute')['publicURL']
2113
    compute_client = ComputeClient(compute_url, TOKEN)
2114
    DIMAGES = compute_client.list_images(detail=True)
2115
    DFLAVORS = compute_client.list_flavors(detail=True)
2116

    
2117
    # FIXME: logging, log, LOG PID, TEST_RUN_ID, arguments
2118
    # Run them: FIXME: In parallel, FAILEARLY, catchbreak?
2119
    #unittest.main(verbosity=2, catchbreak=True)
2120

    
2121
    # Get a list of images we are going to test
2122
    if opts.force_imageid == 'all':
2123
        test_images = DIMAGES
2124
    else:
2125
        test_images = filter(lambda x: x["id"] == opts.force_imageid, DIMAGES)
2126

    
2127
    # Create output (logging) folder
2128
    if not os.path.exists(opts.log_folder):
2129
        os.mkdir(opts.log_folder)
2130
    test_folder = os.path.join(opts.log_folder, TEST_RUN_ID)
2131
    os.mkdir(test_folder)
2132

    
2133
    for image in test_images:
2134
        imageid = str(image["id"])
2135
        imagename = image["name"]
2136
        # Choose a flavor (given from user or random)
2137
        if opts.force_flavorid:
2138
            flavorid = opts.force_flavorid
2139
        else:
2140
            flavorid = choice([f["id"] for f in DFLAVORS if f["disk"] >= 20])
2141
        # Personality dictionary for file injection test
2142
        if opts.personality_path is not None:
2143
            f = open(opts.personality_path)
2144
            content = b64encode(f.read())
2145
            personality = []
2146
            st = os.stat(opts.personality_path)
2147
            personality.append({
2148
                'path': '/root/test_inj_file',
2149
                'owner': 'root',
2150
                'group': 'root',
2151
                'mode': 0x7777 & st.st_mode,
2152
                'contents': content})
2153
        else:
2154
            personality = None
2155
        # Give a name to our test servers
2156
        servername = "%s%s for %s" % (SNF_TEST_PREFIX, TEST_RUN_ID, imagename)
2157
        is_windows = imagename.lower().find("windows") >= 0
2158

    
2159
        # Create Server TestCases
2160
        ServerTestCase = _spawn_server_test_case(
2161
            imageid=imageid,
2162
            flavorid=flavorid,
2163
            imagename=imagename,
2164
            personality=personality,
2165
            servername=servername,
2166
            is_windows=is_windows,
2167
            action_timeout=opts.action_timeout,
2168
            build_warning=opts.build_warning,
2169
            build_fail=opts.build_fail,
2170
            query_interval=opts.query_interval)
2171
        # Create Network TestCases
2172
        NetworkTestCase = _spawn_network_test_case(
2173
            action_timeout=opts.action_timeout,
2174
            imageid=imageid,
2175
            flavorid=flavorid,
2176
            imagename=imagename,
2177
            query_interval=opts.query_interval)
2178
        # Create Images TestCase
2179
        CImagesTestCase = _images_test_case(
2180
            action_timeout=opts.action_timeout,
2181
            imageid=imageid,
2182
            flavorid=flavorid,
2183
            imagename=imagename,
2184
            query_interval=opts.query_interval)
2185

    
2186
        # Choose the tests we are going to run
2187
        test_dict = {'auth': UnauthorizedTestCase,
2188
                     'images': CImagesTestCase,
2189
                     'flavors': FlavorsTestCase,
2190
                     'servers': ServersTestCase,
2191
                     'pithos': PithosTestCase,
2192
                     'server_spawn': ServerTestCase,
2193
                     'network_spawn': NetworkTestCase}
2194
        seq_cases = []
2195
        if 'all' in opts.tests:
2196
            seq_cases = [UnauthorizedTestCase, CImagesTestCase,
2197
                         FlavorsTestCase, ServersTestCase,
2198
                         PithosTestCase, ServerTestCase,
2199
                         NetworkTestCase]
2200
        else:
2201
            for test in opts.tests:
2202
                seq_cases.append(test_dict[test])
2203

    
2204
        # Folder for each image
2205
        image_folder = os.path.join(test_folder, imageid)
2206
        os.mkdir(image_folder)
2207

    
2208
        # Run each test
2209
        if opts.fanout > 1:
2210
            _run_cases_in_parallel(seq_cases, opts.fanout, image_folder)
2211
        else:
2212
            _run_cases_in_series(seq_cases, image_folder)
2213

    
2214

    
2215
# --------------------------------------------------------------------
2216
# Call main
2217
if __name__ == "__main__":
2218
    sys.exit(main())