Statistics
| Branch: | Tag: | Revision:

root / snf-tools / synnefo_tools / burnin / cyclades_common.py @ 7c37ab19

History | View | Annotate | Download (23 kB)

1
# Copyright 2013 GRNET S.A. All rights reserved.
2
#
3
# Redistribution and use in source and binary forms, with or
4
# without modification, are permitted provided that the following
5
# conditions are met:
6
#
7
#   1. Redistributions of source code must retain the above
8
#      copyright notice, this list of conditions and the following
9
#      disclaimer.
10
#
11
#   2. Redistributions in binary form must reproduce the above
12
#      copyright notice, this list of conditions and the following
13
#      disclaimer in the documentation and/or other materials
14
#      provided with the distribution.
15
#
16
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
17
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
19
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
20
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
23
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
24
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
26
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
27
# POSSIBILITY OF SUCH DAMAGE.
28
#
29
# The views and conclusions contained in the software and
30
# documentation are those of the authors and should not be
31
# interpreted as representing official policies, either expressed
32
# or implied, of GRNET S.A.
33

    
34
"""
35
Utility functions for Cyclades Tests
36
Cyclades require a lot helper functions and `common'
37
had grown too much.
38

39
"""
40

    
41
import time
42
import IPy
43
import base64
44
import socket
45
import random
46
import paramiko
47
import tempfile
48
import subprocess
49

    
50
from kamaki.clients import ClientError
51

    
52
from synnefo_tools.burnin.common import BurninTests, MB, GB
53

    
54

    
55
# Too many public methods. pylint: disable-msg=R0904
56
class CycladesTests(BurninTests):
57
    """Extends the BurninTests class for Cyclades"""
58
    def _try_until_timeout_expires(self, opmsg, check_fun):
59
        """Try to perform an action until timeout expires"""
60
        assert callable(check_fun), "Not a function"
61

    
62
        action_timeout = self.action_timeout
63
        action_warning = self.action_warning
64
        if action_warning > action_timeout:
65
            action_warning = action_timeout
66

    
67
        start_time = int(time.time())
68
        end_time = start_time + action_warning
69
        while end_time > time.time():
70
            try:
71
                ret_value = check_fun()
72
                self.info("Operation `%s' finished in %s seconds",
73
                          opmsg, int(time.time()) - start_time)
74
                return ret_value
75
            except Retry:
76
                time.sleep(self.query_interval)
77
        self.warning("Operation `%s' is taking too long after %s seconds",
78
                     opmsg, int(time.time()) - start_time)
79

    
80
        end_time = start_time + action_timeout
81
        while end_time > time.time():
82
            try:
83
                ret_value = check_fun()
84
                self.info("Operation `%s' finished in %s seconds",
85
                          opmsg, int(time.time()) - start_time)
86
                return ret_value
87
            except Retry:
88
                time.sleep(self.query_interval)
89
        self.error("Operation `%s' timed out after %s seconds",
90
                   opmsg, int(time.time()) - start_time)
91
        self.fail("time out")
92

    
93
    def _get_list_of_servers(self, detail=False):
94
        """Get (detailed) list of servers"""
95
        if detail:
96
            self.info("Getting detailed list of servers")
97
        else:
98
            self.info("Getting simple list of servers")
99
        return self.clients.cyclades.list_servers(detail=detail)
100

    
101
    def _get_list_of_networks(self, detail=False):
102
        """Get (detailed) list of networks"""
103
        if detail:
104
            self.info("Getting detailed list of networks")
105
        else:
106
            self.info("Getting simple list of networks")
107
        return self.clients.network.list_networks(detail=detail)
108

    
109
    def _get_server_details(self, server, quiet=False):
110
        """Get details for a server"""
111
        if not quiet:
112
            self.info("Getting details for server %s with id %s",
113
                      server['name'], server['id'])
114
        return self.clients.cyclades.get_server_details(server['id'])
115

    
116
    def _create_server(self, image, flavor, personality=None, network=False):
117
        """Create a new server"""
118
        if network:
119
            fip = self._create_floating_ip()
120
            port = self._create_port(fip['floating_network_id'],
121
                                     floating_ip=fip)
122
            networks = [{'port': port['id']}]
123
        else:
124
            networks = None
125

    
126
        servername = "%s for %s" % (self.run_id, image['name'])
127
        self.info("Creating a server with name %s", servername)
128
        self.info("Using image %s with id %s", image['name'], image['id'])
129
        self.info("Using flavor %s with id %s", flavor['name'], flavor['id'])
130
        server = self.clients.cyclades.create_server(
131
            servername, flavor['id'], image['id'],
132
            personality=personality, networks=networks)
133

    
134
        self.info("Server id: %s", server['id'])
135
        self.info("Server password: %s", server['adminPass'])
136

    
137
        self.assertEqual(server['name'], servername)
138
        self.assertEqual(server['flavor']['id'], flavor['id'])
139
        self.assertEqual(server['image']['id'], image['id'])
140
        self.assertEqual(server['status'], "BUILD")
141

    
142
        # Verify quotas
143
        self._check_quotas(disk=+int(flavor['disk'])*GB,
144
                           vm=+1,
145
                           ram=+int(flavor['ram'])*MB,
146
                           cpu=+int(flavor['vcpus']))
147

    
148
        return server
149

    
150
    def _delete_servers(self, servers, error=False):
151
        """Deleting a number of servers in parallel"""
152
        # Disconnect floating IPs
153
        if not error:
154
            # If there is the possibility for the machine to be in
155
            # ERROR state we cannot delete its ports.
156
            for srv in servers:
157
                self.info(
158
                    "Disconnecting all floating IPs from server with id %s",
159
                    srv['id'])
160
                self._disconnect_from_network(srv)
161

    
162
        # Delete servers
163
        for srv in servers:
164
            self.info("Sending the delete request for server with id %s",
165
                      srv['id'])
166
            self.clients.cyclades.delete_server(srv['id'])
167

    
168
        if error:
169
            curr_states = ["ACTIVE", "ERROR", "STOPPED", "BUILD"]
170
        else:
171
            curr_states = ["ACTIVE"]
172
        for srv in servers:
173
            self._insist_on_server_transition(srv, curr_states, "DELETED")
174

    
175
        # Servers no longer in server list
176
        new_servers = [s['id'] for s in self._get_list_of_servers()]
177
        for srv in servers:
178
            self.info("Verifying that server with id %s is no longer in "
179
                      "server list", srv['id'])
180
            self.assertNotIn(srv['id'], new_servers)
181

    
182
        # Verify quotas
183
        flavors = \
184
            [self.clients.compute.get_flavor_details(srv['flavor']['id'])
185
             for srv in servers]
186
        self._verify_quotas_deleted(flavors)
187

    
188
    def _verify_quotas_deleted(self, flavors):
189
        """Verify quotas for a number of deleted servers"""
190
        used_disk = 0
191
        used_vm = 0
192
        used_ram = 0
193
        used_cpu = 0
194
        for flavor in flavors:
195
            used_disk += int(flavor['disk']) * GB
196
            used_vm += 1
197
            used_ram += int(flavor['ram']) * MB
198
            used_cpu += int(flavor['vcpus'])
199
        self._check_quotas(disk=-used_disk,
200
                           vm=-used_vm,
201
                           ram=-used_ram,
202
                           cpu=-used_cpu)
203

    
204
    def _get_connection_username(self, server):
205
        """Determine the username to use to connect to the server"""
206
        users = server['metadata'].get("users", None)
207
        ret_user = None
208
        if users is not None:
209
            user_list = users.split()
210
            if "root" in user_list:
211
                ret_user = "root"
212
            else:
213
                ret_user = random.choice(user_list)
214
        else:
215
            # Return the login name for connections based on the server OS
216
            self.info("Could not find `users' metadata in server. Let's guess")
217
            os_value = server['metadata'].get("os")
218
            if os_value in ("Ubuntu", "Kubuntu", "Fedora"):
219
                ret_user = "user"
220
            elif os_value in ("windows", "windows_alpha1"):
221
                ret_user = "Administrator"
222
            else:
223
                ret_user = "root"
224

    
225
        self.assertIsNotNone(ret_user)
226
        self.info("User's login name: %s", ret_user)
227
        return ret_user
228

    
229
    def _insist_on_server_transition(self, server, curr_statuses, new_status):
230
        """Insist on server transiting from curr_statuses to new_status"""
231
        def check_fun():
232
            """Check server status"""
233
            srv = self._get_server_details(server, quiet=True)
234
            if srv['status'] in curr_statuses:
235
                raise Retry()
236
            elif srv['status'] == new_status:
237
                return
238
            else:
239
                msg = "Server \"%s\" with id %s went to unexpected status %s"
240
                self.error(msg, server['name'], server['id'], srv['status'])
241
                self.fail(msg % (server['name'], server['id'], srv['status']))
242
        opmsg = "Waiting for server \"%s\" with id %s to become %s"
243
        self.info(opmsg, server['name'], server['id'], new_status)
244
        opmsg = opmsg % (server['name'], server['id'], new_status)
245
        self._try_until_timeout_expires(opmsg, check_fun)
246

    
247
    def _insist_on_network_transition(self, network,
248
                                      curr_statuses, new_status):
249
        """Insist on network transiting from curr_statuses to new_status"""
250
        def check_fun():
251
            """Check network status"""
252
            ntw = self.clients.network.get_network_details(network['id'])
253
            if ntw['status'] in curr_statuses:
254
                raise Retry()
255
            elif ntw['status'] == new_status:
256
                return
257
            else:
258
                msg = "Network %s with id %s went to unexpected status %s"
259
                self.error(msg, network['name'], network['id'], ntw['status'])
260
                self.fail(msg %
261
                          (network['name'], network['id'], ntw['status']))
262
        opmsg = "Waiting for network \"%s\" with id %s to become %s"
263
        self.info(opmsg, network['name'], network['id'], new_status)
264
        opmsg = opmsg % (network['name'], network['id'], new_status)
265
        self._try_until_timeout_expires(opmsg, check_fun)
266

    
267
    def _insist_on_tcp_connection(self, family, host, port):
268
        """Insist on tcp connection"""
269
        def check_fun():
270
            """Get a connected socket from the specified family to host:port"""
271
            sock = None
272
            for res in socket.getaddrinfo(host, port, family,
273
                                          socket.SOCK_STREAM, 0,
274
                                          socket.AI_PASSIVE):
275
                fam, socktype, proto, _, saddr = res
276
                try:
277
                    sock = socket.socket(fam, socktype, proto)
278
                except socket.error:
279
                    sock = None
280
                    continue
281
                try:
282
                    sock.connect(saddr)
283
                except socket.error:
284
                    sock.close()
285
                    sock = None
286
                    continue
287
            if sock is None:
288
                raise Retry
289
            return sock
290
        familystr = {socket.AF_INET: "IPv4", socket.AF_INET6: "IPv6",
291
                     socket.AF_UNSPEC: "Unspecified-IPv4/6"}
292
        opmsg = "Connecting over %s to %s:%s"
293
        self.info(opmsg, familystr.get(family, "Unknown"), host, port)
294
        opmsg = opmsg % (familystr.get(family, "Unknown"), host, port)
295
        return self._try_until_timeout_expires(opmsg, check_fun)
296

    
297
    def _get_ips(self, server, version=4, network=None):
298
        """Get the IPs of a server from the detailed server info
299

300
        If network not given then get the public IPs. Else the IPs
301
        attached to that network
302

303
        """
304
        assert version in (4, 6)
305

    
306
        nics = server['attachments']
307
        addrs = []
308
        for nic in nics:
309
            net_id = nic['network_id']
310
            if network is None:
311
                if self.clients.network.get_network_details(net_id)['public']:
312
                    if nic['ipv' + str(version)]:
313
                        addrs.append(nic['ipv' + str(version)])
314
            else:
315
                if net_id == network['id']:
316
                    if nic['ipv' + str(version)]:
317
                        addrs.append(nic['ipv' + str(version)])
318

    
319
        self.assertGreater(len(addrs), 0,
320
                           "Can not get IPs from server attachments")
321

    
322
        for addr in addrs:
323
            self.assertEquals(IPy.IP(addr).version(), version)
324

    
325
        if network is None:
326
            msg = "Server's public IPv%s is %s"
327
            for addr in addrs:
328
                self.info(msg, version, addr)
329
        else:
330
            msg = "Server's IPv%s attached to network \"%s\" is %s"
331
            for addr in addrs:
332
                self.info(msg, version, network['id'], addr)
333
        return addrs
334

    
335
    def _insist_on_ping(self, ip_addr, version=4):
336
        """Test server responds to a single IPv4 of IPv6 ping"""
337
        def check_fun():
338
            """Ping to server"""
339
            cmd = ("ping%s -c 3 -w 20 %s" %
340
                   ("6" if version == 6 else "", ip_addr))
341
            ping = subprocess.Popen(
342
                cmd, shell=True, stdout=subprocess.PIPE,
343
                stderr=subprocess.PIPE)
344
            ping.communicate()
345
            ret = ping.wait()
346
            if ret != 0:
347
                raise Retry
348
        assert version in (4, 6)
349
        opmsg = "Sent IPv%s ping requests to %s"
350
        self.info(opmsg, version, ip_addr)
351
        opmsg = opmsg % (version, ip_addr)
352
        self._try_until_timeout_expires(opmsg, check_fun)
353

    
354
    def _image_is(self, image, osfamily):
355
        """Return true if the image is of `osfamily'"""
356
        d_image = self.clients.cyclades.get_image_details(image['id'])
357
        return d_image['metadata']['osfamily'].lower().find(osfamily) >= 0
358

    
359
    def _ssh_execute(self, hostip, username, password, command):
360
        """Execute a command via ssh"""
361
        ssh = paramiko.SSHClient()
362
        ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
363
        try:
364
            ssh.connect(hostip, username=username, password=password)
365
        except socket.error as err:
366
            self.fail(err)
367
        try:
368
            _, stdout, _ = ssh.exec_command(command)
369
        except paramiko.SSHException as err:
370
            self.fail(err)
371
        status = stdout.channel.recv_exit_status()
372
        output = stdout.readlines()
373
        ssh.close()
374
        return output, status
375

    
376
    def _insist_get_hostname_over_ssh(self, hostip, username, password):
377
        """Connect to server using ssh and get it's hostname"""
378
        def check_fun():
379
            """Get hostname"""
380
            try:
381
                lines, status = self._ssh_execute(
382
                    hostip, username, password, "hostname")
383
                self.assertEqual(status, 0)
384
                self.assertEqual(len(lines), 1)
385
                # Remove new line
386
                return lines[0].strip('\n')
387
            except AssertionError:
388
                raise Retry()
389
        opmsg = "Connecting to server using ssh and get it's hostname"
390
        self.info(opmsg)
391
        hostname = self._try_until_timeout_expires(opmsg, check_fun)
392
        self.info("Server's hostname is %s", hostname)
393
        return hostname
394

    
395
    # Too many arguments. pylint: disable-msg=R0913
396
    def _check_file_through_ssh(self, hostip, username, password,
397
                                remotepath, content):
398
        """Fetch file from server and compare contents"""
399
        self.info("Fetching file %s from remote server", remotepath)
400
        transport = paramiko.Transport((hostip, 22))
401
        transport.connect(username=username, password=password)
402
        with tempfile.NamedTemporaryFile() as ftmp:
403
            sftp = paramiko.SFTPClient.from_transport(transport)
404
            sftp.get(remotepath, ftmp.name)
405
            sftp.close()
406
            transport.close()
407
            self.info("Comparing file contents")
408
            remote_content = base64.b64encode(ftmp.read())
409
            self.assertEqual(content, remote_content)
410

    
411
    # ----------------------------------
412
    # Networks
413
    def _create_network(self, cidr="10.0.1.0/28", dhcp=True):
414
        """Create a new private network"""
415
        name = self.run_id
416
        network = self.clients.network.create_network(
417
            "MAC_FILTERED", name=name, shared=False)
418
        self.info("Network with id %s created", network['id'])
419
        subnet = self.clients.network.create_subnet(
420
            network['id'], cidr=cidr, enable_dhcp=dhcp)
421
        self.info("Subnet with id %s created", subnet['id'])
422

    
423
        # Verify quotas
424
        self._check_quotas(network=+1)
425

    
426
        #Test if the right name is assigned
427
        self.assertEqual(network['name'], name)
428

    
429
        return network
430

    
431
    def _delete_networks(self, networks, error=False):
432
        """Delete a network"""
433
        for net in networks:
434
            self.info("Deleting network with id %s", net['id'])
435
            self.clients.network.delete_network(net['id'])
436

    
437
        if error:
438
            curr_states = ["ACTIVE", "SNF:DRAINED", "ERROR"]
439
        else:
440
            curr_states = ["ACTIVE", "SNF:DRAINED"]
441
        for net in networks:
442
            self._insist_on_network_transition(net, curr_states, "DELETED")
443

    
444
        # Networks no longer in network list
445
        new_networks = [n['id'] for n in self._get_list_of_networks()]
446
        for net in networks:
447
            self.info("Verifying that network with id %s is no longer in "
448
                      "network list", net['id'])
449
            self.assertNotIn(net['id'], new_networks)
450

    
451
        # Verify quotas
452
        self._check_quotas(network=-len(networks))
453

    
454
    def _get_public_network(self, networks=None):
455
        """Get the public network"""
456
        if networks is None:
457
            networks = self._get_list_of_networks(detail=True)
458
        self.info("Getting the public network")
459
        for net in networks:
460
            if net['SNF:floating_ip_pool'] and net['public']:
461
                return net
462
        self.fail("Could not find a public network to use")
463

    
464
    def _create_floating_ip(self):
465
        """Create a new floating ip"""
466
        pub_net = self._get_public_network()
467
        self.info("Creating a new floating ip for network with id %s",
468
                  pub_net['id'])
469
        fip = self.clients.network.create_floatingip(pub_net['id'])
470
        # Verify that floating ip has been created
471
        fips = self.clients.network.list_floatingips()
472
        fips = [f['id'] for f in fips]
473
        self.assertIn(fip['id'], fips)
474
        # Verify quotas
475
        self._check_quotas(ip=+1)
476
        # Check that IP is IPv4
477
        self.assertEquals(IPy.IP(fip['floating_ip_address']).version(), 4)
478

    
479
        self.info("Floating IP %s with id %s created",
480
                  fip['floating_ip_address'], fip['id'])
481
        return fip
482

    
483
    def _create_port(self, network_id, device_id=None, floating_ip=None):
484
        """Create a new port attached to the a specific network"""
485
        self.info("Creating a new port to network with id %s", network_id)
486
        if floating_ip is not None:
487
            fixed_ips = [{'ip_address': floating_ip['floating_ip_address']}]
488
        else:
489
            fixed_ips = None
490
        port = self.clients.network.create_port(network_id,
491
                                                device_id=device_id,
492
                                                fixed_ips=fixed_ips)
493
        # Verify that port created
494
        ports = self.clients.network.list_ports()
495
        ports = [p['id'] for p in ports]
496
        self.assertIn(port['id'], ports)
497
        # Insist on creation
498
        if device_id is None:
499
            self._insist_on_port_transition(port, ["BUILD"], "DOWN")
500
        else:
501
            self._insist_on_port_transition(port, ["BUILD", "DOWN"], "ACTIVE")
502

    
503
        self.info("Port with id %s created", port['id'])
504
        return port
505

    
506
    def _insist_on_port_transition(self, port, curr_statuses, new_status):
507
        """Insist on port transiting from curr_statuses to new_status"""
508
        def check_fun():
509
            """Check port status"""
510
            portd = self.clients.network.get_port_details(port['id'])
511
            if portd['status'] in curr_statuses:
512
                raise Retry()
513
            elif portd['status'] == new_status:
514
                return
515
            else:
516
                msg = "Port %s went to unexpected status %s"
517
                self.fail(msg % (portd['id'], portd['status']))
518
        opmsg = "Waiting for port %s to become %s"
519
        self.info(opmsg, port['id'], new_status)
520
        opmsg = opmsg % (port['id'], new_status)
521
        self._try_until_timeout_expires(opmsg, check_fun)
522

    
523
    def _insist_on_port_deletion(self, portid):
524
        """Insist on port deletion"""
525
        def check_fun():
526
            """Check port details"""
527
            try:
528
                self.clients.network.get_port_details(portid)
529
            except ClientError as err:
530
                if err.status != 404:
531
                    raise
532
            else:
533
                raise Retry()
534
        opmsg = "Waiting for port %s to be deleted"
535
        self.info(opmsg, portid)
536
        opmsg = opmsg % portid
537
        self._try_until_timeout_expires(opmsg, check_fun)
538

    
539
    def _disconnect_from_network(self, server, network=None):
540
        """Disconnnect server from network"""
541
        if network is None:
542
            # Disconnect from public network
543
            network = self._get_public_network()
544

    
545
        lports = self.clients.network.list_ports()
546
        ports = []
547
        for port in lports:
548
            dport = self.clients.network.get_port_details(port['id'])
549
            if str(dport['network_id']) == str(network['id']) \
550
                    and str(dport['device_id']) == str(server['id']):
551
                ports.append(dport)
552

    
553
        # Find floating IPs attached to these ports
554
        ports_id = [p['id'] for p in ports]
555
        fips = [f for f in self.clients.network.list_floatingips()
556
                if str(f['port_id']) in ports_id]
557

    
558
        # First destroy the ports
559
        for port in ports:
560
            self.info("Destroying port with id %s", port['id'])
561
            self.clients.network.delete_port(port['id'])
562
            self._insist_on_port_deletion(port['id'])
563

    
564
        # Then delete the floating IPs
565
        self._delete_floating_ips(fips)
566

    
567
    def _delete_floating_ips(self, fips):
568
        """Delete floating ips"""
569
        for fip in fips:
570
            self.info("Destroying floating IP %s with id %s",
571
                      fip['floating_ip_address'], fip['id'])
572
            self.clients.network.delete_floatingip(fip['id'])
573

    
574
        # Check that floating IPs have been deleted
575
        list_ips = [f['id'] for f in self.clients.network.list_floatingips()]
576
        for fip in fips:
577
            self.assertNotIn(fip['id'], list_ips)
578
        # Verify quotas
579
        self._check_quotas(ip=-len(fips))
580

    
581

    
582
class Retry(Exception):
583
    """Retry the action
584

585
    This is used by _try_unit_timeout_expires method.
586

587
    """