Statistics
| Branch: | Tag: | Revision:

root / snf-tools / synnefo_tools / burnin / cyclades_common.py @ 3966cea9

History | View | Annotate | Download (22.9 kB)

1
# Copyright 2013 GRNET S.A. All rights reserved.
2
#
3
# Redistribution and use in source and binary forms, with or
4
# without modification, are permitted provided that the following
5
# conditions are met:
6
#
7
#   1. Redistributions of source code must retain the above
8
#      copyright notice, this list of conditions and the following
9
#      disclaimer.
10
#
11
#   2. Redistributions in binary form must reproduce the above
12
#      copyright notice, this list of conditions and the following
13
#      disclaimer in the documentation and/or other materials
14
#      provided with the distribution.
15
#
16
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
17
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
19
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
20
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
23
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
24
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
26
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
27
# POSSIBILITY OF SUCH DAMAGE.
28
#
29
# The views and conclusions contained in the software and
30
# documentation are those of the authors and should not be
31
# interpreted as representing official policies, either expressed
32
# or implied, of GRNET S.A.
33

    
34
"""
35
Utility functions for Cyclades Tests
36
Cyclades require a lot helper functions and `common'
37
had grown too much.
38

39
"""
40

    
41
import time
42
import IPy
43
import base64
44
import socket
45
import random
46
import paramiko
47
import tempfile
48
import subprocess
49

    
50
from kamaki.clients import ClientError
51

    
52
from synnefo_tools.burnin.common import BurninTests, MB, GB
53

    
54

    
55
# Too many public methods. pylint: disable-msg=R0904
56
class CycladesTests(BurninTests):
57
    """Extends the BurninTests class for Cyclades"""
58
    def _try_until_timeout_expires(self, opmsg, check_fun):
59
        """Try to perform an action until timeout expires"""
60
        assert callable(check_fun), "Not a function"
61

    
62
        action_timeout = self.action_timeout
63
        action_warning = self.action_warning
64
        if action_warning > action_timeout:
65
            action_warning = action_timeout
66

    
67
        start_time = int(time.time())
68
        end_time = start_time + action_warning
69
        while end_time > time.time():
70
            try:
71
                ret_value = check_fun()
72
                self.info("Operation `%s' finished in %s seconds",
73
                          opmsg, int(time.time()) - start_time)
74
                return ret_value
75
            except Retry:
76
                time.sleep(self.query_interval)
77
        self.warning("Operation `%s' is taking too long after %s seconds",
78
                     opmsg, int(time.time()) - start_time)
79

    
80
        end_time = start_time + action_timeout
81
        while end_time > time.time():
82
            try:
83
                ret_value = check_fun()
84
                self.info("Operation `%s' finished in %s seconds",
85
                          opmsg, int(time.time()) - start_time)
86
                return ret_value
87
            except Retry:
88
                time.sleep(self.query_interval)
89
        self.error("Operation `%s' timed out after %s seconds",
90
                   opmsg, int(time.time()) - start_time)
91
        self.fail("time out")
92

    
93
    def _get_list_of_servers(self, detail=False):
94
        """Get (detailed) list of servers"""
95
        if detail:
96
            self.info("Getting detailed list of servers")
97
        else:
98
            self.info("Getting simple list of servers")
99
        return self.clients.cyclades.list_servers(detail=detail)
100

    
101
    def _get_list_of_networks(self, detail=False):
102
        """Get (detailed) list of networks"""
103
        if detail:
104
            self.info("Getting detailed list of networks")
105
        else:
106
            self.info("Getting simple list of networks")
107
        return self.clients.network.list_networks(detail=detail)
108

    
109
    def _get_server_details(self, server, quiet=False):
110
        """Get details for a server"""
111
        if not quiet:
112
            self.info("Getting details for server %s with id %s",
113
                      server['name'], server['id'])
114
        return self.clients.cyclades.get_server_details(server['id'])
115

    
116
    def _create_server(self, image, flavor, personality=None, network=False):
117
        """Create a new server"""
118
        if network:
119
            fip = self._create_floating_ip()
120
            port = self._create_port(fip['floating_network_id'],
121
                                     floating_ip=fip)
122
            networks = [{'port': port['id']}]
123
        else:
124
            networks = None
125

    
126
        servername = "%s for %s" % (self.run_id, image['name'])
127
        self.info("Creating a server with name %s", servername)
128
        self.info("Using image %s with id %s", image['name'], image['id'])
129
        self.info("Using flavor %s with id %s", flavor['name'], flavor['id'])
130
        server = self.clients.cyclades.create_server(
131
            servername, flavor['id'], image['id'],
132
            personality=personality, networks=networks)
133

    
134
        self.info("Server id: %s", server['id'])
135
        self.info("Server password: %s", server['adminPass'])
136

    
137
        self.assertEqual(server['name'], servername)
138
        self.assertEqual(server['flavor']['id'], flavor['id'])
139
        self.assertEqual(server['image']['id'], image['id'])
140
        self.assertEqual(server['status'], "BUILD")
141

    
142
        # Verify quotas
143
        self._check_quotas(disk=+int(flavor['disk'])*GB,
144
                           vm=+1,
145
                           ram=+int(flavor['ram'])*MB,
146
                           cpu=+int(flavor['vcpus']))
147

    
148
        return server
149

    
150
    def _delete_servers(self, servers, error=False):
151
        """Deleting a number of servers in parallel"""
152
        # Disconnect floating IPs
153
        for srv in servers:
154
            self.info("Disconnecting all floating IPs from server with id %s",
155
                      srv['id'])
156
            self._disconnect_from_network(srv)
157

    
158
        # Delete servers
159
        for srv in servers:
160
            self.info("Sending the delete request for server with id %s",
161
                      srv['id'])
162
            self.clients.cyclades.delete_server(srv['id'])
163

    
164
        if error:
165
            curr_states = ["ACTIVE", "ERROR", "STOPPED", "BUILD"]
166
        else:
167
            curr_states = ["ACTIVE"]
168
        for srv in servers:
169
            self._insist_on_server_transition(srv, curr_states, "DELETED")
170

    
171
        # Servers no longer in server list
172
        new_servers = [s['id'] for s in self._get_list_of_servers()]
173
        for srv in servers:
174
            self.info("Verifying that server with id %s is no longer in "
175
                      "server list", srv['id'])
176
            self.assertNotIn(srv['id'], new_servers)
177

    
178
        # Verify quotas
179
        flavors = \
180
            [self.clients.compute.get_flavor_details(srv['flavor']['id'])
181
             for srv in servers]
182
        self._verify_quotas_deleted(flavors)
183

    
184
    def _verify_quotas_deleted(self, flavors):
185
        """Verify quotas for a number of deleted servers"""
186
        used_disk = 0
187
        used_vm = 0
188
        used_ram = 0
189
        used_cpu = 0
190
        for flavor in flavors:
191
            used_disk += int(flavor['disk']) * GB
192
            used_vm += 1
193
            used_ram += int(flavor['ram']) * MB
194
            used_cpu += int(flavor['vcpus'])
195
        self._check_quotas(disk=-used_disk,
196
                           vm=-used_vm,
197
                           ram=-used_ram,
198
                           cpu=-used_cpu)
199

    
200
    def _get_connection_username(self, server):
201
        """Determine the username to use to connect to the server"""
202
        users = server['metadata'].get("users", None)
203
        ret_user = None
204
        if users is not None:
205
            user_list = users.split()
206
            if "root" in user_list:
207
                ret_user = "root"
208
            else:
209
                ret_user = random.choice(user_list)
210
        else:
211
            # Return the login name for connections based on the server OS
212
            self.info("Could not find `users' metadata in server. Let's guess")
213
            os_value = server['metadata'].get("os")
214
            if os_value in ("Ubuntu", "Kubuntu", "Fedora"):
215
                ret_user = "user"
216
            elif os_value in ("windows", "windows_alpha1"):
217
                ret_user = "Administrator"
218
            else:
219
                ret_user = "root"
220

    
221
        self.assertIsNotNone(ret_user)
222
        self.info("User's login name: %s", ret_user)
223
        return ret_user
224

    
225
    def _insist_on_server_transition(self, server, curr_statuses, new_status):
226
        """Insist on server transiting from curr_statuses to new_status"""
227
        def check_fun():
228
            """Check server status"""
229
            srv = self._get_server_details(server, quiet=True)
230
            if srv['status'] in curr_statuses:
231
                raise Retry()
232
            elif srv['status'] == new_status:
233
                return
234
            else:
235
                msg = "Server \"%s\" with id %s went to unexpected status %s"
236
                self.error(msg, server['name'], server['id'], srv['status'])
237
                self.fail(msg % (server['name'], server['id'], srv['status']))
238
        opmsg = "Waiting for server \"%s\" with id %s to become %s"
239
        self.info(opmsg, server['name'], server['id'], new_status)
240
        opmsg = opmsg % (server['name'], server['id'], new_status)
241
        self._try_until_timeout_expires(opmsg, check_fun)
242

    
243
    def _insist_on_network_transition(self, network,
244
                                      curr_statuses, new_status):
245
        """Insist on network transiting from curr_statuses to new_status"""
246
        def check_fun():
247
            """Check network status"""
248
            ntw = self.clients.network.get_network_details(network['id'])
249
            if ntw['status'] in curr_statuses:
250
                raise Retry()
251
            elif ntw['status'] == new_status:
252
                return
253
            else:
254
                msg = "Network %s with id %s went to unexpected status %s"
255
                self.error(msg, network['name'], network['id'], ntw['status'])
256
                self.fail(msg %
257
                          (network['name'], network['id'], ntw['status']))
258
        opmsg = "Waiting for network \"%s\" with id %s to become %s"
259
        self.info(opmsg, network['name'], network['id'], new_status)
260
        opmsg = opmsg % (network['name'], network['id'], new_status)
261
        self._try_until_timeout_expires(opmsg, check_fun)
262

    
263
    def _insist_on_tcp_connection(self, family, host, port):
264
        """Insist on tcp connection"""
265
        def check_fun():
266
            """Get a connected socket from the specified family to host:port"""
267
            sock = None
268
            for res in socket.getaddrinfo(host, port, family,
269
                                          socket.SOCK_STREAM, 0,
270
                                          socket.AI_PASSIVE):
271
                fam, socktype, proto, _, saddr = res
272
                try:
273
                    sock = socket.socket(fam, socktype, proto)
274
                except socket.error:
275
                    sock = None
276
                    continue
277
                try:
278
                    sock.connect(saddr)
279
                except socket.error:
280
                    sock.close()
281
                    sock = None
282
                    continue
283
            if sock is None:
284
                raise Retry
285
            return sock
286
        familystr = {socket.AF_INET: "IPv4", socket.AF_INET6: "IPv6",
287
                     socket.AF_UNSPEC: "Unspecified-IPv4/6"}
288
        opmsg = "Connecting over %s to %s:%s"
289
        self.info(opmsg, familystr.get(family, "Unknown"), host, port)
290
        opmsg = opmsg % (familystr.get(family, "Unknown"), host, port)
291
        return self._try_until_timeout_expires(opmsg, check_fun)
292

    
293
    def _get_ips(self, server, version=4, network=None):
294
        """Get the IPs of a server from the detailed server info
295

296
        If network not given then get the public IPs. Else the IPs
297
        attached to that network
298

299
        """
300
        assert version in (4, 6)
301

    
302
        nics = server['attachments']
303
        addrs = []
304
        for nic in nics:
305
            net_id = nic['network_id']
306
            if network is None:
307
                if self.clients.network.get_network_details(net_id)['public']:
308
                    if nic['ipv' + str(version)]:
309
                        addrs.append(nic['ipv' + str(version)])
310
            else:
311
                if net_id == network['id']:
312
                    if nic['ipv' + str(version)]:
313
                        addrs.append(nic['ipv' + str(version)])
314

    
315
        self.assertGreater(len(addrs), 0,
316
                           "Can not get IPs from server attachments")
317

    
318
        for addr in addrs:
319
            self.assertEquals(IPy.IP(addr).version(), version)
320

    
321
        if network is None:
322
            msg = "Server's public IPv%s is %s"
323
            for addr in addrs:
324
                self.info(msg, version, addr)
325
        else:
326
            msg = "Server's IPv%s attached to network \"%s\" is %s"
327
            for addr in addrs:
328
                self.info(msg, version, network['id'], addr)
329
        return addrs
330

    
331
    def _insist_on_ping(self, ip_addr, version=4):
332
        """Test server responds to a single IPv4 of IPv6 ping"""
333
        def check_fun():
334
            """Ping to server"""
335
            cmd = ("ping%s -c 3 -w 20 %s" %
336
                   ("6" if version == 6 else "", ip_addr))
337
            ping = subprocess.Popen(
338
                cmd, shell=True, stdout=subprocess.PIPE,
339
                stderr=subprocess.PIPE)
340
            ping.communicate()
341
            ret = ping.wait()
342
            if ret != 0:
343
                raise Retry
344
        assert version in (4, 6)
345
        opmsg = "Sent IPv%s ping requests to %s"
346
        self.info(opmsg, version, ip_addr)
347
        opmsg = opmsg % (version, ip_addr)
348
        self._try_until_timeout_expires(opmsg, check_fun)
349

    
350
    def _image_is(self, image, osfamily):
351
        """Return true if the image is of `osfamily'"""
352
        d_image = self.clients.cyclades.get_image_details(image['id'])
353
        return d_image['metadata']['osfamily'].lower().find(osfamily) >= 0
354

    
355
    def _ssh_execute(self, hostip, username, password, command):
356
        """Execute a command via ssh"""
357
        ssh = paramiko.SSHClient()
358
        ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
359
        try:
360
            ssh.connect(hostip, username=username, password=password)
361
        except socket.error as err:
362
            self.fail(err)
363
        try:
364
            _, stdout, _ = ssh.exec_command(command)
365
        except paramiko.SSHException as err:
366
            self.fail(err)
367
        status = stdout.channel.recv_exit_status()
368
        output = stdout.readlines()
369
        ssh.close()
370
        return output, status
371

    
372
    def _insist_get_hostname_over_ssh(self, hostip, username, password):
373
        """Connect to server using ssh and get it's hostname"""
374
        def check_fun():
375
            """Get hostname"""
376
            try:
377
                lines, status = self._ssh_execute(
378
                    hostip, username, password, "hostname")
379
                self.assertEqual(status, 0)
380
                self.assertEqual(len(lines), 1)
381
                # Remove new line
382
                return lines[0].strip('\n')
383
            except AssertionError:
384
                raise Retry()
385
        opmsg = "Connecting to server using ssh and get it's hostname"
386
        self.info(opmsg)
387
        hostname = self._try_until_timeout_expires(opmsg, check_fun)
388
        self.info("Server's hostname is %s", hostname)
389
        return hostname
390

    
391
    # Too many arguments. pylint: disable-msg=R0913
392
    def _check_file_through_ssh(self, hostip, username, password,
393
                                remotepath, content):
394
        """Fetch file from server and compare contents"""
395
        self.info("Fetching file %s from remote server", remotepath)
396
        transport = paramiko.Transport((hostip, 22))
397
        transport.connect(username=username, password=password)
398
        with tempfile.NamedTemporaryFile() as ftmp:
399
            sftp = paramiko.SFTPClient.from_transport(transport)
400
            sftp.get(remotepath, ftmp.name)
401
            sftp.close()
402
            transport.close()
403
            self.info("Comparing file contents")
404
            remote_content = base64.b64encode(ftmp.read())
405
            self.assertEqual(content, remote_content)
406

    
407
    # ----------------------------------
408
    # Networks
409
    def _create_network(self, cidr="10.0.1.0/28", dhcp=True):
410
        """Create a new private network"""
411
        name = self.run_id
412
        network = self.clients.network.create_network(
413
            "MAC_FILTERED", name=name, shared=False)
414
        self.info("Network with id %s created", network['id'])
415
        subnet = self.clients.network.create_subnet(
416
            network['id'], cidr=cidr, enable_dhcp=dhcp)
417
        self.info("Subnet with id %s created", subnet['id'])
418

    
419
        # Verify quotas
420
        self._check_quotas(network=+1)
421

    
422
        #Test if the right name is assigned
423
        self.assertEqual(network['name'], name)
424

    
425
        return network
426

    
427
    def _delete_networks(self, networks, error=False):
428
        """Delete a network"""
429
        for net in networks:
430
            self.info("Deleting network with id %s", net['id'])
431
            self.clients.network.delete_network(net['id'])
432

    
433
        if error:
434
            curr_states = ["ACTIVE", "SNF:DRAINED", "ERROR"]
435
        else:
436
            curr_states = ["ACTIVE", "SNF:DRAINED"]
437
        for net in networks:
438
            self._insist_on_network_transition(net, curr_states, "DELETED")
439

    
440
        # Networks no longer in network list
441
        new_networks = [n['id'] for n in self._get_list_of_networks()]
442
        for net in networks:
443
            self.info("Verifying that network with id %s is no longer in "
444
                      "network list", net['id'])
445
            self.assertNotIn(net['id'], new_networks)
446

    
447
        # Verify quotas
448
        self._check_quotas(network=-len(networks))
449

    
450
    def _get_public_network(self, networks=None):
451
        """Get the public network"""
452
        if networks is None:
453
            networks = self._get_list_of_networks(detail=True)
454
        self.info("Getting the public network")
455
        for net in networks:
456
            if net['SNF:floating_ip_pool'] and net['public']:
457
                return net
458
        self.fail("Could not find a public network to use")
459

    
460
    def _create_floating_ip(self):
461
        """Create a new floating ip"""
462
        pub_net = self._get_public_network()
463
        self.info("Creating a new floating ip for network with id %s",
464
                  pub_net['id'])
465
        fip = self.clients.network.create_floatingip(pub_net['id'])
466
        # Verify that floating ip has been created
467
        fips = self.clients.network.list_floatingips()
468
        fips = [f['id'] for f in fips]
469
        self.assertIn(fip['id'], fips)
470
        # Verify quotas
471
        self._check_quotas(ip=+1)
472
        # Check that IP is IPv4
473
        self.assertEquals(IPy.IP(fip['floating_ip_address']).version(), 4)
474

    
475
        self.info("Floating IP %s with id %s created",
476
                  fip['floating_ip_address'], fip['id'])
477
        return fip
478

    
479
    def _create_port(self, network_id, device_id=None, floating_ip=None):
480
        """Create a new port attached to the a specific network"""
481
        self.info("Creating a new port to network with id %s", network_id)
482
        if floating_ip is not None:
483
            fixed_ips = [{'ip_address': floating_ip['floating_ip_address']}]
484
        else:
485
            fixed_ips = None
486
        port = self.clients.network.create_port(network_id,
487
                                                device_id=device_id,
488
                                                fixed_ips=fixed_ips)
489
        # Verify that port created
490
        ports = self.clients.network.list_ports()
491
        ports = [p['id'] for p in ports]
492
        self.assertIn(port['id'], ports)
493
        # Insist on creation
494
        if device_id is None:
495
            self._insist_on_port_transition(port, ["BUILD"], "DOWN")
496
        else:
497
            self._insist_on_port_transition(port, ["BUILD", "DOWN"], "ACTIVE")
498

    
499
        self.info("Port with id %s created", port['id'])
500
        return port
501

    
502
    def _insist_on_port_transition(self, port, curr_statuses, new_status):
503
        """Insist on port transiting from curr_statuses to new_status"""
504
        def check_fun():
505
            """Check port status"""
506
            portd = self.clients.network.get_port_details(port['id'])
507
            if portd['status'] in curr_statuses:
508
                raise Retry()
509
            elif portd['status'] == new_status:
510
                return
511
            else:
512
                msg = "Port %s went to unexpected status %s"
513
                self.fail(msg % (portd['id'], portd['status']))
514
        opmsg = "Waiting for port %s to become %s"
515
        self.info(opmsg, port['id'], new_status)
516
        opmsg = opmsg % (port['id'], new_status)
517
        self._try_until_timeout_expires(opmsg, check_fun)
518

    
519
    def _insist_on_port_deletion(self, portid):
520
        """Insist on port deletion"""
521
        def check_fun():
522
            """Check port details"""
523
            try:
524
                self.clients.network.get_port_details(portid)
525
            except ClientError as err:
526
                if err.status != 404:
527
                    raise
528
            else:
529
                raise Retry()
530
        opmsg = "Waiting for port %s to be deleted"
531
        self.info(opmsg, portid)
532
        opmsg = opmsg % portid
533
        self._try_until_timeout_expires(opmsg, check_fun)
534

    
535
    def _disconnect_from_network(self, server, network=None):
536
        """Disconnnect server from network"""
537
        if network is None:
538
            # Disconnect from public network
539
            network = self._get_public_network()
540

    
541
        lports = self.clients.network.list_ports()
542
        ports = []
543
        for port in lports:
544
            dport = self.clients.network.get_port_details(port['id'])
545
            if str(dport['network_id']) == str(network['id']) \
546
                    and str(dport['device_id']) == str(server['id']):
547
                ports.append(dport)
548

    
549
        # Find floating IPs attached to these ports
550
        ports_id = [p['id'] for p in ports]
551
        fips = [f for f in self.clients.network.list_floatingips()
552
                if str(f['port_id']) in ports_id]
553

    
554
        # First destroy the ports
555
        for port in ports:
556
            self.info("Destroying port with id %s", port['id'])
557
            self.clients.network.delete_port(port['id'])
558
            self._insist_on_port_deletion(port['id'])
559

    
560
        # Then delete the floating IPs
561
        self._delete_floating_ips(fips)
562

    
563
    def _delete_floating_ips(self, fips):
564
        """Delete floating ips"""
565
        for fip in fips:
566
            self.info("Destroying floating IP %s with id %s",
567
                      fip['floating_ip_address'], fip['id'])
568
            self.clients.network.delete_floatingip(fip['id'])
569

    
570
        # Check that floating IPs have been deleted
571
        list_ips = [f['id'] for f in self.clients.network.list_floatingips()]
572
        for fip in fips:
573
            self.assertNotIn(fip['id'], list_ips)
574
        # Verify quotas
575
        self._check_quotas(ip=-len(fips))
576

    
577

    
578
class Retry(Exception):
579
    """Retry the action
580

581
    This is used by _try_unit_timeout_expires method.
582

583
    """