Statistics
| Branch: | Tag: | Revision:

root / snf-tools / synnefo_tools / burnin / cyclades_common.py @ 449a1c40

History | View | Annotate | Download (23.1 kB)

1
# Copyright 2013 GRNET S.A. All rights reserved.
2
#
3
# Redistribution and use in source and binary forms, with or
4
# without modification, are permitted provided that the following
5
# conditions are met:
6
#
7
#   1. Redistributions of source code must retain the above
8
#      copyright notice, this list of conditions and the following
9
#      disclaimer.
10
#
11
#   2. Redistributions in binary form must reproduce the above
12
#      copyright notice, this list of conditions and the following
13
#      disclaimer in the documentation and/or other materials
14
#      provided with the distribution.
15
#
16
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
17
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
19
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
20
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
23
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
24
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
26
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
27
# POSSIBILITY OF SUCH DAMAGE.
28
#
29
# The views and conclusions contained in the software and
30
# documentation are those of the authors and should not be
31
# interpreted as representing official policies, either expressed
32
# or implied, of GRNET S.A.
33

    
34
"""
35
Utility functions for Cyclades Tests
36
Cyclades require a lot helper functions and `common'
37
had grown too much.
38

39
"""
40

    
41
import time
42
import IPy
43
import base64
44
import socket
45
import random
46
import paramiko
47
import tempfile
48
import subprocess
49

    
50
from kamaki.clients import ClientError
51

    
52
from synnefo_tools.burnin.common import BurninTests, MB, GB
53

    
54

    
55
# Too many public methods. pylint: disable-msg=R0904
56
class CycladesTests(BurninTests):
57
    """Extends the BurninTests class for Cyclades"""
58
    def _try_until_timeout_expires(self, opmsg, check_fun):
59
        """Try to perform an action until timeout expires"""
60
        assert callable(check_fun), "Not a function"
61

    
62
        action_timeout = self.action_timeout
63
        action_warning = self.action_warning
64
        if action_warning > action_timeout:
65
            action_warning = action_timeout
66

    
67
        start_time = int(time.time())
68
        end_time = start_time + action_warning
69
        while end_time > time.time():
70
            try:
71
                ret_value = check_fun()
72
                self.info("Operation `%s' finished in %s seconds",
73
                          opmsg, int(time.time()) - start_time)
74
                return ret_value
75
            except Retry:
76
                time.sleep(self.query_interval)
77
        self.warning("Operation `%s' is taking too long after %s seconds",
78
                     opmsg, int(time.time()) - start_time)
79

    
80
        end_time = start_time + action_timeout
81
        while end_time > time.time():
82
            try:
83
                ret_value = check_fun()
84
                self.info("Operation `%s' finished in %s seconds",
85
                          opmsg, int(time.time()) - start_time)
86
                return ret_value
87
            except Retry:
88
                time.sleep(self.query_interval)
89
        self.error("Operation `%s' timed out after %s seconds",
90
                   opmsg, int(time.time()) - start_time)
91
        self.fail("time out")
92

    
93
    def _get_list_of_servers(self, detail=False):
94
        """Get (detailed) list of servers"""
95
        if detail:
96
            self.info("Getting detailed list of servers")
97
        else:
98
            self.info("Getting simple list of servers")
99
        return self.clients.cyclades.list_servers(detail=detail)
100

    
101
    def _get_list_of_networks(self, detail=False):
102
        """Get (detailed) list of networks"""
103
        if detail:
104
            self.info("Getting detailed list of networks")
105
        else:
106
            self.info("Getting simple list of networks")
107
        return self.clients.network.list_networks(detail=detail)
108

    
109
    def _get_server_details(self, server, quiet=False):
110
        """Get details for a server"""
111
        if not quiet:
112
            self.info("Getting details for server %s with id %s",
113
                      server['name'], server['id'])
114
        return self.clients.cyclades.get_server_details(server['id'])
115

    
116
    def _create_server(self, image, flavor, personality=None, network=False):
117
        """Create a new server"""
118
        if network:
119
            fip = self._create_floating_ip()
120
            port = self._create_port(fip['floating_network_id'],
121
                                     floating_ip=fip)
122
            networks = [{'port': port['id']}]
123
        else:
124
            networks = None
125

    
126
        servername = "%s for %s" % (self.run_id, image['name'])
127
        self.info("Creating a server with name %s", servername)
128
        self.info("Using image %s with id %s", image['name'], image['id'])
129
        self.info("Using flavor %s with id %s", flavor['name'], flavor['id'])
130
        server = self.clients.cyclades.create_server(
131
            servername, flavor['id'], image['id'],
132
            personality=personality, networks=networks)
133

    
134
        self.info("Server id: %s", server['id'])
135
        self.info("Server password: %s", server['adminPass'])
136

    
137
        self.assertEqual(server['name'], servername)
138
        self.assertEqual(server['flavor']['id'], flavor['id'])
139
        self.assertEqual(server['image']['id'], image['id'])
140
        self.assertEqual(server['status'], "BUILD")
141

    
142
        # Verify quotas
143
        self._check_quotas(disk=+int(flavor['disk'])*GB,
144
                           vm=+1,
145
                           ram=+int(flavor['ram'])*MB,
146
                           cpu=+int(flavor['vcpus']))
147

    
148
        return server
149

    
150
    def _delete_servers(self, servers, error=False):
151
        """Deleting a number of servers in parallel"""
152
        # Disconnect floating IPs
153
        if not error:
154
            # If there is the possibility for the machine to be in
155
            # ERROR state we cannot delete its ports.
156
            for srv in servers:
157
                self.info(
158
                    "Disconnecting all floating IPs from server with id %s",
159
                    srv['id'])
160
                self._disconnect_from_network(srv)
161

    
162
        # Delete servers
163
        for srv in servers:
164
            self.info("Sending the delete request for server with id %s",
165
                      srv['id'])
166
            self.clients.cyclades.delete_server(srv['id'])
167

    
168
        if error:
169
            curr_states = ["ACTIVE", "ERROR", "STOPPED", "BUILD"]
170
        else:
171
            curr_states = ["ACTIVE"]
172
        for srv in servers:
173
            self._insist_on_server_transition(srv, curr_states, "DELETED")
174

    
175
        # Servers no longer in server list
176
        new_servers = [s['id'] for s in self._get_list_of_servers()]
177
        for srv in servers:
178
            self.info("Verifying that server with id %s is no longer in "
179
                      "server list", srv['id'])
180
            self.assertNotIn(srv['id'], new_servers)
181

    
182
        # Verify quotas
183
        flavors = \
184
            [self.clients.compute.get_flavor_details(srv['flavor']['id'])
185
             for srv in servers]
186
        self._verify_quotas_deleted(flavors)
187

    
188
    def _verify_quotas_deleted(self, flavors):
189
        """Verify quotas for a number of deleted servers"""
190
        used_disk = 0
191
        used_vm = 0
192
        used_ram = 0
193
        used_cpu = 0
194
        for flavor in flavors:
195
            used_disk += int(flavor['disk']) * GB
196
            used_vm += 1
197
            used_ram += int(flavor['ram']) * MB
198
            used_cpu += int(flavor['vcpus'])
199
        self._check_quotas(disk=-used_disk,
200
                           vm=-used_vm,
201
                           ram=-used_ram,
202
                           cpu=-used_cpu)
203

    
204
    def _get_connection_username(self, server):
205
        """Determine the username to use to connect to the server"""
206
        users = server['metadata'].get("users", None)
207
        ret_user = None
208
        if users is not None:
209
            user_list = users.split()
210
            if "root" in user_list:
211
                ret_user = "root"
212
            else:
213
                ret_user = random.choice(user_list)
214
        else:
215
            # Return the login name for connections based on the server OS
216
            self.info("Could not find `users' metadata in server. Let's guess")
217
            os_value = server['metadata'].get("os")
218
            if os_value in ("Ubuntu", "Kubuntu", "Fedora"):
219
                ret_user = "user"
220
            elif os_value in ("windows", "windows_alpha1"):
221
                ret_user = "Administrator"
222
            else:
223
                ret_user = "root"
224

    
225
        self.assertIsNotNone(ret_user)
226
        self.info("User's login name: %s", ret_user)
227
        return ret_user
228

    
229
    def _insist_on_server_transition(self, server, curr_statuses, new_status):
230
        """Insist on server transiting from curr_statuses to new_status"""
231
        def check_fun():
232
            """Check server status"""
233
            srv = self._get_server_details(server, quiet=True)
234
            if srv['status'] in curr_statuses:
235
                raise Retry()
236
            elif srv['status'] == new_status:
237
                return
238
            else:
239
                msg = "Server \"%s\" with id %s went to unexpected status %s"
240
                self.error(msg, server['name'], server['id'], srv['status'])
241
                self.fail(msg % (server['name'], server['id'], srv['status']))
242
        opmsg = "Waiting for server \"%s\" with id %s to become %s"
243
        self.info(opmsg, server['name'], server['id'], new_status)
244
        opmsg = opmsg % (server['name'], server['id'], new_status)
245
        self._try_until_timeout_expires(opmsg, check_fun)
246

    
247
    def _insist_on_network_transition(self, network,
248
                                      curr_statuses, new_status):
249
        """Insist on network transiting from curr_statuses to new_status"""
250
        def check_fun():
251
            """Check network status"""
252
            ntw = self.clients.network.get_network_details(network['id'])
253
            if ntw['status'] in curr_statuses:
254
                raise Retry()
255
            elif ntw['status'] == new_status:
256
                return
257
            else:
258
                msg = "Network %s with id %s went to unexpected status %s"
259
                self.error(msg, network['name'], network['id'], ntw['status'])
260
                self.fail(msg %
261
                          (network['name'], network['id'], ntw['status']))
262
        opmsg = "Waiting for network \"%s\" with id %s to become %s"
263
        self.info(opmsg, network['name'], network['id'], new_status)
264
        opmsg = opmsg % (network['name'], network['id'], new_status)
265
        self._try_until_timeout_expires(opmsg, check_fun)
266

    
267
    def _insist_on_tcp_connection(self, family, host, port):
268
        """Insist on tcp connection"""
269
        def check_fun():
270
            """Get a connected socket from the specified family to host:port"""
271
            sock = None
272
            for res in socket.getaddrinfo(host, port, family,
273
                                          socket.SOCK_STREAM, 0,
274
                                          socket.AI_PASSIVE):
275
                fam, socktype, proto, _, saddr = res
276
                try:
277
                    sock = socket.socket(fam, socktype, proto)
278
                except socket.error:
279
                    sock = None
280
                    continue
281
                try:
282
                    sock.connect(saddr)
283
                except socket.error:
284
                    sock.close()
285
                    sock = None
286
                    continue
287
            if sock is None:
288
                raise Retry
289
            return sock
290
        familystr = {socket.AF_INET: "IPv4", socket.AF_INET6: "IPv6",
291
                     socket.AF_UNSPEC: "Unspecified-IPv4/6"}
292
        opmsg = "Connecting over %s to %s:%s"
293
        self.info(opmsg, familystr.get(family, "Unknown"), host, port)
294
        opmsg = opmsg % (familystr.get(family, "Unknown"), host, port)
295
        return self._try_until_timeout_expires(opmsg, check_fun)
296

    
297
    def _get_ips(self, server, version=4, network=None):
298
        """Get the IPs of a server from the detailed server info
299

300
        If network not given then get the public IPs. Else the IPs
301
        attached to that network
302

303
        """
304
        assert version in (4, 6)
305

    
306
        nics = server['attachments']
307
        addrs = []
308
        for nic in nics:
309
            net_id = nic['network_id']
310
            if network is None:
311
                if self.clients.network.get_network_details(net_id)['public']:
312
                    if nic['ipv' + str(version)]:
313
                        addrs.append(nic['ipv' + str(version)])
314
            else:
315
                if net_id == network['id']:
316
                    if nic['ipv' + str(version)]:
317
                        addrs.append(nic['ipv' + str(version)])
318

    
319
        self.assertGreater(len(addrs), 0,
320
                           "Can not get IPs from server attachments")
321

    
322
        for addr in addrs:
323
            self.assertEquals(IPy.IP(addr).version(), version)
324

    
325
        if network is None:
326
            msg = "Server's public IPv%s is %s"
327
            for addr in addrs:
328
                self.info(msg, version, addr)
329
        else:
330
            msg = "Server's IPv%s attached to network \"%s\" is %s"
331
            for addr in addrs:
332
                self.info(msg, version, network['id'], addr)
333
        return addrs
334

    
335
    def _insist_on_ping(self, ip_addr, version=4):
336
        """Test server responds to a single IPv4 of IPv6 ping"""
337
        def check_fun():
338
            """Ping to server"""
339
            cmd = ("ping%s -c 3 -w 20 %s" %
340
                   ("6" if version == 6 else "", ip_addr))
341
            ping = subprocess.Popen(
342
                cmd, shell=True, stdout=subprocess.PIPE,
343
                stderr=subprocess.PIPE)
344
            ping.communicate()
345
            ret = ping.wait()
346
            if ret != 0:
347
                raise Retry
348
        assert version in (4, 6)
349
        opmsg = "Sent IPv%s ping requests to %s"
350
        self.info(opmsg, version, ip_addr)
351
        opmsg = opmsg % (version, ip_addr)
352
        self._try_until_timeout_expires(opmsg, check_fun)
353

    
354
    def _image_is(self, image, osfamily):
355
        """Return true if the image is of `osfamily'"""
356
        d_image = self.clients.cyclades.get_image_details(image['id'])
357
        return d_image['metadata']['osfamily'].lower().find(osfamily) >= 0
358

    
359
    # Method could be a function. pylint: disable-msg=R0201
360
    def _ssh_execute(self, hostip, username, password, command):
361
        """Execute a command via ssh"""
362
        ssh = paramiko.SSHClient()
363
        ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
364
        try:
365
            ssh.connect(hostip, username=username, password=password)
366
        except paramiko.SSHException as excpt:
367
            if excpt.code == 104:  # Error reading SSH protocol banner
368
                raise Retry()
369
            else:
370
                raise
371
        _, stdout, _ = ssh.exec_command(command)
372
        status = stdout.channel.recv_exit_status()
373
        output = stdout.readlines()
374
        ssh.close()
375
        return output, status
376

    
377
    def _insist_get_hostname_over_ssh(self, hostip, username, password):
378
        """Connect to server using ssh and get it's hostname"""
379
        def check_fun():
380
            """Get hostname"""
381
            try:
382
                lines, status = self._ssh_execute(
383
                    hostip, username, password, "hostname")
384
                self.assertEqual(status, 0)
385
                self.assertEqual(len(lines), 1)
386
                # Remove new line
387
                return lines[0].strip('\n')
388
            except AssertionError:
389
                raise Retry()
390
        opmsg = "Connecting to server using ssh and get it's hostname"
391
        self.info(opmsg)
392
        hostname = self._try_until_timeout_expires(opmsg, check_fun)
393
        self.info("Server's hostname is %s", hostname)
394
        return hostname
395

    
396
    # Too many arguments. pylint: disable-msg=R0913
397
    def _check_file_through_ssh(self, hostip, username, password,
398
                                remotepath, content):
399
        """Fetch file from server and compare contents"""
400
        self.info("Fetching file %s from remote server", remotepath)
401
        transport = paramiko.Transport((hostip, 22))
402
        transport.connect(username=username, password=password)
403
        with tempfile.NamedTemporaryFile() as ftmp:
404
            sftp = paramiko.SFTPClient.from_transport(transport)
405
            sftp.get(remotepath, ftmp.name)
406
            sftp.close()
407
            transport.close()
408
            self.info("Comparing file contents")
409
            remote_content = base64.b64encode(ftmp.read())
410
            self.assertEqual(content, remote_content)
411

    
412
    # ----------------------------------
413
    # Networks
414
    def _create_network(self, cidr="10.0.1.0/28", dhcp=True):
415
        """Create a new private network"""
416
        name = self.run_id
417
        network = self.clients.network.create_network(
418
            "MAC_FILTERED", name=name, shared=False)
419
        self.info("Network with id %s created", network['id'])
420
        subnet = self.clients.network.create_subnet(
421
            network['id'], cidr=cidr, enable_dhcp=dhcp)
422
        self.info("Subnet with id %s created", subnet['id'])
423

    
424
        # Verify quotas
425
        self._check_quotas(network=+1)
426

    
427
        #Test if the right name is assigned
428
        self.assertEqual(network['name'], name)
429

    
430
        return network
431

    
432
    def _delete_networks(self, networks, error=False):
433
        """Delete a network"""
434
        for net in networks:
435
            self.info("Deleting network with id %s", net['id'])
436
            self.clients.network.delete_network(net['id'])
437

    
438
        if error:
439
            curr_states = ["ACTIVE", "SNF:DRAINED", "ERROR"]
440
        else:
441
            curr_states = ["ACTIVE", "SNF:DRAINED"]
442
        for net in networks:
443
            self._insist_on_network_transition(net, curr_states, "DELETED")
444

    
445
        # Networks no longer in network list
446
        new_networks = [n['id'] for n in self._get_list_of_networks()]
447
        for net in networks:
448
            self.info("Verifying that network with id %s is no longer in "
449
                      "network list", net['id'])
450
            self.assertNotIn(net['id'], new_networks)
451

    
452
        # Verify quotas
453
        self._check_quotas(network=-len(networks))
454

    
455
    def _get_public_network(self, networks=None):
456
        """Get the public network"""
457
        if networks is None:
458
            networks = self._get_list_of_networks(detail=True)
459
        self.info("Getting the public network")
460
        for net in networks:
461
            if net['SNF:floating_ip_pool'] and net['public']:
462
                return net
463
        self.fail("Could not find a public network to use")
464

    
465
    def _create_floating_ip(self):
466
        """Create a new floating ip"""
467
        pub_net = self._get_public_network()
468
        self.info("Creating a new floating ip for network with id %s",
469
                  pub_net['id'])
470
        fip = self.clients.network.create_floatingip(pub_net['id'])
471
        # Verify that floating ip has been created
472
        fips = self.clients.network.list_floatingips()
473
        fips = [f['id'] for f in fips]
474
        self.assertIn(fip['id'], fips)
475
        # Verify quotas
476
        self._check_quotas(ip=+1)
477
        # Check that IP is IPv4
478
        self.assertEquals(IPy.IP(fip['floating_ip_address']).version(), 4)
479

    
480
        self.info("Floating IP %s with id %s created",
481
                  fip['floating_ip_address'], fip['id'])
482
        return fip
483

    
484
    def _create_port(self, network_id, device_id=None, floating_ip=None):
485
        """Create a new port attached to the a specific network"""
486
        self.info("Creating a new port to network with id %s", network_id)
487
        if floating_ip is not None:
488
            fixed_ips = [{'ip_address': floating_ip['floating_ip_address']}]
489
        else:
490
            fixed_ips = None
491
        port = self.clients.network.create_port(network_id,
492
                                                device_id=device_id,
493
                                                fixed_ips=fixed_ips)
494
        # Verify that port created
495
        ports = self.clients.network.list_ports()
496
        ports = [p['id'] for p in ports]
497
        self.assertIn(port['id'], ports)
498
        # Insist on creation
499
        if device_id is None:
500
            self._insist_on_port_transition(port, ["BUILD"], "DOWN")
501
        else:
502
            self._insist_on_port_transition(port, ["BUILD", "DOWN"], "ACTIVE")
503

    
504
        self.info("Port with id %s created", port['id'])
505
        return port
506

    
507
    def _insist_on_port_transition(self, port, curr_statuses, new_status):
508
        """Insist on port transiting from curr_statuses to new_status"""
509
        def check_fun():
510
            """Check port status"""
511
            portd = self.clients.network.get_port_details(port['id'])
512
            if portd['status'] in curr_statuses:
513
                raise Retry()
514
            elif portd['status'] == new_status:
515
                return
516
            else:
517
                msg = "Port %s went to unexpected status %s"
518
                self.fail(msg % (portd['id'], portd['status']))
519
        opmsg = "Waiting for port %s to become %s"
520
        self.info(opmsg, port['id'], new_status)
521
        opmsg = opmsg % (port['id'], new_status)
522
        self._try_until_timeout_expires(opmsg, check_fun)
523

    
524
    def _insist_on_port_deletion(self, portid):
525
        """Insist on port deletion"""
526
        def check_fun():
527
            """Check port details"""
528
            try:
529
                self.clients.network.get_port_details(portid)
530
            except ClientError as err:
531
                if err.status != 404:
532
                    raise
533
            else:
534
                raise Retry()
535
        opmsg = "Waiting for port %s to be deleted"
536
        self.info(opmsg, portid)
537
        opmsg = opmsg % portid
538
        self._try_until_timeout_expires(opmsg, check_fun)
539

    
540
    def _disconnect_from_network(self, server, network=None):
541
        """Disconnnect server from network"""
542
        if network is None:
543
            # Disconnect from public network
544
            network = self._get_public_network()
545

    
546
        lports = self.clients.network.list_ports()
547
        ports = []
548
        for port in lports:
549
            dport = self.clients.network.get_port_details(port['id'])
550
            if str(dport['network_id']) == str(network['id']) \
551
                    and str(dport['device_id']) == str(server['id']):
552
                ports.append(dport)
553

    
554
        # Find floating IPs attached to these ports
555
        ports_id = [p['id'] for p in ports]
556
        fips = [f for f in self.clients.network.list_floatingips()
557
                if str(f['port_id']) in ports_id]
558

    
559
        # First destroy the ports
560
        for port in ports:
561
            self.info("Destroying port with id %s", port['id'])
562
            self.clients.network.delete_port(port['id'])
563
            self._insist_on_port_deletion(port['id'])
564

    
565
        # Then delete the floating IPs
566
        self._delete_floating_ips(fips)
567

    
568
    def _delete_floating_ips(self, fips):
569
        """Delete floating ips"""
570
        for fip in fips:
571
            self.info("Destroying floating IP %s with id %s",
572
                      fip['floating_ip_address'], fip['id'])
573
            self.clients.network.delete_floatingip(fip['id'])
574

    
575
        # Check that floating IPs have been deleted
576
        list_ips = [f['id'] for f in self.clients.network.list_floatingips()]
577
        for fip in fips:
578
            self.assertNotIn(fip['id'], list_ips)
579
        # Verify quotas
580
        self._check_quotas(ip=-len(fips))
581

    
582

    
583
class Retry(Exception):
584
    """Retry the action
585

586
    This is used by _try_unit_timeout_expires method.
587

588
    """