Statistics
| Branch: | Tag: | Revision:

root / ci / utils.py @ d1f57c47

History | View | Annotate | Download (27 kB)

1
#!/usr/bin/env python
2

    
3
"""
4
Synnefo ci utils module
5
"""
6

    
7
import os
8
import re
9
import sys
10
import time
11
import logging
12
import fabric.api as fabric
13
import subprocess
14
import tempfile
15
from ConfigParser import ConfigParser, DuplicateSectionError
16

    
17
from kamaki.cli import config as kamaki_config
18
from kamaki.clients.astakos import AstakosClient
19
from kamaki.clients.cyclades import CycladesClient
20
from kamaki.clients.image import ImageClient
21
from kamaki.clients.compute import ComputeClient
22

    
23
DEFAULT_CONFIG_FILE = "new_config"
24
# UUID of owner of system images
25
DEFAULT_SYSTEM_IMAGES_UUID = [
26
    "25ecced9-bf53-4145-91ee-cf47377e9fb2",  # production (okeanos.grnet.gr)
27
    "04cbe33f-29b7-4ef1-94fb-015929e5fc06",  # testing (okeanos.io)
28
    ]
29

    
30

    
31
def _run(cmd, verbose):
32
    """Run fabric with verbose level"""
33
    if verbose:
34
        args = ('running',)
35
    else:
36
        args = ('running', 'stdout',)
37
    with fabric.hide(*args):  # Used * or ** magic. pylint: disable-msg=W0142
38
        return fabric.run(cmd)
39

    
40

    
41
def _put(local, remote):
42
    """Run fabric put command without output"""
43
    with fabric.quiet():
44
        fabric.put(local, remote)
45

    
46

    
47
def _red(msg):
48
    """Red color"""
49
    #return "\x1b[31m" + str(msg) + "\x1b[0m"
50
    return str(msg)
51

    
52

    
53
def _yellow(msg):
54
    """Yellow color"""
55
    #return "\x1b[33m" + str(msg) + "\x1b[0m"
56
    return str(msg)
57

    
58

    
59
def _green(msg):
60
    """Green color"""
61
    #return "\x1b[32m" + str(msg) + "\x1b[0m"
62
    return str(msg)
63

    
64

    
65
def _check_fabric(fun):
66
    """Check if fabric env has been set"""
67
    def wrapper(self, *args, **kwargs):
68
        """wrapper function"""
69
        if not self.fabric_installed:
70
            self.setup_fabric()
71
            self.fabric_installed = True
72
        return fun(self, *args, **kwargs)
73
    return wrapper
74

    
75

    
76
def _check_kamaki(fun):
77
    """Check if kamaki has been initialized"""
78
    def wrapper(self, *args, **kwargs):
79
        """wrapper function"""
80
        if not self.kamaki_installed:
81
            self.setup_kamaki()
82
            self.kamaki_installed = True
83
        return fun(self, *args, **kwargs)
84
    return wrapper
85

    
86

    
87
class _MyFormatter(logging.Formatter):
88
    """Logging Formatter"""
89
    def format(self, record):
90
        format_orig = self._fmt
91
        if record.levelno == logging.DEBUG:
92
            self._fmt = "  %(msg)s"
93
        elif record.levelno == logging.INFO:
94
            self._fmt = "%(msg)s"
95
        elif record.levelno == logging.WARNING:
96
            self._fmt = _yellow("[W] %(msg)s")
97
        elif record.levelno == logging.ERROR:
98
            self._fmt = _red("[E] %(msg)s")
99
        result = logging.Formatter.format(self, record)
100
        self._fmt = format_orig
101
        return result
102

    
103

    
104
# Too few public methods. pylint: disable-msg=R0903
105
class _InfoFilter(logging.Filter):
106
    """Logging Filter that allows DEBUG and INFO messages only"""
107
    def filter(self, rec):
108
        """The filter"""
109
        return rec.levelno in (logging.DEBUG, logging.INFO)
110

    
111

    
112
# Too many instance attributes. pylint: disable-msg=R0902
113
class SynnefoCI(object):
114
    """SynnefoCI python class"""
115

    
116
    def __init__(self, config_file=None, build_id=None, cloud=None):
117
        """ Initialize SynnefoCI python class
118

119
        Setup logger, local_dir, config and kamaki
120
        """
121
        # Setup logger
122
        self.logger = logging.getLogger('synnefo-ci')
123
        self.logger.setLevel(logging.DEBUG)
124

    
125
        handler1 = logging.StreamHandler(sys.stdout)
126
        handler1.setLevel(logging.DEBUG)
127
        handler1.addFilter(_InfoFilter())
128
        handler1.setFormatter(_MyFormatter())
129
        handler2 = logging.StreamHandler(sys.stderr)
130
        handler2.setLevel(logging.WARNING)
131
        handler2.setFormatter(_MyFormatter())
132

    
133
        self.logger.addHandler(handler1)
134
        self.logger.addHandler(handler2)
135

    
136
        # Get our local dir
137
        self.ci_dir = os.path.dirname(os.path.abspath(__file__))
138
        self.repo_dir = os.path.dirname(self.ci_dir)
139

    
140
        # Read config file
141
        if config_file is None:
142
            config_file = DEFAULT_CONFIG_FILE
143
        if not os.path.isabs(config_file):
144
            config_file = os.path.join(self.ci_dir, config_file)
145
        self.config = ConfigParser()
146
        self.config.optionxform = str
147
        self.config.read(config_file)
148

    
149
        # Read temporary_config file
150
        temp_config = self.config.get('Global', 'temporary_config')
151
        self.temp_config = ConfigParser()
152
        self.temp_config.optionxform = str
153
        self.temp_config.read(temp_config)
154
        self.build_id = build_id
155
        self.logger.info("Will use \"%s\" as build id" % _green(self.build_id))
156

    
157
        # Set kamaki cloud
158
        if cloud is not None:
159
            self.kamaki_cloud = cloud
160
        elif self.config.has_option("Deployment", "kamaki_cloud"):
161
            kamaki_cloud = self.config.get("Deployment", "kamaki_cloud")
162
            if kamaki_cloud == "":
163
                self.kamaki_cloud = None
164
        else:
165
            self.kamaki_cloud = None
166

    
167
        # Initialize variables
168
        self.fabric_installed = False
169
        self.kamaki_installed = False
170
        self.cyclades_client = None
171
        self.compute_client = None
172
        self.image_client = None
173

    
174
    def setup_kamaki(self):
175
        """Initialize kamaki
176

177
        Setup cyclades_client, image_client and compute_client
178
        """
179

    
180
        config = kamaki_config.Config()
181
        if self.kamaki_cloud is None:
182
            self.kamaki_cloud = config.get_global("default_cloud")
183

    
184
        self.logger.info("Setup kamaki client, using cloud '%s'.." %
185
                         self.kamaki_cloud)
186
        auth_url = config.get_cloud(self.kamaki_cloud, "url")
187
        self.logger.debug("Authentication URL is %s" % _green(auth_url))
188
        token = config.get_cloud(self.kamaki_cloud, "token")
189
        #self.logger.debug("Token is %s" % _green(token))
190

    
191
        astakos_client = AstakosClient(auth_url, token)
192

    
193
        cyclades_url = \
194
            astakos_client.get_service_endpoints('compute')['publicURL']
195
        self.logger.debug("Cyclades API url is %s" % _green(cyclades_url))
196
        self.cyclades_client = CycladesClient(cyclades_url, token)
197
        self.cyclades_client.CONNECTION_RETRY_LIMIT = 2
198

    
199
        image_url = \
200
            astakos_client.get_service_endpoints('image')['publicURL']
201
        self.logger.debug("Images API url is %s" % _green(image_url))
202
        self.image_client = ImageClient(cyclades_url, token)
203
        self.image_client.CONNECTION_RETRY_LIMIT = 2
204

    
205
        compute_url = \
206
            astakos_client.get_service_endpoints('compute')['publicURL']
207
        self.logger.debug("Compute API url is %s" % _green(compute_url))
208
        self.compute_client = ComputeClient(compute_url, token)
209
        self.compute_client.CONNECTION_RETRY_LIMIT = 2
210

    
211
    def _wait_transition(self, server_id, current_status, new_status):
212
        """Wait for server to go from current_status to new_status"""
213
        self.logger.debug("Waiting for server to become %s" % new_status)
214
        timeout = self.config.getint('Global', 'build_timeout')
215
        sleep_time = 5
216
        while True:
217
            server = self.cyclades_client.get_server_details(server_id)
218
            if server['status'] == new_status:
219
                return server
220
            elif timeout < 0:
221
                self.logger.error(
222
                    "Waiting for server to become %s timed out" % new_status)
223
                self.destroy_server(False)
224
                sys.exit(-1)
225
            elif server['status'] == current_status:
226
                # Sleep for #n secs and continue
227
                timeout = timeout - sleep_time
228
                time.sleep(sleep_time)
229
            else:
230
                self.logger.error(
231
                    "Server failed with status %s" % server['status'])
232
                self.destroy_server(False)
233
                sys.exit(-1)
234

    
235
    @_check_kamaki
236
    def destroy_server(self, wait=True):
237
        """Destroy slave server"""
238
        server_id = int(self.read_temp_config('server_id'))
239
        self.logger.info("Destoying server with id %s " % server_id)
240
        self.cyclades_client.delete_server(server_id)
241
        if wait:
242
            self._wait_transition(server_id, "ACTIVE", "DELETED")
243

    
244
    @_check_kamaki
245
    def create_server(self, image_id=None, flavor_name=None, ssh_keys=None):
246
        """Create slave server"""
247
        self.logger.info("Create a new server..")
248

    
249
        # Find a build_id to use
250
        if self.build_id is None:
251
            # If build_id is given use this, else ..
252
            # Find a uniq build_id to use
253
            ids = self.temp_config.sections()
254
            if ids:
255
                max_id = int(max(self.temp_config.sections(), key=int))
256
                self.build_id = max_id + 1
257
            else:
258
                self.build_id = 1
259
        self.logger.debug("New build id \"%s\" was created"
260
                          % _green(self.build_id))
261

    
262
        # Find an image to use
263
        if image_id is None:
264
            image = self._find_image()
265
            self.logger.debug("Will use image \"%s\"" % _green(image['name']))
266
            image_id = image["id"]
267
        self.logger.debug("Image has id %s" % _green(image_id))
268
        # Find a flavor to use
269
        flavor_id = self._find_flavor(flavor_name)
270
        server = self.cyclades_client.create_server(
271
            self.config.get('Deployment', 'server_name'),
272
            flavor_id,
273
            image_id)
274
        server_id = server['id']
275
        self.write_temp_config('server_id', server_id)
276
        self.logger.debug("Server got id %s" % _green(server_id))
277
        server_user = server['metadata']['users']
278
        self.write_temp_config('server_user', server_user)
279
        self.logger.debug("Server's admin user is %s" % _green(server_user))
280
        server_passwd = server['adminPass']
281
        self.write_temp_config('server_passwd', server_passwd)
282

    
283
        server = self._wait_transition(server_id, "BUILD", "ACTIVE")
284
        self._get_server_ip_and_port(server)
285
        self._copy_ssh_keys(ssh_keys)
286

    
287
        # Setup Firewall
288
        self.setup_fabric()
289
        self.logger.info("Setup firewall")
290
        accept_ssh_from = self.config.get('Global', 'accept_ssh_from')
291
        if accept_ssh_from != "":
292
            self.logger.debug("Block ssh except from %s" % accept_ssh_from)
293
            cmd = """
294
            local_ip=$(/sbin/ifconfig eth0 | grep 'inet addr:' | \
295
                cut -d':' -f2 | cut -d' ' -f1)
296
            iptables -A INPUT -s localhost -j ACCEPT
297
            iptables -A INPUT -s $local_ip -j ACCEPT
298
            iptables -A INPUT -s {0} -p tcp --dport 22 -j ACCEPT
299
            iptables -A INPUT -p tcp --dport 22 -j DROP
300
            """.format(accept_ssh_from)
301
            _run(cmd, False)
302

    
303
        # Setup apt, download packages
304
        self.logger.debug("Setup apt. Install x2goserver and firefox")
305
        cmd = """
306
        echo 'APT::Install-Suggests "false";' >> /etc/apt/apt.conf
307
        apt-get update
308
        apt-get install curl --yes
309
        echo -e "\n\n{0}" >> /etc/apt/sources.list
310
        # Synnefo repo's key
311
        curl https://dev.grnet.gr/files/apt-grnetdev.pub | apt-key add -
312
        # X2GO Key
313
        apt-key adv --recv-keys --keyserver keys.gnupg.net E1F958385BFE2B6E
314
        apt-get install x2go-keyring
315
        apt-get update
316
        apt-get install x2goserver x2goserver-xsession iceweasel
317
        """.format(self.config.get('Global', 'apt_repo'))
318
        _run(cmd, False)
319

    
320
    def _find_flavor(self, flavor_name):
321
        """Given a flavor_name (reg expression) find a flavor id to use"""
322
        # Get a list of flavor names from config file
323
        flavor_names = self.config.get('Deployment', 'flavor_name').split(",")
324
        if flavor_name is not None:
325
            # If we have a flavor_name to use, add it to our list
326
            flavor_names.insert(0, flavor_name)
327

    
328
        flavors = self.compute_client.list_flavors()
329
        for flname in flavor_names:
330
            sflname = flname.strip()
331
            self.logger.debug("Try to find a flavor with name \"%s\"" % sflname)
332
            fls = [f for f in flavors
333
                   if re.search(sflname, f['name']) is not None]
334
            if fls:
335
                self.logger.debug("Will use %s with id %s"
336
                                  % (fls[0]['name'], fls[0]['id']))
337
                return fls[0]['id']
338

    
339
        self.logger.error("No matching flavor found.. aborting")
340
        sys.exit(1)
341

    
342
    def _find_image(self):
343
        """Find a suitable image to use
344

345
        It has to belong to one of the `DEFAULT_SYSTEM_IMAGES_UUID'
346
        users and contain the word given by `image_name' option.
347
        """
348
        image_name = self.config.get('Deployment', 'image_name').lower()
349
        images = self.image_client.list_public(detail=True)['images']
350
        # Select images by `system_uuid' user
351
        images = [x for x in images
352
                  if x['user_id'] in DEFAULT_SYSTEM_IMAGES_UUID]
353
        # Select images with `image_name' in their names
354
        images = [x for x in images
355
                  if x['name'].lower().find(image_name) != -1]
356
        # Let's select the first one
357
        return images[0]
358

    
359
    def _get_server_ip_and_port(self, server):
360
        """Compute server's IPv4 and ssh port number"""
361
        self.logger.info("Get server connection details..")
362
        server_ip = server['attachments'][0]['ipv4']
363
        if ".okeanos.io" in self.cyclades_client.base_url:
364
            tmp1 = int(server_ip.split(".")[2])
365
            tmp2 = int(server_ip.split(".")[3])
366
            server_ip = "gate.okeanos.io"
367
            server_port = 10000 + tmp1 * 256 + tmp2
368
        else:
369
            server_port = 22
370
        self.write_temp_config('server_ip', server_ip)
371
        self.logger.debug("Server's IPv4 is %s" % _green(server_ip))
372
        self.write_temp_config('server_port', server_port)
373
        self.logger.debug("Server's ssh port is %s" % _green(server_port))
374
        self.logger.debug("Access server using \"ssh -X -p %s %s@%s\"" %
375
                          (server_port, server['metadata']['users'], server_ip))
376

    
377
    @_check_fabric
378
    def _copy_ssh_keys(self, ssh_keys):
379
        """Upload/Install ssh keys to server"""
380
        self.logger.debug("Check for authentication keys to use")
381
        if ssh_keys is None:
382
            ssh_keys = self.config.get("Deployment", "ssh_keys")
383

    
384
        if ssh_keys != "":
385
            self.logger.debug("Will use %s authentication keys file" % ssh_keys)
386
            keyfile = '/tmp/%s.pub' % fabric.env.user
387
            _run('mkdir -p ~/.ssh && chmod 700 ~/.ssh', False)
388
            if ssh_keys.startswith("http://") or \
389
                    ssh_keys.startswith("https://") or \
390
                    ssh_keys.startswith("ftp://"):
391
                cmd = """
392
                apt-get update
393
                apt-get install wget --yes
394
                wget {0} -O {1} --no-check-certificate
395
                """.format(ssh_keys, keyfile)
396
                _run(cmd, False)
397
            elif os.path.exists(ssh_keys):
398
                _put(ssh_keys, keyfile)
399
            else:
400
                self.logger.debug("No ssh keys found")
401
            _run('cat %s >> ~/.ssh/authorized_keys' % keyfile, False)
402
            _run('rm %s' % keyfile, False)
403
            self.logger.debug("Uploaded ssh authorized keys")
404
        else:
405
            self.logger.debug("No ssh keys found")
406

    
407
    def write_temp_config(self, option, value):
408
        """Write changes back to config file"""
409
        # If build_id section doesn't exist create a new one
410
        try:
411
            self.temp_config.add_section(str(self.build_id))
412
            creation_time = time.strftime("%a, %d %b %Y %X", time.localtime())
413
            self.write_temp_config("created", creation_time)
414
        except DuplicateSectionError:
415
            pass
416
        self.temp_config.set(str(self.build_id), option, str(value))
417
        curr_time = time.strftime("%a, %d %b %Y %X", time.localtime())
418
        self.temp_config.set(str(self.build_id), "modified", curr_time)
419
        temp_conf_file = self.config.get('Global', 'temporary_config')
420
        with open(temp_conf_file, 'wb') as tcf:
421
            self.temp_config.write(tcf)
422

    
423
    def read_temp_config(self, option):
424
        """Read from temporary_config file"""
425
        # If build_id is None use the latest one
426
        if self.build_id is None:
427
            ids = self.temp_config.sections()
428
            if ids:
429
                self.build_id = int(ids[-1])
430
            else:
431
                self.logger.error("No sections in temporary config file")
432
                sys.exit(1)
433
            self.logger.debug("Will use \"%s\" as build id"
434
                              % _green(self.build_id))
435
        # Read specified option
436
        return self.temp_config.get(str(self.build_id), option)
437

    
438
    def setup_fabric(self):
439
        """Setup fabric environment"""
440
        self.logger.info("Setup fabric parameters..")
441
        fabric.env.user = self.read_temp_config('server_user')
442
        fabric.env.host_string = self.read_temp_config('server_ip')
443
        fabric.env.port = int(self.read_temp_config('server_port'))
444
        fabric.env.password = self.read_temp_config('server_passwd')
445
        fabric.env.connection_attempts = 10
446
        fabric.env.shell = "/bin/bash -c"
447
        fabric.env.disable_known_hosts = True
448
        fabric.env.output_prefix = None
449

    
450
    def _check_hash_sum(self, localfile, remotefile):
451
        """Check hash sums of two files"""
452
        self.logger.debug("Check hash sum for local file %s" % localfile)
453
        hash1 = os.popen("sha256sum %s" % localfile).read().split(' ')[0]
454
        self.logger.debug("Local file has sha256 hash %s" % hash1)
455
        self.logger.debug("Check hash sum for remote file %s" % remotefile)
456
        hash2 = _run("sha256sum %s" % remotefile, False)
457
        hash2 = hash2.split(' ')[0]
458
        self.logger.debug("Remote file has sha256 hash %s" % hash2)
459
        if hash1 != hash2:
460
            self.logger.error("Hashes differ.. aborting")
461
            sys.exit(-1)
462

    
463
    @_check_fabric
464
    def clone_repo(self, local_repo=False):
465
        """Clone Synnefo repo from slave server"""
466
        self.logger.info("Configure repositories on remote server..")
467
        self.logger.debug("Install/Setup git")
468
        cmd = """
469
        apt-get install git --yes
470
        git config --global user.name {0}
471
        git config --global user.email {1}
472
        """.format(self.config.get('Global', 'git_config_name'),
473
                   self.config.get('Global', 'git_config_mail'))
474
        _run(cmd, False)
475

    
476
        # Find synnefo_repo and synnefo_branch to use
477
        synnefo_repo = self.config.get('Global', 'synnefo_repo')
478
        synnefo_branch = self.config.get("Global", "synnefo_branch")
479
        if synnefo_branch == "":
480
            synnefo_branch = \
481
                subprocess.Popen(
482
                    ["git", "rev-parse", "--abbrev-ref", "HEAD"],
483
                    stdout=subprocess.PIPE).communicate()[0].strip()
484
            if synnefo_branch == "HEAD":
485
                synnefo_branch = \
486
                    subprocess.Popen(
487
                        ["git", "rev-parse", "--short", "HEAD"],
488
                        stdout=subprocess.PIPE).communicate()[0].strip()
489
        self.logger.info("Will use branch %s" % synnefo_branch)
490

    
491
        if local_repo or synnefo_branch == "":
492
            # Use local_repo
493
            self.logger.debug("Push local repo to server")
494
            # Firstly create the remote repo
495
            _run("git init synnefo", False)
496
            # Then push our local repo over ssh
497
            # We have to pass some arguments to ssh command
498
            # namely to disable host checking.
499
            (temp_ssh_file_handle, temp_ssh_file) = tempfile.mkstemp()
500
            os.close(temp_ssh_file_handle)
501
            cmd = """
502
            echo 'exec ssh -o "StrictHostKeyChecking no" \
503
                           -o "UserKnownHostsFile /dev/null" \
504
                           -q "$@"' > {4}
505
            chmod u+x {4}
506
            export GIT_SSH="{4}"
507
            echo "{0}" | git push --mirror ssh://{1}@{2}:{3}/~/synnefo
508
            rm -f {4}
509
            """.format(fabric.env.password,
510
                       fabric.env.user,
511
                       fabric.env.host_string,
512
                       fabric.env.port,
513
                       temp_ssh_file)
514
            os.system(cmd)
515
        else:
516
            # Clone Synnefo from remote repo
517
            # Currently clonning synnefo can fail unexpectedly
518
            cloned = False
519
            for i in range(10):
520
                self.logger.debug("Clone synnefo from %s" % synnefo_repo)
521
                try:
522
                    _run("git clone %s synnefo" % synnefo_repo, False)
523
                    cloned = True
524
                    break
525
                except BaseException:
526
                    self.logger.warning(
527
                        "Clonning synnefo failed.. retrying %s" % i)
528
            if not cloned:
529
                self.logger.error("Can not clone Synnefo repo.")
530
                sys.exit(-1)
531

    
532
        # Checkout the desired synnefo_branch
533
        self.logger.debug("Checkout \"%s\" branch/commit" % synnefo_branch)
534
        cmd = """
535
        cd synnefo
536
        for branch in `git branch -a | grep remotes | \
537
                       grep -v HEAD | grep -v master`; do
538
            git branch --track ${branch##*/} $branch
539
        done
540
        git checkout %s
541
        """ % (synnefo_branch)
542
        _run(cmd, False)
543

    
544
    @_check_fabric
545
    def build_synnefo(self):
546
        """Build Synnefo packages"""
547
        self.logger.info("Build Synnefo packages..")
548
        self.logger.debug("Install development packages")
549
        cmd = """
550
        apt-get update
551
        apt-get install zlib1g-dev dpkg-dev debhelper git-buildpackage \
552
                python-dev python-all python-pip --yes
553
        pip install devflow
554
        """
555
        _run(cmd, False)
556

    
557
        if self.config.get('Global', 'patch_pydist') == "True":
558
            self.logger.debug("Patch pydist.py module")
559
            cmd = r"""
560
            sed -r -i 's/(\(\?P<name>\[A-Za-z\]\[A-Za-z0-9_\.)/\1\\\-/' \
561
                /usr/share/python/debpython/pydist.py
562
            """
563
            _run(cmd, False)
564

565
        # Build synnefo packages
566
        self.logger.debug("Build synnefo packages")
567
        cmd = """
568
        devflow-autopkg snapshot -b ~/synnefo_build-area --no-sign
569
        """
570
        with fabric.cd("synnefo"):
571
            _run(cmd, True)
572

573
        # Install snf-deploy package
574
        self.logger.debug("Install snf-deploy package")
575
        cmd = """
576
        dpkg -i snf-deploy*.deb
577
        apt-get -f install --yes
578
        """
579
        with fabric.cd("synnefo_build-area"):
580
            with fabric.settings(warn_only=True):
581
                _run(cmd, True)
582

583
        # Setup synnefo packages for snf-deploy
584
        self.logger.debug("Copy synnefo debs to snf-deploy packages dir")
585
        cmd = """
586
        cp ~/synnefo_build-area/*.deb /var/lib/snf-deploy/packages/
587
        """
588
        _run(cmd, False)
589

590
    @_check_fabric
591
    def build_documentation(self):
592
        """Build Synnefo documentation"""
593
        self.logger.info("Build Synnefo documentation..")
594
        _run("pip install -U Sphinx", False)
595
        with fabric.cd("synnefo"):
596
            _run("devflow-update-version; "
597
                 "./ci/make_docs.sh synnefo_documentation", False)
598

599
    def fetch_documentation(self, dest=None):
600
        """Fetch Synnefo documentation"""
601
        self.logger.info("Fetch Synnefo documentation..")
602
        if dest is None:
603
            dest = "synnefo_documentation"
604
        dest = os.path.abspath(dest)
605
        if not os.path.exists(dest):
606
            os.makedirs(dest)
607
        self.fetch_compressed("synnefo/synnefo_documentation", dest)
608
        self.logger.info("Downloaded documentation to %s" %
609
                         _green(dest))
610

611
    @_check_fabric
612
    def deploy_synnefo(self, schema=None):
613
        """Deploy Synnefo using snf-deploy"""
614
        self.logger.info("Deploy Synnefo..")
615
        if schema is None:
616
            schema = self.config.get('Global', 'schema')
617
        self.logger.debug("Will use %s schema" % schema)
618

619
        schema_dir = os.path.join(self.ci_dir, "schemas/%s" % schema)
620
        if not (os.path.exists(schema_dir) and os.path.isdir(schema_dir)):
621
            raise ValueError("Unknown schema: %s" % schema)
622

623
        self.logger.debug("Upload schema files to server")
624
        _put(os.path.join(schema_dir, "*"), "/etc/snf-deploy/")
625

626
        self.logger.debug("Change password in nodes.conf file")
627
        cmd = """
628
        sed -i 's/^password =.*/password = {0}/' /etc/snf-deploy/nodes.conf
629
        """.format(fabric.env.password)
630
        _run(cmd, False)
631

632
        self.logger.debug("Run snf-deploy")
633
        cmd = """
634
        snf-deploy --disable-colors --autoconf all
635
        """
636
        _run(cmd, True)
637

638
    @_check_fabric
639
    def unit_test(self):
640
        """Run Synnefo unit test suite"""
641
        self.logger.info("Run Synnefo unit test suite")
642
        component = self.config.get('Unit Tests', 'component')
643

644
        self.logger.debug("Install needed packages")
645
        cmd = """
646
        pip install mock
647
        pip install factory_boy
648
        """
649
        _run(cmd, False)
650

651
        self.logger.debug("Upload tests.sh file")
652
        unit_tests_file = os.path.join(self.ci_dir, "tests.sh")
653
        _put(unit_tests_file, ".")
654

655
        self.logger.debug("Run unit tests")
656
        cmd = """
657
        bash tests.sh {0}
658
        """.format(component)
659
        _run(cmd, True)
660

661
    @_check_fabric
662
    def run_burnin(self):
663
        """Run burnin functional test suite"""
664
        self.logger.info("Run Burnin functional test suite")
665
        cmd = """
666
        auth_url=$(grep -e '^url =' .kamakirc | cut -d' ' -f3)
667
        token=$(grep -e '^token =' .kamakirc | cut -d' ' -f3)
668
        images_user=$(kamaki image list -l | grep owner | \
669
                      cut -d':' -f2 | tr -d ' ')
670
        snf-burnin --auth-url=$auth_url --token=$token \
671
            --force-flavor=2 --image-id=all \
672
            --system-images-user=$images_user \
673
            {0}
674
        log_folder=$(ls -1d /var/log/burnin/* | tail -n1)
675
        for i in $(ls $log_folder/*/details*); do
676
            echo -e "\\n\\n"
677
            echo -e "***** $i\\n"
678
            cat $i
679
        done
680
        """.format(self.config.get('Burnin', 'cmd_options'))
681
        _run(cmd, True)
682

683
    @_check_fabric
684
    def fetch_compressed(self, src, dest=None):
685
        """Create a tarball and fetch it locally"""
686
        self.logger.debug("Creating tarball of %s" % src)
687
        basename = os.path.basename(src)
688
        tar_file = basename + ".tgz"
689
        cmd = "tar czf %s %s" % (tar_file, src)
690
        _run(cmd, False)
691
        if not os.path.exists(dest):
692
            os.makedirs(dest)
693

694
        tmp_dir = tempfile.mkdtemp()
695
        fabric.get(tar_file, tmp_dir)
696

697
        dest_file = os.path.join(tmp_dir, tar_file)
698
        self._check_hash_sum(dest_file, tar_file)
699
        self.logger.debug("Untar packages file %s" % dest_file)
700
        cmd = """
701
        cd %s
702
        tar xzf %s
703
        cp -r %s/* %s
704
        rm -r %s
705
        """ % (tmp_dir, tar_file, src, dest, tmp_dir)
706
        os.system(cmd)
707
        self.logger.info("Downloaded %s to %s" %
708
                         (src, _green(dest)))
709

710
    @_check_fabric
711
    def fetch_packages(self, dest=None):
712
        """Fetch Synnefo packages"""
713
        if dest is None:
714
            dest = self.config.get('Global', 'pkgs_dir')
715
        dest = os.path.abspath(dest)
716
        if not os.path.exists(dest):
717
            os.makedirs(dest)
718
        self.fetch_compressed("synnefo_build-area", dest)
719
        self.logger.info("Downloaded debian packages to %s" %
720
                         _green(dest))
721