Statistics
| Branch: | Tag: | Revision:

root / ci / utils.py @ 85bcc8da

History | View | Annotate | Download (19 kB)

1
#!/usr/bin/env python
2

    
3
"""
4
Synnefo ci utils module
5
"""
6

    
7
import os
8
import sys
9
import time
10
import logging
11
import fabric.api as fabric
12
from ConfigParser import ConfigParser, DuplicateSectionError
13

    
14
from kamaki.cli import config as kamaki_config
15
from kamaki.clients.astakos import AstakosClient
16
from kamaki.clients.cyclades import CycladesClient
17
from kamaki.clients.image import ImageClient
18

    
19
DEFAULT_CONFIG_FILE = "new_config"
20

    
21

    
22
def _run(cmd, verbose):
23
    """Run fabric with verbose level"""
24
    if verbose:
25
        args = ('running',)
26
    else:
27
        args = ('running', 'stdout',)
28
    with fabric.hide(*args):
29
        return fabric.run(cmd)
30

    
31

    
32
def _red(msg):
33
    """Red color"""
34
    #return "\x1b[31m" + str(msg) + "\x1b[0m"
35
    return str(msg)
36

    
37

    
38
def _yellow(msg):
39
    """Yellow color"""
40
    #return "\x1b[33m" + str(msg) + "\x1b[0m"
41
    return str(msg)
42

    
43

    
44
def _green(msg):
45
    """Green color"""
46
    #return "\x1b[32m" + str(msg) + "\x1b[0m"
47
    return str(msg)
48

    
49

    
50
def _check_fabric(fun):
51
    """Check if fabric env has been set"""
52
    def wrapper(self, *args, **kwargs):
53
        """wrapper function"""
54
        if not self.fabric_installed:
55
            self.setup_fabric()
56
        return fun(self, *args, **kwargs)
57
    return wrapper
58

    
59

    
60
def _check_kamaki(fun):
61
    """Check if kamaki has been initialized"""
62
    def wrapper(self, *args, **kwargs):
63
        """wrapper function"""
64
        if not self.kamaki_installed:
65
            self.setup_kamaki()
66
        return fun(self, *args, **kwargs)
67
    return wrapper
68

    
69

    
70
class _MyFormatter(logging.Formatter):
71
    """Logging Formatter"""
72
    def format(self, record):
73
        format_orig = self._fmt
74
        if record.levelno == logging.DEBUG:
75
            self._fmt = "  %(msg)s"
76
        elif record.levelno == logging.INFO:
77
            self._fmt = "%(msg)s"
78
        elif record.levelno == logging.WARNING:
79
            self._fmt = _yellow("[W] %(msg)s")
80
        elif record.levelno == logging.ERROR:
81
            self._fmt = _red("[E] %(msg)s")
82
        result = logging.Formatter.format(self, record)
83
        self._fmt = format_orig
84
        return result
85

    
86

    
87
class SynnefoCI(object):
88
    """SynnefoCI python class"""
89

    
90
    def __init__(self, config_file=None, cleanup_config=False, cloud=None):
91
        """ Initialize SynnefoCI python class
92

93
        Setup logger, local_dir, config and kamaki
94
        """
95
        # Setup logger
96
        self.logger = logging.getLogger('synnefo-ci')
97
        self.logger.setLevel(logging.DEBUG)
98
        handler = logging.StreamHandler()
99
        handler.setFormatter(_MyFormatter())
100
        self.logger.addHandler(handler)
101

    
102
        # Get our local dir
103
        self.ci_dir = os.path.dirname(os.path.abspath(__file__))
104
        self.repo_dir = os.path.dirname(self.ci_dir)
105

    
106
        # Read config file
107
        if config_file is None:
108
            config_file = DEFAULT_CONFIG_FILE
109
        if not os.path.isabs(config_file):
110
            config_file = os.path.join(self.ci_dir, config_file)
111

    
112
        self.config = ConfigParser()
113
        self.config.optionxform = str
114
        self.config.read(config_file)
115
        temp_config = self.config.get('Global', 'temporary_config')
116
        if cleanup_config:
117
            try:
118
                os.remove(temp_config)
119
            except:
120
                pass
121
        else:
122
            self.config.read(self.config.get('Global', 'temporary_config'))
123

    
124
        # Set kamaki cloud
125
        if cloud is not None:
126
            self.kamaki_cloud = cloud
127
        elif self.config.has_option("Deployment", "kamaki_cloud"):
128
            kamaki_cloud = self.config.get("Deployment", "kamaki_cloud")
129
            if kamaki_cloud == "":
130
                self.kamaki_cloud = None
131
        else:
132
            self.kamaki_cloud = None
133

    
134
        # Initialize variables
135
        self.fabric_installed = False
136
        self.kamaki_installed = False
137
        self.cyclades_client = None
138
        self.image_client = None
139

    
140
    def setup_kamaki(self):
141
        """Initialize kamaki
142

143
        Setup cyclades_client and image_client
144
        """
145

    
146
        config = kamaki_config.Config()
147
        if self.kamaki_cloud is None:
148
            self.kamaki_cloud = config.get_global("default_cloud")
149

    
150
        self.logger.info("Setup kamaki client, using cloud '%s'.." %
151
                         self.kamaki_cloud)
152
        auth_url = config.get_cloud(self.kamaki_cloud, "url")
153
        self.logger.debug("Authentication URL is %s" % _green(auth_url))
154
        token = config.get_cloud(self.kamaki_cloud, "token")
155
        #self.logger.debug("Token is %s" % _green(token))
156

    
157
        astakos_client = AstakosClient(auth_url, token)
158

    
159
        cyclades_url = \
160
            astakos_client.get_service_endpoints('compute')['publicURL']
161
        self.logger.debug("Cyclades API url is %s" % _green(cyclades_url))
162
        self.cyclades_client = CycladesClient(cyclades_url, token)
163
        self.cyclades_client.CONNECTION_RETRY_LIMIT = 2
164

    
165
        image_url = \
166
            astakos_client.get_service_endpoints('image')['publicURL']
167
        self.logger.debug("Images API url is %s" % _green(image_url))
168
        self.image_client = ImageClient(cyclades_url, token)
169
        self.image_client.CONNECTION_RETRY_LIMIT = 2
170

    
171
    def _wait_transition(self, server_id, current_status, new_status):
172
        """Wait for server to go from current_status to new_status"""
173
        self.logger.debug("Waiting for server to become %s" % new_status)
174
        timeout = self.config.getint('Global', 'build_timeout')
175
        sleep_time = 5
176
        while True:
177
            server = self.cyclades_client.get_server_details(server_id)
178
            if server['status'] == new_status:
179
                return server
180
            elif timeout < 0:
181
                self.logger.error(
182
                    "Waiting for server to become %s timed out" % new_status)
183
                self.destroy_server(False)
184
                sys.exit(-1)
185
            elif server['status'] == current_status:
186
                # Sleep for #n secs and continue
187
                timeout = timeout - sleep_time
188
                time.sleep(sleep_time)
189
            else:
190
                self.logger.error(
191
                    "Server failed with status %s" % server['status'])
192
                self.destroy_server(False)
193
                sys.exit(-1)
194

    
195
    @_check_kamaki
196
    def destroy_server(self, wait=True):
197
        """Destroy slave server"""
198
        server_id = self.config.getint('Temporary Options', 'server_id')
199
        self.logger.info("Destoying server with id %s " % server_id)
200
        self.cyclades_client.delete_server(server_id)
201
        if wait:
202
            self._wait_transition(server_id, "ACTIVE", "DELETED")
203

    
204
    @_check_kamaki
205
    def create_server(self, image_id=None, flavor_id=None):
206
        """Create slave server"""
207
        self.logger.info("Create a new server..")
208
        if image_id is None:
209
            image = self._find_image()
210
            self.logger.debug("Will use image \"%s\"" % _green(image['name']))
211
            image_id = image["id"]
212
        self.logger.debug("Image has id %s" % _green(image_id))
213
        if flavor_id is None:
214
            flavor_id = self.config.getint("Deployment", "flavor_id")
215
        server = self.cyclades_client.create_server(
216
            self.config.get('Deployment', 'server_name'),
217
            flavor_id,
218
            image_id)
219
        server_id = server['id']
220
        self.write_config('server_id', server_id)
221
        self.logger.debug("Server got id %s" % _green(server_id))
222
        server_user = server['metadata']['users']
223
        self.write_config('server_user', server_user)
224
        self.logger.debug("Server's admin user is %s" % _green(server_user))
225
        server_passwd = server['adminPass']
226
        self.write_config('server_passwd', server_passwd)
227

    
228
        server = self._wait_transition(server_id, "BUILD", "ACTIVE")
229
        self._get_server_ip_and_port(server)
230
        self._copy_ssh_keys()
231

    
232
        self.setup_fabric()
233
        self.logger.info("Setup firewall")
234
        accept_ssh_from = self.config.get('Global', 'filter_access_network')
235
        if accept_ssh_from != "":
236
            self.logger.debug("Block ssh except from %s" % accept_ssh_from)
237
            cmd = """
238
            local_ip=$(/sbin/ifconfig eth0 | grep 'inet addr:' | \
239
                cut -d':' -f2 | cut -d' ' -f1)
240
            iptables -A INPUT -s localhost -j ACCEPT
241
            iptables -A INPUT -s $local_ip -j ACCEPT
242
            iptables -A INPUT -s {0} -p tcp --dport 22 -j ACCEPT
243
            iptables -A INPUT -p tcp --dport 22 -j DROP
244
            """.format(accept_ssh_from)
245
            _run(cmd, False)
246

    
247
    def _find_image(self):
248
        """Find a suitable image to use
249

250
        It has to belong to the `system_uuid' user and
251
        contain the word `image_name'
252
        """
253
        system_uuid = self.config.get('Deployment', 'system_uuid')
254
        image_name = self.config.get('Deployment', 'image_name').lower()
255
        images = self.image_client.list_public(detail=True)['images']
256
        # Select images by `system_uuid' user
257
        images = [x for x in images if x['user_id'] == system_uuid]
258
        # Select images with `image_name' in their names
259
        images = \
260
            [x for x in images if x['name'].lower().find(image_name) != -1]
261
        # Let's select the first one
262
        return images[0]
263

    
264
    def _get_server_ip_and_port(self, server):
265
        """Compute server's IPv4 and ssh port number"""
266
        self.logger.info("Get server connection details..")
267
        server_ip = server['attachments'][0]['ipv4']
268
        if ".okeanos.io" in self.cyclades_client.base_url:
269
            tmp1 = int(server_ip.split(".")[2])
270
            tmp2 = int(server_ip.split(".")[3])
271
            server_ip = "gate.okeanos.io"
272
            server_port = 10000 + tmp1 * 256 + tmp2
273
        else:
274
            server_port = 22
275
        self.write_config('server_ip', server_ip)
276
        self.logger.debug("Server's IPv4 is %s" % _green(server_ip))
277
        self.write_config('server_port', server_port)
278
        self.logger.debug("Server's ssh port is %s" % _green(server_port))
279

    
280
    @_check_fabric
281
    def _copy_ssh_keys(self):
282
        if not self.config.has_option("Deployment", "ssh_keys"):
283
            return
284
        authorized_keys = self.config.get("Deployment",
285
                                          "ssh_keys")
286
        if authorized_keys != "" and os.path.exists(authorized_keys):
287
            keyfile = '/tmp/%s.pub' % fabric.env.user
288
            _run('mkdir -p ~/.ssh && chmod 700 ~/.ssh', False)
289
            fabric.put(authorized_keys, keyfile)
290
            _run('cat %s >> ~/.ssh/authorized_keys' % keyfile, False)
291
            _run('rm %s' % keyfile, False)
292
            self.logger.debug("Uploaded ssh authorized keys")
293
        else:
294
            self.logger.debug("No ssh keys found")
295

    
296
    def write_config(self, option, value, section="Temporary Options"):
297
        """Write changes back to config file"""
298
        try:
299
            self.config.add_section(section)
300
        except DuplicateSectionError:
301
            pass
302
        self.config.set(section, option, str(value))
303
        temp_conf_file = self.config.get('Global', 'temporary_config')
304
        with open(temp_conf_file, 'wb') as tcf:
305
            self.config.write(tcf)
306

    
307
    def setup_fabric(self):
308
        """Setup fabric environment"""
309
        self.logger.info("Setup fabric parameters..")
310
        fabric.env.user = self.config.get('Temporary Options', 'server_user')
311
        fabric.env.host_string = \
312
            self.config.get('Temporary Options', 'server_ip')
313
        fabric.env.port = self.config.getint('Temporary Options',
314
                                             'server_port')
315
        fabric.env.password = self.config.get('Temporary Options',
316
                                              'server_passwd')
317
        fabric.env.connection_attempts = 10
318
        fabric.env.shell = "/bin/bash -c"
319
        fabric.env.disable_known_hosts = True
320
        fabric.env.output_prefix = None
321

    
322
    def _check_hash_sum(self, localfile, remotefile):
323
        """Check hash sums of two files"""
324
        self.logger.debug("Check hash sum for local file %s" % localfile)
325
        hash1 = os.popen("sha256sum %s" % localfile).read().split(' ')[0]
326
        self.logger.debug("Local file has sha256 hash %s" % hash1)
327
        self.logger.debug("Check hash sum for remote file %s" % remotefile)
328
        hash2 = _run("sha256sum %s" % remotefile, False)
329
        hash2 = hash2.split(' ')[0]
330
        self.logger.debug("Remote file has sha256 hash %s" % hash2)
331
        if hash1 != hash2:
332
            self.logger.error("Hashes differ.. aborting")
333
            sys.exit(-1)
334

    
335
    @_check_fabric
336
    def clone_repo(self):
337
        """Clone Synnefo repo from slave server"""
338
        self.logger.info("Configure repositories on remote server..")
339
        self.logger.debug("Setup apt, install curl and git")
340
        cmd = """
341
        echo 'APT::Install-Suggests "false";' >> /etc/apt/apt.conf
342
        apt-get update
343
        apt-get install curl git --yes
344
        echo -e "\n\ndeb {0}" >> /etc/apt/sources.list
345
        curl https://dev.grnet.gr/files/apt-grnetdev.pub | apt-key add -
346
        apt-get update
347
        git config --global user.name {1}
348
        git config --global user.mail {2}
349
        """.format(self.config.get('Global', 'apt_repo'),
350
                   self.config.get('Global', 'git_config_name'),
351
                   self.config.get('Global', 'git_config_mail'))
352
        _run(cmd, False)
353

    
354
        synnefo_repo = self.config.get('Global', 'synnefo_repo')
355
        synnefo_branch = self.config.get('Global', 'synnefo_branch')
356
        # Currently clonning synnefo can fail unexpectedly
357
        cloned = False
358
        for i in range(3):
359
            self.logger.debug("Clone synnefo from %s" % synnefo_repo)
360
            cmd = ("git clone --branch %s %s"
361
                   % (synnefo_branch, synnefo_repo))
362
            try:
363
                _run(cmd, False)
364
                cloned = True
365
                break
366
            except:
367
                self.logger.warning("Clonning synnefo failed.. retrying %s"
368
                                    % i)
369
        if not cloned:
370
            self.logger.error("Can not clone Synnefo repo.")
371
            sys.exit(-1)
372

    
373
        deploy_repo = self.config.get('Global', 'deploy_repo')
374
        self.logger.debug("Clone snf-deploy from %s" % deploy_repo)
375
        _run("git clone --depth 1 %s" % deploy_repo, False)
376

    
377
    @_check_fabric
378
    def build_synnefo(self):
379
        """Build Synnefo packages"""
380
        self.logger.info("Build Synnefo packages..")
381
        self.logger.debug("Install development packages")
382
        cmd = """
383
        apt-get update
384
        apt-get install zlib1g-dev dpkg-dev debhelper git-buildpackage \
385
                python-dev python-all python-pip --yes
386
        pip install devflow
387
        """
388
        _run(cmd, False)
389

    
390
        if self.config.get('Global', 'patch_pydist') == "True":
391
            self.logger.debug("Patch pydist.py module")
392
            cmd = r"""
393
            sed -r -i 's/(\(\?P<name>\[A-Za-z\]\[A-Za-z0-9_\.)/\1\\\-/' \
394
                /usr/share/python/debpython/pydist.py
395
            """
396
            _run(cmd, False)
397

398
        self.logger.debug("Build snf-deploy package")
399
        cmd = """
400
        git checkout -t origin/debian
401
        git-buildpackage --git-upstream-branch=master \
402
                --git-debian-branch=debian \
403
                --git-export-dir=../snf-deploy_build-area \
404
                -uc -us
405
        """
406
        with fabric.cd("snf-deploy"):
407
            _run(cmd, True)
408

409
        self.logger.debug("Install snf-deploy package")
410
        cmd = """
411
        dpkg -i snf-deploy*.deb
412
        apt-get -f install --yes
413
        """
414
        with fabric.cd("snf-deploy_build-area"):
415
            with fabric.settings(warn_only=True):
416
                _run(cmd, True)
417

418
        self.logger.debug("Build synnefo packages")
419
        cmd = """
420
        devflow-autopkg snapshot -b ~/synnefo_build-area --no-sign
421
        """
422
        with fabric.cd("synnefo"):
423
            _run(cmd, True)
424

425
        self.logger.debug("Copy synnefo debs to snf-deploy packages dir")
426
        cmd = """
427
        cp ~/synnefo_build-area/*.deb /var/lib/snf-deploy/packages/
428
        """
429
        _run(cmd, False)
430

431
    @_check_fabric
432
    def deploy_synnefo(self):
433
        """Deploy Synnefo using snf-deploy"""
434
        self.logger.info("Deploy Synnefo..")
435
        schema = self.config.get('Global', 'schema')
436
        schema_files = os.path.join(self.ci_dir, "schemas/%s/*" % schema)
437
        self.logger.debug("Will use %s schema" % schema)
438

439
        self.logger.debug("Upload schema files to server")
440
        with fabric.quiet():
441
            fabric.put(schema_files, "/etc/snf-deploy/")
442

443
        self.logger.debug("Change password in nodes.conf file")
444
        cmd = """
445
        sed -i 's/^password =.*/password = {0}/' /etc/snf-deploy/nodes.conf
446
        """.format(fabric.env.password)
447
        _run(cmd, False)
448

449
        self.logger.debug("Run snf-deploy")
450
        cmd = """
451
        snf-deploy all --autoconf
452
        """
453
        _run(cmd, True)
454

455
    @_check_fabric
456
    def unit_test(self):
457
        """Run Synnefo unit test suite"""
458
        self.logger.info("Run Synnefo unit test suite")
459
        component = self.config.get('Unit Tests', 'component')
460

461
        self.logger.debug("Install needed packages")
462
        cmd = """
463
        pip install mock
464
        pip install factory_boy
465
        """
466
        _run(cmd, False)
467

468
        self.logger.debug("Upload tests.sh file")
469
        unit_tests_file = os.path.join(self.ci_dir, "tests.sh")
470
        with fabric.quiet():
471
            fabric.put(unit_tests_file, ".")
472

473
        self.logger.debug("Run unit tests")
474
        cmd = """
475
        bash tests.sh {0}
476
        """.format(component)
477
        _run(cmd, True)
478

479
    @_check_fabric
480
    def run_burnin(self):
481
        """Run burnin functional test suite"""
482
        self.logger.info("Run Burnin functional test suite")
483
        cmd = """
484
        auth_url=$(grep -e '^url =' .kamakirc | cut -d' ' -f3)
485
        token=$(grep -e '^token =' .kamakirc | cut -d' ' -f3)
486
        images_user=$(kamaki image list -l | grep owner | \
487
                      cut -d':' -f2 | tr -d ' ')
488
        snf-burnin --auth-url=$auth_url --token=$token \
489
            --force-flavor=2 --image-id=all \
490
            --system-images-user=$images_user \
491
            {0}
492
        log_folder=$(ls -1d /var/log/burnin/* | tail -n1)
493
        for i in $(ls $log_folder/*/details*); do
494
            echo -e "\\n\\n"
495
            echo -e "***** $i\\n"
496
            cat $i
497
        done
498
        """.format(self.config.get('Burnin', 'cmd_options'))
499
        _run(cmd, True)
500

501
    @_check_fabric
502
    def fetch_packages(self):
503
        """Download Synnefo packages"""
504
        self.logger.info("Download Synnefo packages")
505
        self.logger.debug("Create tarball with packages")
506
        cmd = """
507
        tar czf synnefo_build-area.tgz synnefo_build-area
508
        """
509
        _run(cmd, False)
510

511
        pkgs_dir = self.config.get('Global', 'pkgs_dir')
512
        self.logger.debug("Fetch packages to local dir %s" % pkgs_dir)
513
        os.makedirs(pkgs_dir)
514
        with fabric.quiet():
515
            fabric.get("synnefo_build-area.tgz", pkgs_dir)
516

517
        pkgs_file = os.path.join(pkgs_dir, "synnefo_build-area.tgz")
518
        self._check_hash_sum(pkgs_file, "synnefo_build-area.tgz")
519

520
        self.logger.debug("Untar packages file %s" % pkgs_file)
521
        os.system("cd %s; tar xzf synnefo_build-area.tgz" % pkgs_dir)
522
        self.logger.info("Downloaded debian packages to %s" %
523
                         _green(pkgs_dir))
524