Statistics
| Branch: | Tag: | Revision:

root / ci / utils.py @ 0082c787

History | View | Annotate | Download (18.9 kB)

1
#!/usr/bin/env python
2

    
3
"""
4
Synnefo ci utils module
5
"""
6

    
7
import os
8
import sys
9
import time
10
import logging
11
import fabric.api as fabric
12
from ConfigParser import ConfigParser, DuplicateSectionError
13

    
14
from kamaki.cli import config as kamaki_config
15
from kamaki.clients.astakos import AstakosClient
16
from kamaki.clients.cyclades import CycladesClient
17
from kamaki.clients.image import ImageClient
18

    
19

    
20
def _run(cmd, verbose):
21
    """Run fabric with verbose level"""
22
    if verbose:
23
        args = ('running',)
24
    else:
25
        args = ('running', 'stdout',)
26
    with fabric.hide(*args):
27
        return fabric.run(cmd)
28

    
29

    
30
def _red(msg):
31
    """Red color"""
32
    #return "\x1b[31m" + str(msg) + "\x1b[0m"
33
    return str(msg)
34

    
35

    
36
def _yellow(msg):
37
    """Yellow color"""
38
    #return "\x1b[33m" + str(msg) + "\x1b[0m"
39
    return str(msg)
40

    
41

    
42
def _green(msg):
43
    """Green color"""
44
    #return "\x1b[32m" + str(msg) + "\x1b[0m"
45
    return str(msg)
46

    
47

    
48
def _check_fabric(fun):
49
    """Check if fabric env has been set"""
50
    def wrapper(self, *args, **kwargs):
51
        """wrapper function"""
52
        if not self.fabric_installed:
53
            self.setup_fabric()
54
        return fun(self, *args, **kwargs)
55
    return wrapper
56

    
57

    
58
def _check_kamaki(fun):
59
    """Check if kamaki has been initialized"""
60
    def wrapper(self, *args, **kwargs):
61
        """wrapper function"""
62
        if not self.kamaki_installed:
63
            self.setup_kamaki()
64
        return fun(self, *args, **kwargs)
65
    return wrapper
66

    
67

    
68
class _MyFormatter(logging.Formatter):
69
    """Logging Formatter"""
70
    def format(self, record):
71
        format_orig = self._fmt
72
        if record.levelno == logging.DEBUG:
73
            self._fmt = "  %(msg)s"
74
        elif record.levelno == logging.INFO:
75
            self._fmt = "%(msg)s"
76
        elif record.levelno == logging.WARNING:
77
            self._fmt = _yellow("[W] %(msg)s")
78
        elif record.levelno == logging.ERROR:
79
            self._fmt = _red("[E] %(msg)s")
80
        result = logging.Formatter.format(self, record)
81
        self._fmt = format_orig
82
        return result
83

    
84

    
85
class SynnefoCI(object):
86
    """SynnefoCI python class"""
87

    
88
    def __init__(self, cleanup_config=False, cloud=None):
89
        """ Initialize SynnefoCI python class
90

91
        Setup logger, local_dir, config and kamaki
92
        """
93
        # Setup logger
94
        self.logger = logging.getLogger('synnefo-ci')
95
        self.logger.setLevel(logging.DEBUG)
96
        handler = logging.StreamHandler()
97
        handler.setFormatter(_MyFormatter())
98
        self.logger.addHandler(handler)
99

    
100
        # Get our local dir
101
        self.ci_dir = os.path.dirname(os.path.abspath(__file__))
102
        self.repo_dir = os.path.dirname(self.ci_dir)
103

    
104
        # Read config file
105
        default_conffile = os.path.join(self.ci_dir, "new_config")
106
        self.conffile = os.environ.get("SYNNEFO_CI_CONFIG_FILE",
107
                                       default_conffile)
108

    
109
        self.config = ConfigParser()
110
        self.config.optionxform = str
111
        self.config.read(self.conffile)
112
        temp_config = self.config.get('Global', 'temporary_config')
113
        if cleanup_config:
114
            try:
115
                os.remove(temp_config)
116
            except:
117
                pass
118
        else:
119
            self.config.read(self.config.get('Global', 'temporary_config'))
120

    
121
        # Set kamaki cloud
122
        if cloud is not None:
123
            self.kamaki_cloud = cloud
124
        elif self.config.has_option("Deployment", "kamaki_cloud"):
125
            kamaki_cloud = self.config.get("Deployment", "kamaki_cloud")
126
            if kamaki_cloud == "":
127
                self.kamaki_cloud = None
128
        else:
129
            self.kamaki_cloud = None
130

    
131
        # Initialize variables
132
        self.fabric_installed = False
133
        self.kamaki_installed = False
134
        self.cyclades_client = None
135
        self.image_client = None
136

    
137
    def setup_kamaki(self):
138
        """Initialize kamaki
139

140
        Setup cyclades_client and image_client
141
        """
142

    
143
        config = kamaki_config.Config()
144
        if self.kamaki_cloud is None:
145
            self.kamaki_cloud = config.get_global("default_cloud")
146

    
147
        self.logger.info("Setup kamaki client, using cloud '%s'.." %
148
                         self.kamaki_cloud)
149
        auth_url = config.get_cloud(self.kamaki_cloud, "url")
150
        self.logger.debug("Authentication URL is %s" % _green(auth_url))
151
        token = config.get_cloud(self.kamaki_cloud, "token")
152
        #self.logger.debug("Token is %s" % _green(token))
153

    
154
        astakos_client = AstakosClient(auth_url, token)
155

    
156
        cyclades_url = \
157
            astakos_client.get_service_endpoints('compute')['publicURL']
158
        self.logger.debug("Cyclades API url is %s" % _green(cyclades_url))
159
        self.cyclades_client = CycladesClient(cyclades_url, token)
160
        self.cyclades_client.CONNECTION_RETRY_LIMIT = 2
161

    
162
        image_url = \
163
            astakos_client.get_service_endpoints('image')['publicURL']
164
        self.logger.debug("Images API url is %s" % _green(image_url))
165
        self.image_client = ImageClient(cyclades_url, token)
166
        self.image_client.CONNECTION_RETRY_LIMIT = 2
167

    
168
    def _wait_transition(self, server_id, current_status, new_status):
169
        """Wait for server to go from current_status to new_status"""
170
        self.logger.debug("Waiting for server to become %s" % new_status)
171
        timeout = self.config.getint('Global', 'build_timeout')
172
        sleep_time = 5
173
        while True:
174
            server = self.cyclades_client.get_server_details(server_id)
175
            if server['status'] == new_status:
176
                return server
177
            elif timeout < 0:
178
                self.logger.error(
179
                    "Waiting for server to become %s timed out" % new_status)
180
                self.destroy_server(False)
181
                sys.exit(-1)
182
            elif server['status'] == current_status:
183
                # Sleep for #n secs and continue
184
                timeout = timeout - sleep_time
185
                time.sleep(sleep_time)
186
            else:
187
                self.logger.error(
188
                    "Server failed with status %s" % server['status'])
189
                self.destroy_server(False)
190
                sys.exit(-1)
191

    
192
    @_check_kamaki
193
    def destroy_server(self, wait=True):
194
        """Destroy slave server"""
195
        server_id = self.config.getint('Temporary Options', 'server_id')
196
        self.logger.info("Destoying server with id %s " % server_id)
197
        self.cyclades_client.delete_server(server_id)
198
        if wait:
199
            self._wait_transition(server_id, "ACTIVE", "DELETED")
200

    
201
    @_check_kamaki
202
    def create_server(self, image_id=None, flavor_id=None):
203
        """Create slave server"""
204
        self.logger.info("Create a new server..")
205
        if image_id is None:
206
            image = self._find_image()
207
            self.logger.debug("Will use image \"%s\"" % _green(image['name']))
208
            image_id = image["id"]
209
        self.logger.debug("Image has id %s" % _green(image_id))
210
        if flavor_id is None:
211
            flavor_id = self.config.getint("Deployment", "flavor_id")
212
        server = self.cyclades_client.create_server(
213
            self.config.get('Deployment', 'server_name'),
214
            flavor_id,
215
            image_id)
216
        server_id = server['id']
217
        self.write_config('server_id', server_id)
218
        self.logger.debug("Server got id %s" % _green(server_id))
219
        server_user = server['metadata']['users']
220
        self.write_config('server_user', server_user)
221
        self.logger.debug("Server's admin user is %s" % _green(server_user))
222
        server_passwd = server['adminPass']
223
        self.write_config('server_passwd', server_passwd)
224

    
225
        server = self._wait_transition(server_id, "BUILD", "ACTIVE")
226
        self._get_server_ip_and_port(server)
227
        self._copy_ssh_keys()
228

    
229
        self.setup_fabric()
230
        self.logger.info("Setup firewall")
231
        accept_ssh_from = self.config.get('Global', 'filter_access_network')
232
        if accept_ssh_from != "":
233
            self.logger.debug("Block ssh except from %s" % accept_ssh_from)
234
            cmd = """
235
            local_ip=$(/sbin/ifconfig eth0 | grep 'inet addr:' | \
236
                cut -d':' -f2 | cut -d' ' -f1)
237
            iptables -A INPUT -s localhost -j ACCEPT
238
            iptables -A INPUT -s $local_ip -j ACCEPT
239
            iptables -A INPUT -s {0} -p tcp --dport 22 -j ACCEPT
240
            iptables -A INPUT -p tcp --dport 22 -j DROP
241
            """.format(accept_ssh_from)
242
            _run(cmd, False)
243

    
244
    def _find_image(self):
245
        """Find a suitable image to use
246

247
        It has to belong to the `system_uuid' user and
248
        contain the word `image_name'
249
        """
250
        system_uuid = self.config.get('Deployment', 'system_uuid')
251
        image_name = self.config.get('Deployment', 'image_name').lower()
252
        images = self.image_client.list_public(detail=True)['images']
253
        # Select images by `system_uuid' user
254
        images = [x for x in images if x['user_id'] == system_uuid]
255
        # Select images with `image_name' in their names
256
        images = \
257
            [x for x in images if x['name'].lower().find(image_name) != -1]
258
        # Let's select the first one
259
        return images[0]
260

    
261
    def _get_server_ip_and_port(self, server):
262
        """Compute server's IPv4 and ssh port number"""
263
        self.logger.info("Get server connection details..")
264
        server_ip = server['attachments'][0]['ipv4']
265
        if ".okeanos.io" in self.cyclades_client.base_url:
266
            tmp1 = int(server_ip.split(".")[2])
267
            tmp2 = int(server_ip.split(".")[3])
268
            server_ip = "gate.okeanos.io"
269
            server_port = 10000 + tmp1 * 256 + tmp2
270
        else:
271
            server_port = 22
272
        self.write_config('server_ip', server_ip)
273
        self.logger.debug("Server's IPv4 is %s" % _green(server_ip))
274
        self.write_config('server_port', server_port)
275
        self.logger.debug("Server's ssh port is %s" % _green(server_port))
276

    
277
    @_check_fabric
278
    def _copy_ssh_keys(self):
279
        if not self.config.has_option("Deployment", "ssh_keys"):
280
            return
281
        authorized_keys = self.config.get("Deployment",
282
                                          "ssh_keys")
283
        if authorized_keys != "" and os.path.exists(authorized_keys):
284
            keyfile = '/tmp/%s.pub' % fabric.env.user
285
            _run('mkdir -p ~/.ssh && chmod 700 ~/.ssh', False)
286
            fabric.put(authorized_keys, keyfile)
287
            _run('cat %s >> ~/.ssh/authorized_keys' % keyfile, False)
288
            _run('rm %s' % keyfile, False)
289
            self.logger.debug("Uploaded ssh authorized keys")
290
        else:
291
            self.logger.debug("No ssh keys found")
292

    
293
    def write_config(self, option, value, section="Temporary Options"):
294
        """Write changes back to config file"""
295
        try:
296
            self.config.add_section(section)
297
        except DuplicateSectionError:
298
            pass
299
        self.config.set(section, option, str(value))
300
        temp_conf_file = self.config.get('Global', 'temporary_config')
301
        with open(temp_conf_file, 'wb') as tcf:
302
            self.config.write(tcf)
303

    
304
    def setup_fabric(self):
305
        """Setup fabric environment"""
306
        self.logger.info("Setup fabric parameters..")
307
        fabric.env.user = self.config.get('Temporary Options', 'server_user')
308
        fabric.env.host_string = \
309
            self.config.get('Temporary Options', 'server_ip')
310
        fabric.env.port = self.config.getint('Temporary Options',
311
                                             'server_port')
312
        fabric.env.password = self.config.get('Temporary Options',
313
                                              'server_passwd')
314
        fabric.env.connection_attempts = 10
315
        fabric.env.shell = "/bin/bash -c"
316
        fabric.env.disable_known_hosts = True
317
        fabric.env.output_prefix = None
318

    
319
    def _check_hash_sum(self, localfile, remotefile):
320
        """Check hash sums of two files"""
321
        self.logger.debug("Check hash sum for local file %s" % localfile)
322
        hash1 = os.popen("sha256sum %s" % localfile).read().split(' ')[0]
323
        self.logger.debug("Local file has sha256 hash %s" % hash1)
324
        self.logger.debug("Check hash sum for remote file %s" % remotefile)
325
        hash2 = _run("sha256sum %s" % remotefile, False)
326
        hash2 = hash2.split(' ')[0]
327
        self.logger.debug("Remote file has sha256 hash %s" % hash2)
328
        if hash1 != hash2:
329
            self.logger.error("Hashes differ.. aborting")
330
            sys.exit(-1)
331

    
332
    @_check_fabric
333
    def clone_repo(self):
334
        """Clone Synnefo repo from slave server"""
335
        self.logger.info("Configure repositories on remote server..")
336
        self.logger.debug("Setup apt, install curl and git")
337
        cmd = """
338
        echo 'APT::Install-Suggests "false";' >> /etc/apt/apt.conf
339
        apt-get update
340
        apt-get install curl git --yes
341
        echo -e "\n\ndeb {0}" >> /etc/apt/sources.list
342
        curl https://dev.grnet.gr/files/apt-grnetdev.pub | apt-key add -
343
        apt-get update
344
        git config --global user.name {1}
345
        git config --global user.mail {2}
346
        """.format(self.config.get('Global', 'apt_repo'),
347
                   self.config.get('Global', 'git_config_name'),
348
                   self.config.get('Global', 'git_config_mail'))
349
        _run(cmd, False)
350

    
351
        synnefo_repo = self.config.get('Global', 'synnefo_repo')
352
        synnefo_branch = self.config.get('Global', 'synnefo_branch')
353
        # Currently clonning synnefo can fail unexpectedly
354
        cloned = False
355
        for i in range(3):
356
            self.logger.debug("Clone synnefo from %s" % synnefo_repo)
357
            cmd = ("git clone --branch %s %s"
358
                   % (synnefo_branch, synnefo_repo))
359
            try:
360
                _run(cmd, False)
361
                cloned = True
362
                break
363
            except:
364
                self.logger.warning("Clonning synnefo failed.. retrying %s"
365
                                    % i)
366
        if not cloned:
367
            self.logger.error("Can not clone Synnefo repo.")
368
            sys.exit(-1)
369

    
370
        deploy_repo = self.config.get('Global', 'deploy_repo')
371
        self.logger.debug("Clone snf-deploy from %s" % deploy_repo)
372
        _run("git clone --depth 1 %s" % deploy_repo, False)
373

    
374
    @_check_fabric
375
    def build_synnefo(self):
376
        """Build Synnefo packages"""
377
        self.logger.info("Build Synnefo packages..")
378
        self.logger.debug("Install development packages")
379
        cmd = """
380
        apt-get update
381
        apt-get install zlib1g-dev dpkg-dev debhelper git-buildpackage \
382
                python-dev python-all python-pip --yes
383
        pip install devflow
384
        """
385
        _run(cmd, False)
386

    
387
        if self.config.get('Global', 'patch_pydist') == "True":
388
            self.logger.debug("Patch pydist.py module")
389
            cmd = r"""
390
            sed -r -i 's/(\(\?P<name>\[A-Za-z\]\[A-Za-z0-9_\.)/\1\\\-/' \
391
                /usr/share/python/debpython/pydist.py
392
            """
393
            _run(cmd, False)
394

395
        self.logger.debug("Build snf-deploy package")
396
        cmd = """
397
        git checkout -t origin/debian
398
        git-buildpackage --git-upstream-branch=master \
399
                --git-debian-branch=debian \
400
                --git-export-dir=../snf-deploy_build-area \
401
                -uc -us
402
        """
403
        with fabric.cd("snf-deploy"):
404
            _run(cmd, True)
405

406
        self.logger.debug("Install snf-deploy package")
407
        cmd = """
408
        dpkg -i snf-deploy*.deb
409
        apt-get -f install --yes
410
        """
411
        with fabric.cd("snf-deploy_build-area"):
412
            with fabric.settings(warn_only=True):
413
                _run(cmd, True)
414

415
        self.logger.debug("Build synnefo packages")
416
        cmd = """
417
        devflow-autopkg snapshot -b ~/synnefo_build-area --no-sign
418
        """
419
        with fabric.cd("synnefo"):
420
            _run(cmd, True)
421

422
        self.logger.debug("Copy synnefo debs to snf-deploy packages dir")
423
        cmd = """
424
        cp ~/synnefo_build-area/*.deb /var/lib/snf-deploy/packages/
425
        """
426
        _run(cmd, False)
427

428
    @_check_fabric
429
    def deploy_synnefo(self):
430
        """Deploy Synnefo using snf-deploy"""
431
        self.logger.info("Deploy Synnefo..")
432
        schema = self.config.get('Global', 'schema')
433
        schema_files = os.path.join(self.ci_dir, "schemas/%s/*" % schema)
434
        self.logger.debug("Will use %s schema" % schema)
435

436
        self.logger.debug("Upload schema files to server")
437
        with fabric.quiet():
438
            fabric.put(schema_files, "/etc/snf-deploy/")
439

440
        self.logger.debug("Change password in nodes.conf file")
441
        cmd = """
442
        sed -i 's/^password =.*/password = {0}/' /etc/snf-deploy/nodes.conf
443
        """.format(fabric.env.password)
444
        _run(cmd, False)
445

446
        self.logger.debug("Run snf-deploy")
447
        cmd = """
448
        snf-deploy all --autoconf
449
        """
450
        _run(cmd, True)
451

452
    @_check_fabric
453
    def unit_test(self):
454
        """Run Synnefo unit test suite"""
455
        self.logger.info("Run Synnefo unit test suite")
456
        component = self.config.get('Unit Tests', 'component')
457

458
        self.logger.debug("Install needed packages")
459
        cmd = """
460
        pip install mock
461
        pip install factory_boy
462
        """
463
        _run(cmd, False)
464

465
        self.logger.debug("Upload tests.sh file")
466
        unit_tests_file = os.path.join(self.ci_dir, "tests.sh")
467
        with fabric.quiet():
468
            fabric.put(unit_tests_file, ".")
469

470
        self.logger.debug("Run unit tests")
471
        cmd = """
472
        bash tests.sh {0}
473
        """.format(component)
474
        _run(cmd, True)
475

476
    @_check_fabric
477
    def run_burnin(self):
478
        """Run burnin functional test suite"""
479
        self.logger.info("Run Burnin functional test suite")
480
        cmd = """
481
        auth_url=$(grep -e '^url =' .kamakirc | cut -d' ' -f3)
482
        token=$(grep -e '^token =' .kamakirc | cut -d' ' -f3)
483
        images_user=$(kamaki image list -l | grep owner | \
484
                      cut -d':' -f2 | tr -d ' ')
485
        snf-burnin --auth-url=$auth_url --token=$token \
486
            --force-flavor=2 --image-id=all \
487
            --system-images-user=$images_user \
488
            {0}
489
        log_folder=$(ls -1d /var/log/burnin/* | tail -n1)
490
        for i in $(ls $log_folder/*/details*); do
491
            echo -e "\\n\\n"
492
            echo -e "***** $i\\n"
493
            cat $i
494
        done
495
        """.format(self.config.get('Burnin', 'cmd_options'))
496
        _run(cmd, True)
497

498
    @_check_fabric
499
    def fetch_packages(self):
500
        """Download Synnefo packages"""
501
        self.logger.info("Download Synnefo packages")
502
        self.logger.debug("Create tarball with packages")
503
        cmd = """
504
        tar czf synnefo_build-area.tgz synnefo_build-area
505
        """
506
        _run(cmd, False)
507

508
        pkgs_dir = self.config.get('Global', 'pkgs_dir')
509
        self.logger.debug("Fetch packages to local dir %s" % pkgs_dir)
510
        os.makedirs(pkgs_dir)
511
        with fabric.quiet():
512
            fabric.get("synnefo_build-area.tgz", pkgs_dir)
513

514
        pkgs_file = os.path.join(pkgs_dir, "synnefo_build-area.tgz")
515
        self._check_hash_sum(pkgs_file, "synnefo_build-area.tgz")
516

517
        self.logger.debug("Untar packages file %s" % pkgs_file)
518
        os.system("cd %s; tar xzf synnefo_build-area.tgz" % pkgs_dir)
519