Statistics
| Branch: | Tag: | Revision:

root / ci / utils.py @ 2255812d

History | View | Annotate | Download (17.8 kB)

1
#!/usr/bin/env python
2

    
3
"""
4
Synnefo ci utils module
5
"""
6

    
7
import os
8
import sys
9
import time
10
import logging
11
import fabric.api as fabric
12
from ConfigParser import ConfigParser, DuplicateSectionError
13

    
14
from kamaki.clients.astakos import AstakosClient
15
from kamaki.clients.cyclades import CycladesClient
16
from kamaki.clients.image import ImageClient
17

    
18

    
19
def _run(cmd, verbose):
20
    """Run fabric with verbose level"""
21
    if verbose:
22
        args = ('running',)
23
    else:
24
        args = ('running', 'stdout',)
25
    with fabric.hide(*args):
26
        return fabric.run(cmd)
27

    
28

    
29
def _red(msg):
30
    """Red color"""
31
    #return "\x1b[31m" + str(msg) + "\x1b[0m"
32
    return str(msg)
33

    
34

    
35
def _yellow(msg):
36
    """Yellow color"""
37
    #return "\x1b[33m" + str(msg) + "\x1b[0m"
38
    return str(msg)
39

    
40

    
41
def _green(msg):
42
    """Green color"""
43
    #return "\x1b[32m" + str(msg) + "\x1b[0m"
44
    return str(msg)
45

    
46

    
47
def _check_fabric(fun):
48
    """Check if fabric env has been set"""
49
    def wrapper(self, *args):
50
        """wrapper function"""
51
        if not self.fabric_installed:
52
            self.setup_fabric()
53
        return fun(self, *args)
54
    return wrapper
55

    
56

    
57
def _check_kamaki(fun):
58
    """Check if kamaki has been initialized"""
59
    def wrapper(self, *args):
60
        """wrapper function"""
61
        if not self.kamaki_installed:
62
            self.setup_kamaki()
63
        return fun(self, *args)
64
    return wrapper
65

    
66

    
67
class _MyFormatter(logging.Formatter):
68
    """Logging Formatter"""
69
    def format(self, record):
70
        format_orig = self._fmt
71
        if record.levelno == logging.DEBUG:
72
            self._fmt = "  %(msg)s"
73
        elif record.levelno == logging.INFO:
74
            self._fmt = "%(msg)s"
75
        elif record.levelno == logging.WARNING:
76
            self._fmt = _yellow("[W] %(msg)s")
77
        elif record.levelno == logging.ERROR:
78
            self._fmt = _red("[E] %(msg)s")
79
        result = logging.Formatter.format(self, record)
80
        self._fmt = format_orig
81
        return result
82

    
83

    
84
class SynnefoCI(object):
85
    """SynnefoCI python class"""
86

    
87
    def __init__(self, cleanup_config=False):
88
        """ Initialize SynnefoCI python class
89

90
        Setup logger, local_dir, config and kamaki
91
        """
92
        # Setup logger
93
        self.logger = logging.getLogger('synnefo-ci')
94
        self.logger.setLevel(logging.DEBUG)
95
        handler = logging.StreamHandler()
96
        handler.setFormatter(_MyFormatter())
97
        self.logger.addHandler(handler)
98

    
99
        # Get our local dir
100
        self.ci_dir = os.path.dirname(os.path.abspath(__file__))
101
        self.repo_dir = os.path.dirname(self.ci_dir)
102

    
103
        # Read config file
104
        self.conffile = os.path.join(self.ci_dir, "new_config")
105
        self.config = ConfigParser()
106
        self.config.optionxform = str
107
        self.config.read(self.conffile)
108
        temp_config = self.config.get('Global', 'temporary_config')
109
        if cleanup_config:
110
            try:
111
                os.remove(temp_config)
112
            except:
113
                pass
114
        else:
115
            self.config.read(self.config.get('Global', 'temporary_config'))
116

    
117
        # Initialize variables
118
        self.fabric_installed = False
119
        self.kamaki_installed = False
120
        self.cyclades_client = None
121
        self.image_client = None
122

    
123
    def setup_kamaki(self):
124
        """Initialize kamaki
125

126
        Setup cyclades_client and image_client
127
        """
128
        self.logger.info("Setup kamaki client..")
129
        auth_url = self.config.get('Deployment', 'auth_url')
130
        self.logger.debug("Authentication URL is %s" % _green(auth_url))
131
        token = self.config.get('Deployment', 'token')
132
        #self.logger.debug("Token is %s" % _green(token))
133

    
134
        astakos_client = AstakosClient(auth_url, token)
135

    
136
        cyclades_url = \
137
            astakos_client.get_service_endpoints('compute')['publicURL']
138
        self.logger.debug("Cyclades API url is %s" % _green(cyclades_url))
139
        self.cyclades_client = CycladesClient(cyclades_url, token)
140
        self.cyclades_client.CONNECTION_RETRY_LIMIT = 2
141

    
142
        image_url = \
143
            astakos_client.get_service_endpoints('image')['publicURL']
144
        self.logger.debug("Images API url is %s" % _green(image_url))
145
        self.image_client = ImageClient(cyclades_url, token)
146
        self.image_client.CONNECTION_RETRY_LIMIT = 2
147

    
148
    def _wait_transition(self, server_id, current_status, new_status):
149
        """Wait for server to go from current_status to new_status"""
150
        self.logger.debug("Waiting for server to become %s" % new_status)
151
        timeout = self.config.getint('Global', 'build_timeout')
152
        sleep_time = 5
153
        while True:
154
            server = self.cyclades_client.get_server_details(server_id)
155
            if server['status'] == new_status:
156
                return server
157
            elif timeout < 0:
158
                self.logger.error(
159
                    "Waiting for server to become %s timed out" % new_status)
160
                self.destroy_server(False)
161
                sys.exit(-1)
162
            elif server['status'] == current_status:
163
                # Sleep for #n secs and continue
164
                timeout = timeout - sleep_time
165
                time.sleep(sleep_time)
166
            else:
167
                self.logger.error(
168
                    "Server failed with status %s" % server['status'])
169
                self.destroy_server(False)
170
                sys.exit(-1)
171

    
172
    @_check_kamaki
173
    def destroy_server(self, wait=True):
174
        """Destroy slave server"""
175
        server_id = self.config.getint('Temporary Options', 'server_id')
176
        self.logger.info("Destoying server with id %s " % server_id)
177
        self.cyclades_client.delete_server(server_id)
178
        if wait:
179
            self._wait_transition(server_id, "ACTIVE", "DELETED")
180

    
181
    @_check_kamaki
182
    def create_server(self):
183
        """Create slave server"""
184
        self.logger.info("Create a new server..")
185
        image = self._find_image()
186
        self.logger.debug("Will use image \"%s\"" % _green(image['name']))
187
        self.logger.debug("Image has id %s" % _green(image['id']))
188
        server = self.cyclades_client.create_server(
189
            self.config.get('Deployment', 'server_name'),
190
            self.config.getint('Deployment', 'flavor_id'),
191
            image['id'])
192
        server_id = server['id']
193
        self.write_config('server_id', server_id)
194
        self.logger.debug("Server got id %s" % _green(server_id))
195
        server_user = server['metadata']['users']
196
        self.write_config('server_user', server_user)
197
        self.logger.debug("Server's admin user is %s" % _green(server_user))
198
        server_passwd = server['adminPass']
199
        self.write_config('server_passwd', server_passwd)
200
        self.logger.debug(
201
            "Server's admin password is %s" % _green(server_passwd))
202

    
203
        server = self._wait_transition(server_id, "BUILD", "ACTIVE")
204
        self._get_server_ip_and_port(server)
205

    
206
        self.setup_fabric()
207
        self.logger.info("Setup firewall")
208
        accept_ssh_from = self.config.get('Global', 'filter_access_network')
209
        self.logger.debug("Block ssh except from %s" % accept_ssh_from)
210
        cmd = """
211
        local_ip=/sbin/ifconfig eth0 | grep 'inet addr:' | \
212
            cut -d':' -f2 | cut -d' ' -f1
213
        iptables -A INPUT -s localhost -j ACCEPT
214
        iptables -A INPUT -s $local_ip -j ACCEPT
215
        iptables -A INPUT -s {0} -p tcp --dport 22 -j ACCEPT
216
        iptables -A INPUT -p tcp --dport 22 -j DROP
217
        """.format(accept_ssh_from)
218
        _run(cmd, False)
219

    
220
    def _find_image(self):
221
        """Find a suitable image to use
222

223
        It has to belong to the `system_uuid' user and
224
        contain the word `image_name'
225
        """
226
        system_uuid = self.config.get('Deployment', 'system_uuid')
227
        image_name = self.config.get('Deployment', 'image_name').lower()
228
        images = self.image_client.list_public(detail=True)['images']
229
        # Select images by `system_uuid' user
230
        images = [x for x in images if x['user_id'] == system_uuid]
231
        # Select images with `image_name' in their names
232
        images = \
233
            [x for x in images if x['name'].lower().find(image_name) != -1]
234
        # Let's select the first one
235
        return images[0]
236

    
237
    def _get_server_ip_and_port(self, server):
238
        """Compute server's IPv4 and ssh port number"""
239
        self.logger.info("Get server connection details..")
240
        # XXX: check if this IP is from public network
241
        server_ip = server['attachments'][0]['ipv4']
242
        if eval(self.config.get('Deployment', 'deploy_on_io')):
243
            tmp1 = int(server_ip.split(".")[2])
244
            tmp2 = int(server_ip.split(".")[3])
245
            server_ip = "gate.okeanos.io"
246
            server_port = 10000 + tmp1 * 256 + tmp2
247
        else:
248
            server_port = 22
249
        self.write_config('server_ip', server_ip)
250
        self.logger.debug("Server's IPv4 is %s" % _green(server_ip))
251
        self.write_config('server_port', server_port)
252
        self.logger.debug("Server's ssh port is %s" % _green(server_port))
253

    
254
    def write_config(self, option, value, section="Temporary Options"):
255
        """Write changes back to config file"""
256
        try:
257
            self.config.add_section(section)
258
        except DuplicateSectionError:
259
            pass
260
        self.config.set(section, option, str(value))
261
        temp_conf_file = self.config.get('Global', 'temporary_config')
262
        with open(temp_conf_file, 'wb') as tcf:
263
            self.config.write(tcf)
264

    
265
    def setup_fabric(self):
266
        """Setup fabric environment"""
267
        self.logger.info("Setup fabric parameters..")
268
        fabric.env.user = self.config.get('Temporary Options', 'server_user')
269
        fabric.env.host_string = \
270
            self.config.get('Temporary Options', 'server_ip')
271
        fabric.env.port = self.config.getint('Temporary Options', 'server_port')
272
        fabric.env.password = \
273
            self.config.get('Temporary Options', 'server_passwd')
274
        fabric.env.connection_attempts = 10
275
        fabric.env.shell = "/bin/bash -c"
276
        fabric.env.disable_known_hosts = True
277
        fabric.env.output_prefix = None
278

    
279
    def _check_hash_sum(self, localfile, remotefile):
280
        """Check hash sums of two files"""
281
        self.logger.debug("Check hash sum for local file %s" % localfile)
282
        hash1 = os.popen("sha256sum %s" % localfile).read().split(' ')[0]
283
        self.logger.debug("Local file has sha256 hash %s" % hash1)
284
        self.logger.debug("Check hash sum for remote file %s" % remotefile)
285
        hash2 = _run("sha256sum %s" % remotefile, False)
286
        hash2 = hash2.split(' ')[0]
287
        self.logger.debug("Remote file has sha256 hash %s" % hash2)
288
        if hash1 != hash2:
289
            self.logger.error("Hashes differ.. aborting")
290
            sys.exit(-1)
291

    
292
    @_check_fabric
293
    def clone_repo(self):
294
        """Clone Synnefo repo from slave server"""
295
        self.logger.info("Configure repositories on remote server..")
296
        self.logger.debug("Setup apt, install curl and git")
297
        cmd = """
298
        echo 'APT::Install-Suggests "false";' >> /etc/apt/apt.conf
299
        apt-get update
300
        apt-get install curl git --yes
301
        echo -e "\n\ndeb {0}" >> /etc/apt/sources.list
302
        curl https://dev.grnet.gr/files/apt-grnetdev.pub | apt-key add -
303
        apt-get update
304
        git config --global user.name {1}
305
        git config --global user.mail {2}
306
        """.format(self.config.get('Global', 'apt_repo'),
307
                   self.config.get('Global', 'git_config_name'),
308
                   self.config.get('Global', 'git_config_mail'))
309
        _run(cmd, False)
310

    
311
        synnefo_repo = self.config.get('Global', 'synnefo_repo')
312
        # Currently clonning synnefo can fail unexpectedly
313
        for i in range(3):
314
            self.logger.debug("Clone synnefo from %s" % synnefo_repo)
315
            try:
316
                _run("git clone %s" % synnefo_repo, False)
317
                break
318
            except:
319
                self.logger.warning("Clonning synnefo failed.. retrying %s" % i)
320

    
321
        synnefo_branch = self.config.get('Global', 'synnefo_branch')
322
        if synnefo_branch == "HEAD":
323
            # Get current branch
324
            synnefo_branch = os.popen("git rev-parse HEAD").read().strip()
325
            self.logger.debug(
326
                "Checkout %s in feature-ci branch" % synnefo_branch)
327
            with fabric.cd("synnefo"):
328
                _run("git checkout -b feature-ci %s" % synnefo_branch, False)
329
        elif synnefo_branch == "origin/master":
330
            pass
331
        elif "origin" in synnefo_branch:
332
            self.logger.debug("Checkout %s branch" % synnefo_branch)
333
            with fabric.cd("synnefo"):
334
                _run("git checkout -t %s" % synnefo_branch, False)
335
        else:
336
            self.logger.debug(
337
                "Checkout %s in feature-ci branch" % synnefo_branch)
338
            with fabric.cd("synnefo"):
339
                _run("git checkout -b feature-ci %s" % synnefo_branch, False)
340

    
341
        deploy_repo = self.config.get('Global', 'deploy_repo')
342
        self.logger.debug("Clone snf-deploy from %s" % deploy_repo)
343
        _run("git clone %s" % deploy_repo, False)
344

    
345
    @_check_fabric
346
    def build_synnefo(self):
347
        """Build Synnefo packages"""
348
        self.logger.info("Build Synnefo packages..")
349
        self.logger.debug("Install development packages")
350
        cmd = """
351
        apt-get update
352
        apt-get install zlib1g-dev dpkg-dev debhelper git-buildpackage \
353
                python-dev python-all python-pip --yes
354
        pip install devflow
355
        """
356
        _run(cmd, False)
357

    
358
        if eval(self.config.get('Global', 'patch_pydist')):
359
            self.logger.debug("Patch pydist.py module")
360
            cmd = r"""
361
            sed -r -i 's/(\(\?P<name>\[A-Za-z\]\[A-Za-z0-9_\.)/\1\\\-/' \
362
                /usr/share/python/debpython/pydist.py
363
            """
364
            _run(cmd, False)
365

366
        self.logger.debug("Build snf-deploy package")
367
        cmd = """
368
        git checkout -t origin/debian
369
        git-buildpackage --git-upstream-branch=master \
370
                --git-debian-branch=debian \
371
                --git-export-dir=../snf-deploy_build-area \
372
                -uc -us
373
        """
374
        with fabric.cd("snf-deploy"):
375
            _run(cmd, True)
376

377
        self.logger.debug("Install snf-deploy package")
378
        cmd = """
379
        dpkg -i snf-deploy*.deb
380
        apt-get -f install --yes
381
        """
382
        with fabric.cd("snf-deploy_build-area"):
383
            with fabric.settings(warn_only=True):
384
                _run(cmd, True)
385

386
        self.logger.debug("Build synnefo packages")
387
        cmd = """
388
        devflow-autopkg snapshot -b ~/synnefo_build-area --no-sign
389
        """
390
        with fabric.cd("synnefo"):
391
            _run(cmd, True)
392

393
        self.logger.debug("Copy synnefo debs to snf-deploy packages dir")
394
        cmd = """
395
        cp ~/synnefo_build-area/*.deb /var/lib/snf-deploy/packages/
396
        """
397
        _run(cmd, False)
398

399
    @_check_fabric
400
    def deploy_synnefo(self):
401
        """Deploy Synnefo using snf-deploy"""
402
        self.logger.info("Deploy Synnefo..")
403
        schema = self.config.get('Global', 'schema')
404
        schema_files = os.path.join(self.ci_dir, "schemas/%s/*" % schema)
405
        self.logger.debug("Will use %s schema" % schema)
406

407
        self.logger.debug("Upload schema files to server")
408
        with fabric.quiet():
409
            fabric.put(schema_files, "/etc/snf-deploy/")
410

411
        self.logger.debug("Change password in nodes.conf file")
412
        cmd = """
413
        sed -i 's/^password =.*/password = {0}/' /etc/snf-deploy/nodes.conf
414
        """.format(fabric.env.password)
415
        _run(cmd, False)
416

417
        self.logger.debug("Run snf-deploy")
418
        cmd = """
419
        snf-deploy all --autoconf
420
        """
421
        _run(cmd, True)
422

423
    @_check_fabric
424
    def unit_test(self):
425
        """Run Synnefo unit test suite"""
426
        self.logger.info("Run Synnefo unit test suite")
427
        component = self.config.get('Unit Tests', 'component')
428

429
        self.logger.debug("Install needed packages")
430
        cmd = """
431
        pip install mock
432
        pip install factory_boy
433
        """
434
        _run(cmd, False)
435

436
        self.logger.debug("Upload tests.sh file")
437
        unit_tests_file = os.path.join(self.ci_dir, "tests.sh")
438
        with fabric.quiet():
439
            fabric.put(unit_tests_file, ".")
440

441
        self.logger.debug("Run unit tests")
442
        cmd = """
443
        bash tests.sh {0}
444
        """.format(component)
445
        _run(cmd, True)
446

447
    @_check_fabric
448
    def run_burnin(self):
449
        """Run burnin functional test suite"""
450
        self.logger.info("Run Burnin functional test suite")
451
        cmd = """
452
        auth_url=$(grep -e '^url =' .kamakirc | cut -d' ' -f3)
453
        token=$(grep -e '^token =' .kamakirc | cut -d' ' -f3)
454
        images_user=$(kamaki image list -l | grep owner | \
455
                      cut -d':' -f2 | tr -d ' ')
456
        snf-burnin --auth-url=$auth_url --token=$token \
457
            --force-flavor=2 --image-id=all \
458
            --system-images-user=$images_user \
459
            {0}
460
        log_folder=$(ls -1d /var/log/burnin/* | tail -n1)
461
        for i in $(ls $log_folder/*/details*); do
462
            echo -e "\\n\\n"
463
            echo -e "***** $i\\n"
464
            cat $i
465
        done
466
        """.format(self.config.get('Burnin', 'cmd_options'))
467
        _run(cmd, True)
468

469
    @_check_fabric
470
    def fetch_packages(self):
471
        """Download Synnefo packages"""
472
        self.logger.info("Download Synnefo packages")
473
        self.logger.debug("Create tarball with packages")
474
        cmd = """
475
        tar czf synnefo_build-area.tgz synnefo_build-area
476
        """
477
        _run(cmd, False)
478

479
        pkgs_dir = self.config.get('Global', 'pkgs_dir')
480
        self.logger.debug("Fetch packages to local dir %s" % pkgs_dir)
481
        os.makedirs(pkgs_dir)
482
        with fabric.quiet():
483
            fabric.get("synnefo_build-area.tgz", pkgs_dir)
484

485
        pkgs_file = os.path.join(pkgs_dir, "synnefo_build-area.tgz")
486
        self._check_hash_sum(pkgs_file, "synnefo_build-area.tgz")
487

488
        self.logger.debug("Untar packages file %s" % pkgs_file)
489
        os.system("cd %s; tar xzf synnefo_build-area.tgz" % pkgs_dir)
490