Statistics
| Branch: | Tag: | Revision:

root / ci / utils.py @ 5c3b5c9f

History | View | Annotate | Download (17.9 kB)

1
#!/usr/bin/env python
2

    
3
"""
4
Synnefo ci utils module
5
"""
6

    
7
import os
8
import sys
9
import time
10
import logging
11
import fabric.api as fabric
12
from ConfigParser import ConfigParser, DuplicateSectionError
13

    
14
from kamaki.clients.astakos import AstakosClient
15
from kamaki.clients.cyclades import CycladesClient
16
from kamaki.clients.image import ImageClient
17

    
18

    
19
def _run(cmd, verbose):
20
    """Run fabric with verbose level"""
21
    if verbose:
22
        args = ('running',)
23
    else:
24
        args = ('running', 'stdout',)
25
    with fabric.hide(*args):
26
        return fabric.run(cmd)
27

    
28

    
29
def _red(msg):
30
    """Red color"""
31
    #return "\x1b[31m" + str(msg) + "\x1b[0m"
32
    return str(msg)
33

    
34

    
35
def _yellow(msg):
36
    """Yellow color"""
37
    #return "\x1b[33m" + str(msg) + "\x1b[0m"
38
    return str(msg)
39

    
40

    
41
def _green(msg):
42
    """Green color"""
43
    #return "\x1b[32m" + str(msg) + "\x1b[0m"
44
    return str(msg)
45

    
46

    
47
def _check_fabric(fun):
48
    """Check if fabric env has been set"""
49
    def wrapper(self, *args):
50
        """wrapper function"""
51
        if not self.fabric_installed:
52
            self.setup_fabric()
53
        return fun(self, *args)
54
    return wrapper
55

    
56

    
57
def _check_kamaki(fun):
58
    """Check if kamaki has been initialized"""
59
    def wrapper(self, *args):
60
        """wrapper function"""
61
        if not self.kamaki_installed:
62
            self.setup_kamaki()
63
        return fun(self, *args)
64
    return wrapper
65

    
66

    
67
class _MyFormatter(logging.Formatter):
68
    """Logging Formatter"""
69
    def format(self, record):
70
        format_orig = self._fmt
71
        if record.levelno == logging.DEBUG:
72
            self._fmt = "  %(msg)s"
73
        elif record.levelno == logging.INFO:
74
            self._fmt = "%(msg)s"
75
        elif record.levelno == logging.WARNING:
76
            self._fmt = _yellow("[W] %(msg)s")
77
        elif record.levelno == logging.ERROR:
78
            self._fmt = _red("[E] %(msg)s")
79
        result = logging.Formatter.format(self, record)
80
        self._fmt = format_orig
81
        return result
82

    
83

    
84
class SynnefoCI(object):
85
    """SynnefoCI python class"""
86

    
87
    def __init__(self, cleanup_config=False):
88
        """ Initialize SynnefoCI python class
89

90
        Setup logger, local_dir, config and kamaki
91
        """
92
        # Setup logger
93
        self.logger = logging.getLogger('synnefo-ci')
94
        self.logger.setLevel(logging.DEBUG)
95
        handler = logging.StreamHandler()
96
        handler.setFormatter(_MyFormatter())
97
        self.logger.addHandler(handler)
98

    
99
        # Get our local dir
100
        self.ci_dir = os.path.dirname(os.path.abspath(__file__))
101
        self.repo_dir = os.path.dirname(self.ci_dir)
102

    
103
        # Read config file
104
        default_conffile = os.path.join(self.ci_dir, "new_config")
105
        self.conffile = os.environ.get("SYNNEFO_CI_CONFIG_FILE",
106
                                       default_conffile)
107

    
108
        self.config = ConfigParser()
109
        self.config.optionxform = str
110
        self.config.read(self.conffile)
111
        temp_config = self.config.get('Global', 'temporary_config')
112
        if cleanup_config:
113
            try:
114
                os.remove(temp_config)
115
            except:
116
                pass
117
        else:
118
            self.config.read(self.config.get('Global', 'temporary_config'))
119

    
120
        # Initialize variables
121
        self.fabric_installed = False
122
        self.kamaki_installed = False
123
        self.cyclades_client = None
124
        self.image_client = None
125

    
126
    def setup_kamaki(self):
127
        """Initialize kamaki
128

129
        Setup cyclades_client and image_client
130
        """
131
        self.logger.info("Setup kamaki client..")
132
        auth_url = self.config.get('Deployment', 'auth_url')
133
        self.logger.debug("Authentication URL is %s" % _green(auth_url))
134
        token = self.config.get('Deployment', 'token')
135
        #self.logger.debug("Token is %s" % _green(token))
136

    
137
        astakos_client = AstakosClient(auth_url, token)
138

    
139
        cyclades_url = \
140
            astakos_client.get_service_endpoints('compute')['publicURL']
141
        self.logger.debug("Cyclades API url is %s" % _green(cyclades_url))
142
        self.cyclades_client = CycladesClient(cyclades_url, token)
143
        self.cyclades_client.CONNECTION_RETRY_LIMIT = 2
144

    
145
        image_url = \
146
            astakos_client.get_service_endpoints('image')['publicURL']
147
        self.logger.debug("Images API url is %s" % _green(image_url))
148
        self.image_client = ImageClient(cyclades_url, token)
149
        self.image_client.CONNECTION_RETRY_LIMIT = 2
150

    
151
    def _wait_transition(self, server_id, current_status, new_status):
152
        """Wait for server to go from current_status to new_status"""
153
        self.logger.debug("Waiting for server to become %s" % new_status)
154
        timeout = self.config.getint('Global', 'build_timeout')
155
        sleep_time = 5
156
        while True:
157
            server = self.cyclades_client.get_server_details(server_id)
158
            if server['status'] == new_status:
159
                return server
160
            elif timeout < 0:
161
                self.logger.error(
162
                    "Waiting for server to become %s timed out" % new_status)
163
                self.destroy_server(False)
164
                sys.exit(-1)
165
            elif server['status'] == current_status:
166
                # Sleep for #n secs and continue
167
                timeout = timeout - sleep_time
168
                time.sleep(sleep_time)
169
            else:
170
                self.logger.error(
171
                    "Server failed with status %s" % server['status'])
172
                self.destroy_server(False)
173
                sys.exit(-1)
174

    
175
    @_check_kamaki
176
    def destroy_server(self, wait=True):
177
        """Destroy slave server"""
178
        server_id = self.config.getint('Temporary Options', 'server_id')
179
        self.logger.info("Destoying server with id %s " % server_id)
180
        self.cyclades_client.delete_server(server_id)
181
        if wait:
182
            self._wait_transition(server_id, "ACTIVE", "DELETED")
183

    
184
    @_check_kamaki
185
    def create_server(self):
186
        """Create slave server"""
187
        self.logger.info("Create a new server..")
188
        image = self._find_image()
189
        self.logger.debug("Will use image \"%s\"" % _green(image['name']))
190
        self.logger.debug("Image has id %s" % _green(image['id']))
191
        server = self.cyclades_client.create_server(
192
            self.config.get('Deployment', 'server_name'),
193
            self.config.getint('Deployment', 'flavor_id'),
194
            image['id'])
195
        server_id = server['id']
196
        self.write_config('server_id', server_id)
197
        self.logger.debug("Server got id %s" % _green(server_id))
198
        server_user = server['metadata']['users']
199
        self.write_config('server_user', server_user)
200
        self.logger.debug("Server's admin user is %s" % _green(server_user))
201
        server_passwd = server['adminPass']
202
        self.write_config('server_passwd', server_passwd)
203

    
204
        server = self._wait_transition(server_id, "BUILD", "ACTIVE")
205
        self._get_server_ip_and_port(server)
206
        self._copy_ssh_keys()
207

    
208
        self.setup_fabric()
209
        self.logger.info("Setup firewall")
210
        accept_ssh_from = self.config.get('Global', 'filter_access_network')
211
        self.logger.debug("Block ssh except from %s" % accept_ssh_from)
212
        cmd = """
213
        local_ip=$(/sbin/ifconfig eth0 | grep 'inet addr:' | \
214
            cut -d':' -f2 | cut -d' ' -f1)
215
        iptables -A INPUT -s localhost -j ACCEPT
216
        iptables -A INPUT -s $local_ip -j ACCEPT
217
        iptables -A INPUT -s {0} -p tcp --dport 22 -j ACCEPT
218
        iptables -A INPUT -p tcp --dport 22 -j DROP
219
        """.format(accept_ssh_from)
220
        _run(cmd, False)
221

    
222
    def _find_image(self):
223
        """Find a suitable image to use
224

225
        It has to belong to the `system_uuid' user and
226
        contain the word `image_name'
227
        """
228
        system_uuid = self.config.get('Deployment', 'system_uuid')
229
        image_name = self.config.get('Deployment', 'image_name').lower()
230
        images = self.image_client.list_public(detail=True)['images']
231
        # Select images by `system_uuid' user
232
        images = [x for x in images if x['user_id'] == system_uuid]
233
        # Select images with `image_name' in their names
234
        images = \
235
            [x for x in images if x['name'].lower().find(image_name) != -1]
236
        # Let's select the first one
237
        return images[0]
238

    
239
    def _get_server_ip_and_port(self, server):
240
        """Compute server's IPv4 and ssh port number"""
241
        self.logger.info("Get server connection details..")
242
        # XXX: check if this IP is from public network
243
        server_ip = server['attachments'][0]['ipv4']
244
        if self.config.get('Deployment', 'deploy_on_io') == "True":
245
            tmp1 = int(server_ip.split(".")[2])
246
            tmp2 = int(server_ip.split(".")[3])
247
            server_ip = "gate.okeanos.io"
248
            server_port = 10000 + tmp1 * 256 + tmp2
249
        else:
250
            server_port = 22
251
        self.write_config('server_ip', server_ip)
252
        self.logger.debug("Server's IPv4 is %s" % _green(server_ip))
253
        self.write_config('server_port', server_port)
254
        self.logger.debug("Server's ssh port is %s" % _green(server_port))
255

    
256
    @_check_fabric
257
    def _copy_ssh_keys(self):
258
        authorized_keys = self.config.get("Deployment",
259
                                          "ssh_keys")
260
        if os.path.exists(authorized_keys):
261
            keyfile = '/tmp/%s.pub' % fabric.env.user
262
            _run('mkdir -p ~/.ssh && chmod 700 ~/.ssh', False)
263
            fabric.put(authorized_keys, keyfile)
264
            _run('cat %s >> ~/.ssh/authorized_keys' % keyfile, False)
265
            _run('rm %s' % keyfile, False)
266
            self.logger.debug("Uploaded ssh authorized keys")
267
        else:
268
            self.logger.debug("No ssh keys found")
269

    
270
    def write_config(self, option, value, section="Temporary Options"):
271
        """Write changes back to config file"""
272
        try:
273
            self.config.add_section(section)
274
        except DuplicateSectionError:
275
            pass
276
        self.config.set(section, option, str(value))
277
        temp_conf_file = self.config.get('Global', 'temporary_config')
278
        with open(temp_conf_file, 'wb') as tcf:
279
            self.config.write(tcf)
280

    
281
    def setup_fabric(self):
282
        """Setup fabric environment"""
283
        self.logger.info("Setup fabric parameters..")
284
        fabric.env.user = self.config.get('Temporary Options', 'server_user')
285
        fabric.env.host_string = \
286
            self.config.get('Temporary Options', 'server_ip')
287
        fabric.env.port = self.config.getint('Temporary Options',
288
                                             'server_port')
289
        fabric.env.password = self.config.get('Temporary Options',
290
                                              'server_passwd')
291
        fabric.env.connection_attempts = 10
292
        fabric.env.shell = "/bin/bash -c"
293
        fabric.env.disable_known_hosts = True
294
        fabric.env.output_prefix = None
295

    
296
    def _check_hash_sum(self, localfile, remotefile):
297
        """Check hash sums of two files"""
298
        self.logger.debug("Check hash sum for local file %s" % localfile)
299
        hash1 = os.popen("sha256sum %s" % localfile).read().split(' ')[0]
300
        self.logger.debug("Local file has sha256 hash %s" % hash1)
301
        self.logger.debug("Check hash sum for remote file %s" % remotefile)
302
        hash2 = _run("sha256sum %s" % remotefile, False)
303
        hash2 = hash2.split(' ')[0]
304
        self.logger.debug("Remote file has sha256 hash %s" % hash2)
305
        if hash1 != hash2:
306
            self.logger.error("Hashes differ.. aborting")
307
            sys.exit(-1)
308

    
309
    @_check_fabric
310
    def clone_repo(self):
311
        """Clone Synnefo repo from slave server"""
312
        self.logger.info("Configure repositories on remote server..")
313
        self.logger.debug("Setup apt, install curl and git")
314
        cmd = """
315
        echo 'APT::Install-Suggests "false";' >> /etc/apt/apt.conf
316
        apt-get update
317
        apt-get install curl git --yes
318
        echo -e "\n\ndeb {0}" >> /etc/apt/sources.list
319
        curl https://dev.grnet.gr/files/apt-grnetdev.pub | apt-key add -
320
        apt-get update
321
        git config --global user.name {1}
322
        git config --global user.mail {2}
323
        """.format(self.config.get('Global', 'apt_repo'),
324
                   self.config.get('Global', 'git_config_name'),
325
                   self.config.get('Global', 'git_config_mail'))
326
        _run(cmd, False)
327

    
328
        synnefo_repo = self.config.get('Global', 'synnefo_repo')
329
        synnefo_branch = self.config.get('Global', 'synnefo_branch')
330
        # Currently clonning synnefo can fail unexpectedly
331
        cloned = False
332
        for i in range(3):
333
            self.logger.debug("Clone synnefo from %s" % synnefo_repo)
334
            cmd = ("git clone --branch %s %s"
335
                   % (synnefo_branch, synnefo_repo))
336
            try:
337
                _run(cmd, False)
338
                cloned = True
339
                break
340
            except:
341
                self.logger.warning("Clonning synnefo failed.. retrying %s"
342
                                    % i)
343
        if not cloned:
344
            self.logger.error("Can not clone Synnefo repo.")
345
            sys.exit(-1)
346

    
347
        deploy_repo = self.config.get('Global', 'deploy_repo')
348
        self.logger.debug("Clone snf-deploy from %s" % deploy_repo)
349
        _run("git clone --depth 1 %s" % deploy_repo, False)
350

    
351
    @_check_fabric
352
    def build_synnefo(self):
353
        """Build Synnefo packages"""
354
        self.logger.info("Build Synnefo packages..")
355
        self.logger.debug("Install development packages")
356
        cmd = """
357
        apt-get update
358
        apt-get install zlib1g-dev dpkg-dev debhelper git-buildpackage \
359
                python-dev python-all python-pip --yes
360
        pip install devflow
361
        """
362
        _run(cmd, False)
363

    
364
        if self.config.get('Global', 'patch_pydist') == "True":
365
            self.logger.debug("Patch pydist.py module")
366
            cmd = r"""
367
            sed -r -i 's/(\(\?P<name>\[A-Za-z\]\[A-Za-z0-9_\.)/\1\\\-/' \
368
                /usr/share/python/debpython/pydist.py
369
            """
370
            _run(cmd, False)
371

372
        self.logger.debug("Build snf-deploy package")
373
        cmd = """
374
        git checkout -t origin/debian
375
        git-buildpackage --git-upstream-branch=master \
376
                --git-debian-branch=debian \
377
                --git-export-dir=../snf-deploy_build-area \
378
                -uc -us
379
        """
380
        with fabric.cd("snf-deploy"):
381
            _run(cmd, True)
382

383
        self.logger.debug("Install snf-deploy package")
384
        cmd = """
385
        dpkg -i snf-deploy*.deb
386
        apt-get -f install --yes
387
        """
388
        with fabric.cd("snf-deploy_build-area"):
389
            with fabric.settings(warn_only=True):
390
                _run(cmd, True)
391

392
        self.logger.debug("Build synnefo packages")
393
        cmd = """
394
        devflow-autopkg snapshot -b ~/synnefo_build-area --no-sign
395
        """
396
        with fabric.cd("synnefo"):
397
            _run(cmd, True)
398

399
        self.logger.debug("Copy synnefo debs to snf-deploy packages dir")
400
        cmd = """
401
        cp ~/synnefo_build-area/*.deb /var/lib/snf-deploy/packages/
402
        """
403
        _run(cmd, False)
404

405
    @_check_fabric
406
    def deploy_synnefo(self):
407
        """Deploy Synnefo using snf-deploy"""
408
        self.logger.info("Deploy Synnefo..")
409
        schema = self.config.get('Global', 'schema')
410
        schema_files = os.path.join(self.ci_dir, "schemas/%s/*" % schema)
411
        self.logger.debug("Will use %s schema" % schema)
412

413
        self.logger.debug("Upload schema files to server")
414
        with fabric.quiet():
415
            fabric.put(schema_files, "/etc/snf-deploy/")
416

417
        self.logger.debug("Change password in nodes.conf file")
418
        cmd = """
419
        sed -i 's/^password =.*/password = {0}/' /etc/snf-deploy/nodes.conf
420
        """.format(fabric.env.password)
421
        _run(cmd, False)
422

423
        self.logger.debug("Run snf-deploy")
424
        cmd = """
425
        snf-deploy all --autoconf
426
        """
427
        _run(cmd, True)
428

429
    @_check_fabric
430
    def unit_test(self):
431
        """Run Synnefo unit test suite"""
432
        self.logger.info("Run Synnefo unit test suite")
433
        component = self.config.get('Unit Tests', 'component')
434

435
        self.logger.debug("Install needed packages")
436
        cmd = """
437
        pip install mock
438
        pip install factory_boy
439
        """
440
        _run(cmd, False)
441

442
        self.logger.debug("Upload tests.sh file")
443
        unit_tests_file = os.path.join(self.ci_dir, "tests.sh")
444
        with fabric.quiet():
445
            fabric.put(unit_tests_file, ".")
446

447
        self.logger.debug("Run unit tests")
448
        cmd = """
449
        bash tests.sh {0}
450
        """.format(component)
451
        _run(cmd, True)
452

453
    @_check_fabric
454
    def run_burnin(self):
455
        """Run burnin functional test suite"""
456
        self.logger.info("Run Burnin functional test suite")
457
        cmd = """
458
        auth_url=$(grep -e '^url =' .kamakirc | cut -d' ' -f3)
459
        token=$(grep -e '^token =' .kamakirc | cut -d' ' -f3)
460
        images_user=$(kamaki image list -l | grep owner | \
461
                      cut -d':' -f2 | tr -d ' ')
462
        snf-burnin --auth-url=$auth_url --token=$token \
463
            --force-flavor=2 --image-id=all \
464
            --system-images-user=$images_user \
465
            {0}
466
        log_folder=$(ls -1d /var/log/burnin/* | tail -n1)
467
        for i in $(ls $log_folder/*/details*); do
468
            echo -e "\\n\\n"
469
            echo -e "***** $i\\n"
470
            cat $i
471
        done
472
        """.format(self.config.get('Burnin', 'cmd_options'))
473
        _run(cmd, True)
474

475
    @_check_fabric
476
    def fetch_packages(self):
477
        """Download Synnefo packages"""
478
        self.logger.info("Download Synnefo packages")
479
        self.logger.debug("Create tarball with packages")
480
        cmd = """
481
        tar czf synnefo_build-area.tgz synnefo_build-area
482
        """
483
        _run(cmd, False)
484

485
        pkgs_dir = self.config.get('Global', 'pkgs_dir')
486
        self.logger.debug("Fetch packages to local dir %s" % pkgs_dir)
487
        os.makedirs(pkgs_dir)
488
        with fabric.quiet():
489
            fabric.get("synnefo_build-area.tgz", pkgs_dir)
490

491
        pkgs_file = os.path.join(pkgs_dir, "synnefo_build-area.tgz")
492
        self._check_hash_sum(pkgs_file, "synnefo_build-area.tgz")
493

494
        self.logger.debug("Untar packages file %s" % pkgs_file)
495
        os.system("cd %s; tar xzf synnefo_build-area.tgz" % pkgs_dir)
496