Statistics
| Branch: | Tag: | Revision:

root / ci / utils.py @ 525f2979

History | View | Annotate | Download (17.6 kB)

1
#!/usr/bin/env python
2

    
3
"""
4
Synnefo ci utils module
5
"""
6

    
7
import os
8
import sys
9
import time
10
import logging
11
import fabric.api as fabric
12
from ConfigParser import ConfigParser, DuplicateSectionError
13

    
14
from kamaki.clients.astakos import AstakosClient
15
from kamaki.clients.cyclades import CycladesClient
16
from kamaki.clients.image import ImageClient
17

    
18

    
19
def _run(cmd, verbose):
20
    """Run fabric with verbose level"""
21
    if verbose:
22
        args = ('running',)
23
    else:
24
        args = ('running', 'stdout',)
25
    with fabric.hide(*args):
26
        return fabric.run(cmd)
27

    
28

    
29
def _red(msg):
30
    """Red color"""
31
    #return "\x1b[31m" + str(msg) + "\x1b[0m"
32
    return str(msg)
33

    
34

    
35
def _yellow(msg):
36
    """Yellow color"""
37
    #return "\x1b[33m" + str(msg) + "\x1b[0m"
38
    return str(msg)
39

    
40

    
41
def _green(msg):
42
    """Green color"""
43
    #return "\x1b[32m" + str(msg) + "\x1b[0m"
44
    return str(msg)
45

    
46

    
47
def _check_fabric(fun):
48
    """Check if fabric env has been set"""
49
    def wrapper(self, *args):
50
        """wrapper function"""
51
        if not self.fabric_installed:
52
            self.setup_fabric()
53
        return fun(self, *args)
54
    return wrapper
55

    
56

    
57
def _check_kamaki(fun):
58
    """Check if kamaki has been initialized"""
59
    def wrapper(self, *args):
60
        """wrapper function"""
61
        if not self.kamaki_installed:
62
            self.setup_kamaki()
63
        return fun(self, *args)
64
    return wrapper
65

    
66

    
67
class _MyFormatter(logging.Formatter):
68
    """Logging Formatter"""
69
    def format(self, record):
70
        format_orig = self._fmt
71
        if record.levelno == logging.DEBUG:
72
            self._fmt = "  %(msg)s"
73
        elif record.levelno == logging.INFO:
74
            self._fmt = "%(msg)s"
75
        elif record.levelno == logging.WARNING:
76
            self._fmt = _yellow("[W] %(msg)s")
77
        elif record.levelno == logging.ERROR:
78
            self._fmt = _red("[E] %(msg)s")
79
        result = logging.Formatter.format(self, record)
80
        self._fmt = format_orig
81
        return result
82

    
83

    
84
class SynnefoCI(object):
85
    """SynnefoCI python class"""
86

    
87
    def __init__(self, cleanup_config=False):
88
        """ Initialize SynnefoCI python class
89

90
        Setup logger, local_dir, config and kamaki
91
        """
92
        # Setup logger
93
        self.logger = logging.getLogger('synnefo-ci')
94
        self.logger.setLevel(logging.DEBUG)
95
        handler = logging.StreamHandler()
96
        handler.setFormatter(_MyFormatter())
97
        self.logger.addHandler(handler)
98

    
99
        # Get our local dir
100
        self.ci_dir = os.path.dirname(os.path.abspath(__file__))
101
        self.repo_dir = os.path.dirname(self.ci_dir)
102

    
103
        # Read config file
104
        self.conffile = os.path.join(self.ci_dir, "new_config")
105
        self.config = ConfigParser()
106
        self.config.optionxform = str
107
        self.config.read(self.conffile)
108
        temp_config = self.config.get('Global', 'temporary_config')
109
        if cleanup_config:
110
            try:
111
                os.remove(temp_config)
112
            except:
113
                pass
114
        else:
115
            self.config.read(self.config.get('Global', 'temporary_config'))
116

    
117
        # Initialize variables
118
        self.fabric_installed = False
119
        self.kamaki_installed = False
120
        self.cyclades_client = None
121
        self.image_client = None
122

    
123
    def setup_kamaki(self):
124
        """Initialize kamaki
125

126
        Setup cyclades_client and image_client
127
        """
128
        self.logger.info("Setup kamaki client..")
129
        auth_url = self.config.get('Deployment', 'auth_url')
130
        self.logger.debug("Authentication URL is %s" % _green(auth_url))
131
        token = self.config.get('Deployment', 'token')
132
        #self.logger.debug("Token is %s" % _green(token))
133

    
134
        astakos_client = AstakosClient(auth_url, token)
135

    
136
        cyclades_url = \
137
            astakos_client.get_service_endpoints('compute')['publicURL']
138
        self.logger.debug("Cyclades API url is %s" % _green(cyclades_url))
139
        self.cyclades_client = CycladesClient(cyclades_url, token)
140
        self.cyclades_client.CONNECTION_RETRY_LIMIT = 2
141

    
142
        image_url = \
143
            astakos_client.get_service_endpoints('image')['publicURL']
144
        self.logger.debug("Images API url is %s" % _green(image_url))
145
        self.image_client = ImageClient(cyclades_url, token)
146
        self.image_client.CONNECTION_RETRY_LIMIT = 2
147

    
148
    def _wait_transition(self, server_id, current_status, new_status):
149
        """Wait for server to go from current_status to new_status"""
150
        self.logger.debug("Waiting for server to become %s" % new_status)
151
        timeout = self.config.getint('Global', 'build_timeout')
152
        sleep_time = 5
153
        while True:
154
            server = self.cyclades_client.get_server_details(server_id)
155
            if server['status'] == new_status:
156
                return server
157
            elif timeout < 0:
158
                self.logger.error(
159
                    "Waiting for server to become %s timed out" % new_status)
160
                self.destroy_server(False)
161
                sys.exit(-1)
162
            elif server['status'] == current_status:
163
                # Sleep for #n secs and continue
164
                timeout = timeout - sleep_time
165
                time.sleep(sleep_time)
166
            else:
167
                self.logger.error(
168
                    "Server failed with status %s" % server['status'])
169
                self.destroy_server(False)
170
                sys.exit(-1)
171

    
172
    @_check_kamaki
173
    def destroy_server(self, wait=True):
174
        """Destroy slave server"""
175
        server_id = self.config.getint('Temporary Options', 'server_id')
176
        self.logger.info("Destoying server with id %s " % server_id)
177
        self.cyclades_client.delete_server(server_id)
178
        if wait:
179
            self._wait_transition(server_id, "ACTIVE", "DELETED")
180

    
181
    @_check_kamaki
182
    def create_server(self):
183
        """Create slave server"""
184
        self.logger.info("Create a new server..")
185
        image = self._find_image()
186
        self.logger.debug("Will use image \"%s\"" % _green(image['name']))
187
        self.logger.debug("Image has id %s" % _green(image['id']))
188
        server = self.cyclades_client.create_server(
189
            self.config.get('Deployment', 'server_name'),
190
            self.config.getint('Deployment', 'flavor_id'),
191
            image['id'])
192
        server_id = server['id']
193
        self.write_config('server_id', server_id)
194
        self.logger.debug("Server got id %s" % _green(server_id))
195
        server_user = server['metadata']['users']
196
        self.write_config('server_user', server_user)
197
        self.logger.debug("Server's admin user is %s" % _green(server_user))
198
        server_passwd = server['adminPass']
199
        self.write_config('server_passwd', server_passwd)
200
        self.logger.debug(
201
            "Server's admin password is %s" % _green(server_passwd))
202

    
203
        server = self._wait_transition(server_id, "BUILD", "ACTIVE")
204
        self._get_server_ip_and_port(server)
205

    
206
        self.setup_fabric()
207
        self.logger.info("Setup firewall")
208
        accept_ssh_from = self.config.get('Global', 'filter_access_network')
209
        self.logger.debug("Block ssh except from %s" % accept_ssh_from)
210
        cmd = """
211
        iptables -A INPUT -s localhost -j ACCEPT
212
        iptables -A INPUT -s {0} -p tcp --dport 22 -j ACCEPT
213
        iptables -A INPUT -p tcp --dport 22 -j DROP
214
        """.format(accept_ssh_from)
215
        _run(cmd, False)
216

    
217
    def _find_image(self):
218
        """Find a suitable image to use
219

220
        It has to belong to the `system_uuid' user and
221
        contain the word `image_name'
222
        """
223
        system_uuid = self.config.get('Deployment', 'system_uuid')
224
        image_name = self.config.get('Deployment', 'image_name').lower()
225
        images = self.image_client.list_public(detail=True)['images']
226
        # Select images by `system_uuid' user
227
        images = [x for x in images if x['user_id'] == system_uuid]
228
        # Select images with `image_name' in their names
229
        images = \
230
            [x for x in images if x['name'].lower().find(image_name) != -1]
231
        # Let's select the first one
232
        return images[0]
233

    
234
    def _get_server_ip_and_port(self, server):
235
        """Compute server's IPv4 and ssh port number"""
236
        self.logger.info("Get server connection details..")
237
        # XXX: check if this IP is from public network
238
        server_ip = server['attachments'][0]['ipv4']
239
        if eval(self.config.get('Deployment', 'deploy_on_io')):
240
            tmp1 = int(server_ip.split(".")[2])
241
            tmp2 = int(server_ip.split(".")[3])
242
            server_ip = "gate.okeanos.io"
243
            server_port = 10000 + tmp1 * 256 + tmp2
244
        else:
245
            server_port = 22
246
        self.write_config('server_ip', server_ip)
247
        self.logger.debug("Server's IPv4 is %s" % _green(server_ip))
248
        self.write_config('server_port', server_port)
249
        self.logger.debug("Server's ssh port is %s" % _green(server_port))
250

    
251
    def write_config(self, option, value, section="Temporary Options"):
252
        """Write changes back to config file"""
253
        try:
254
            self.config.add_section(section)
255
        except DuplicateSectionError:
256
            pass
257
        self.config.set(section, option, str(value))
258
        temp_conf_file = self.config.get('Global', 'temporary_config')
259
        with open(temp_conf_file, 'wb') as tcf:
260
            self.config.write(tcf)
261

    
262
    def setup_fabric(self):
263
        """Setup fabric environment"""
264
        self.logger.info("Setup fabric parameters..")
265
        fabric.env.user = self.config.get('Temporary Options', 'server_user')
266
        fabric.env.host_string = \
267
            self.config.get('Temporary Options', 'server_ip')
268
        fabric.env.port = self.config.getint('Temporary Options', 'server_port')
269
        fabric.env.password = \
270
            self.config.get('Temporary Options', 'server_passwd')
271
        fabric.env.connection_attempts = 10
272
        fabric.env.shell = "/bin/bash -c"
273
        fabric.env.disable_known_hosts = True
274
        fabric.env.output_prefix = None
275

    
276
    def _check_hash_sum(self, localfile, remotefile):
277
        """Check hash sums of two files"""
278
        self.logger.debug("Check hash sum for local file %s" % localfile)
279
        hash1 = os.popen("sha256sum %s" % localfile).read().split(' ')[0]
280
        self.logger.debug("Local file has sha256 hash %s" % hash1)
281
        self.logger.debug("Check hash sum for remote file %s" % remotefile)
282
        hash2 = _run("sha256sum %s" % remotefile, False)
283
        hash2 = hash2.split(' ')[0]
284
        self.logger.debug("Remote file has sha256 hash %s" % hash2)
285
        if hash1 != hash2:
286
            self.logger.error("Hashes differ.. aborting")
287
            sys.exit(-1)
288

    
289
    @_check_fabric
290
    def clone_repo(self):
291
        """Clone Synnefo repo from slave server"""
292
        self.logger.info("Configure repositories on remote server..")
293
        self.logger.debug("Setup apt, install curl and git")
294
        cmd = """
295
        echo 'APT::Install-Suggests "false";' >> /etc/apt/apt.conf
296
        apt-get update
297
        apt-get install curl git --yes
298
        echo -e "\n\ndeb {0}" >> /etc/apt/sources.list
299
        curl https://dev.grnet.gr/files/apt-grnetdev.pub | apt-key add -
300
        apt-get update
301
        git config --global user.name {1}
302
        git config --global user.mail {2}
303
        """.format(self.config.get('Global', 'apt_repo'),
304
                   self.config.get('Global', 'git_config_name'),
305
                   self.config.get('Global', 'git_config_mail'))
306
        _run(cmd, False)
307

    
308
        synnefo_repo = self.config.get('Global', 'synnefo_repo')
309
        # Currently clonning synnefo can fail unexpectedly
310
        for i in range(3):
311
            self.logger.debug("Clone synnefo from %s" % synnefo_repo)
312
            try:
313
                _run("git clone %s" % synnefo_repo, False)
314
                break
315
            except:
316
                self.logger.warning("Clonning synnefo failed.. retrying %s" % i)
317

    
318
        synnefo_branch = self.config.get('Global', 'synnefo_branch')
319
        if synnefo_branch == "HEAD":
320
            # Get current branch
321
            synnefo_branch = os.popen("git rev-parse HEAD").read().strip()
322
            self.logger.debug(
323
                "Checkout %s in feature-ci branch" % synnefo_branch)
324
            with fabric.cd("synnefo"):
325
                _run("git checkout -b feature-ci %s" % synnefo_branch, False)
326
        elif synnefo_branch == "origin/master":
327
            pass
328
        elif "origin" in synnefo_branch:
329
            self.logger.debug("Checkout %s branch" % synnefo_branch)
330
            with fabric.cd("synnefo"):
331
                _run("git checkout -t %s" % synnefo_branch, False)
332
        else:
333
            self.logger.debug(
334
                "Checkout %s in feature-ci branch" % synnefo_branch)
335
            with fabric.cd("synnefo"):
336
                _run("git checkout -b feature-ci %s" % synnefo_branch, False)
337

    
338
        deploy_repo = self.config.get('Global', 'deploy_repo')
339
        self.logger.debug("Clone snf-deploy from %s" % deploy_repo)
340
        _run("git clone %s" % deploy_repo, False)
341

    
342
    @_check_fabric
343
    def build_synnefo(self):
344
        """Build Synnefo packages"""
345
        self.logger.info("Build Synnefo packages..")
346
        self.logger.debug("Install development packages")
347
        cmd = """
348
        apt-get update
349
        apt-get install zlib1g-dev dpkg-dev debhelper git-buildpackage \
350
                python-dev python-all python-pip --yes
351
        pip install devflow
352
        """
353
        _run(cmd, False)
354

    
355
        if eval(self.config.get('Global', 'patch_pydist')):
356
            self.logger.debug("Patch pydist.py module")
357
            cmd = r"""
358
            sed -r -i 's/(\(\?P<name>\[A-Za-z\]\[A-Za-z0-9_\.)/\1\\\-/' \
359
                /usr/share/python/debpython/pydist.py
360
            """
361
            _run(cmd, False)
362

363
        self.logger.debug("Build snf-deploy package")
364
        cmd = """
365
        git checkout -t origin/debian
366
        git-buildpackage --git-upstream-branch=master \
367
                --git-debian-branch=debian \
368
                --git-export-dir=../snf-deploy_build-area \
369
                -uc -us
370
        """
371
        with fabric.cd("snf-deploy"):
372
            _run(cmd, True)
373

374
        self.logger.debug("Install snf-deploy package")
375
        cmd = """
376
        dpkg -i snf-deploy*.deb
377
        apt-get -f install --yes
378
        """
379
        with fabric.cd("snf-deploy_build-area"):
380
            with fabric.settings(warn_only=True):
381
                _run(cmd, True)
382

383
        self.logger.debug("Build synnefo packages")
384
        cmd = """
385
        devflow-autopkg snapshot -b ~/synnefo_build-area --no-sign
386
        """
387
        with fabric.cd("synnefo"):
388
            _run(cmd, True)
389

390
        self.logger.debug("Copy synnefo debs to snf-deploy packages dir")
391
        cmd = """
392
        cp ~/synnefo_build-area/*.deb /var/lib/snf-deploy/packages/
393
        """
394
        _run(cmd, False)
395

396
    @_check_fabric
397
    def deploy_synnefo(self):
398
        """Deploy Synnefo using snf-deploy"""
399
        self.logger.info("Deploy Synnefo..")
400
        schema = self.config.get('Global', 'schema')
401
        schema_files = os.path.join(self.ci_dir, "schemas/%s/*" % schema)
402
        self.logger.debug("Will use %s schema" % schema)
403

404
        self.logger.debug("Upload schema files to server")
405
        with fabric.quiet():
406
            fabric.put(schema_files, "/etc/snf-deploy/")
407

408
        self.logger.debug("Change password in nodes.conf file")
409
        cmd = """
410
        sed -i 's/^password =.*/password = {0}/' /etc/snf-deploy/nodes.conf
411
        """.format(fabric.env.password)
412
        _run(cmd, False)
413

414
        self.logger.debug("Run snf-deploy")
415
        cmd = """
416
        snf-deploy all --autoconf
417
        """
418
        _run(cmd, True)
419

420
    @_check_fabric
421
    def unit_test(self):
422
        """Run Synnefo unit test suite"""
423
        self.logger.info("Run Synnefo unit test suite")
424
        component = self.config.get('Unit Tests', 'component')
425

426
        self.logger.debug("Install needed packages")
427
        cmd = """
428
        pip install mock
429
        pip install factory_boy
430
        """
431
        _run(cmd, False)
432

433
        self.logger.debug("Upload local_unit_tests.sh file")
434
        unit_tests_file = os.path.join(self.ci_dir, "local_unit_tests.sh")
435
        with fabric.quiet():
436
            fabric.put(unit_tests_file, ".")
437

438
        self.logger.debug("Run unit tests")
439
        cmd = """
440
        bash local_unit_tests.sh {0}
441
        """.format(component)
442
        _run(cmd, True)
443

444
    @_check_fabric
445
    def run_burnin(self):
446
        """Run burnin functional test suite"""
447
        self.logger.info("Run Burnin functional test suite")
448
        cmd = """
449
        auth_url=$(grep -e '^url =' .kamakirc | cut -d' ' -f3)
450
        token=$(grep -e '^token =' .kamakirc | cut -d' ' -f3)
451
        images_user=$(kamaki image list -l | grep owner | \
452
                      cut -d':' -f2 | tr -d ' ')
453
        snf-burnin --auth-url=$auth_url --token=$token \
454
            --force-flavor=2 --image-id=all \
455
            --system-images-user=$images_user \
456
            {0}
457
        log_folder=$(ls -1d /var/log/burnin/* | tail -n1)
458
        for i in $(ls $log_folder/*/details*); do
459
            echo -e "\\n\\n"
460
            echo -e "***** $i\\n"
461
            cat $i
462
        done
463
        """.format(self.config.get('Burnin', 'cmd_options'))
464
        _run(cmd, True)
465

466
    @_check_fabric
467
    def fetch_packages(self):
468
        """Download Synnefo packages"""
469
        self.logger.info("Download Synnefo packages")
470
        self.logger.debug("Create tarball with packages")
471
        cmd = """
472
        tar czf synnefo_build-area.tgz synnefo_build-area
473
        """
474
        _run(cmd, False)
475

476
        pkgs_dir = self.config.get('Global', 'pkgs_dir')
477
        self.logger.debug("Fetch packages to local dir %s" % pkgs_dir)
478
        os.makedirs(pkgs_dir)
479
        with fabric.quiet():
480
            fabric.get("synnefo_build-area.tgz", pkgs_dir)
481

482
        pkgs_file = os.path.join(pkgs_dir, "synnefo_build-area.tgz")
483
        self._check_hash_sum(pkgs_file, "synnefo_build-area.tgz")
484

485
        self.logger.debug("Untar packages file %s" % pkgs_file)
486
        os.system("cd %s; tar xzf synnefo_build-area.tgz" % pkgs_dir)
487