Statistics
| Branch: | Tag: | Revision:

root / ci / utils.py @ ddd240ff

History | View | Annotate | Download (18.7 kB)

1
#!/usr/bin/env python
2

    
3
"""
4
Synnefo ci utils module
5
"""
6

    
7
import os
8
import sys
9
import time
10
import logging
11
import fabric.api as fabric
12
from ConfigParser import ConfigParser, DuplicateSectionError
13

    
14
from kamaki.cli import config as kamaki_config
15
from kamaki.clients.astakos import AstakosClient
16
from kamaki.clients.cyclades import CycladesClient
17
from kamaki.clients.image import ImageClient
18

    
19

    
20
def _run(cmd, verbose):
21
    """Run fabric with verbose level"""
22
    if verbose:
23
        args = ('running',)
24
    else:
25
        args = ('running', 'stdout',)
26
    with fabric.hide(*args):
27
        return fabric.run(cmd)
28

    
29

    
30
def _red(msg):
31
    """Red color"""
32
    #return "\x1b[31m" + str(msg) + "\x1b[0m"
33
    return str(msg)
34

    
35

    
36
def _yellow(msg):
37
    """Yellow color"""
38
    #return "\x1b[33m" + str(msg) + "\x1b[0m"
39
    return str(msg)
40

    
41

    
42
def _green(msg):
43
    """Green color"""
44
    #return "\x1b[32m" + str(msg) + "\x1b[0m"
45
    return str(msg)
46

    
47

    
48
def _check_fabric(fun):
49
    """Check if fabric env has been set"""
50
    def wrapper(self, *args, **kwargs):
51
        """wrapper function"""
52
        if not self.fabric_installed:
53
            self.setup_fabric()
54
        return fun(self, *args, **kwargs)
55
    return wrapper
56

    
57

    
58
def _check_kamaki(fun):
59
    """Check if kamaki has been initialized"""
60
    def wrapper(self, *args, **kwargs):
61
        """wrapper function"""
62
        if not self.kamaki_installed:
63
            self.setup_kamaki()
64
        return fun(self, *args, **kwargs)
65
    return wrapper
66

    
67

    
68
class _MyFormatter(logging.Formatter):
69
    """Logging Formatter"""
70
    def format(self, record):
71
        format_orig = self._fmt
72
        if record.levelno == logging.DEBUG:
73
            self._fmt = "  %(msg)s"
74
        elif record.levelno == logging.INFO:
75
            self._fmt = "%(msg)s"
76
        elif record.levelno == logging.WARNING:
77
            self._fmt = _yellow("[W] %(msg)s")
78
        elif record.levelno == logging.ERROR:
79
            self._fmt = _red("[E] %(msg)s")
80
        result = logging.Formatter.format(self, record)
81
        self._fmt = format_orig
82
        return result
83

    
84

    
85
class SynnefoCI(object):
86
    """SynnefoCI python class"""
87

    
88
    def __init__(self, cleanup_config=False, cloud=None):
89
        """ Initialize SynnefoCI python class
90

91
        Setup logger, local_dir, config and kamaki
92
        """
93
        # Setup logger
94
        self.logger = logging.getLogger('synnefo-ci')
95
        self.logger.setLevel(logging.DEBUG)
96
        handler = logging.StreamHandler()
97
        handler.setFormatter(_MyFormatter())
98
        self.logger.addHandler(handler)
99

    
100
        # Get our local dir
101
        self.ci_dir = os.path.dirname(os.path.abspath(__file__))
102
        self.repo_dir = os.path.dirname(self.ci_dir)
103

    
104
        # Read config file
105
        default_conffile = os.path.join(self.ci_dir, "new_config")
106
        self.conffile = os.environ.get("SYNNEFO_CI_CONFIG_FILE",
107
                                       default_conffile)
108

    
109
        self.config = ConfigParser()
110
        self.config.optionxform = str
111
        self.config.read(self.conffile)
112
        temp_config = self.config.get('Global', 'temporary_config')
113
        if cleanup_config:
114
            try:
115
                os.remove(temp_config)
116
            except:
117
                pass
118
        else:
119
            self.config.read(self.config.get('Global', 'temporary_config'))
120

    
121
        # Set kamaki cloud
122
        if cloud is not None:
123
            self.kamaki_cloud = cloud
124
        elif self.config.has_option("Deployment", "kamaki_cloud"):
125
            kamaki_cloud = self.config.get("Deployment", "kamaki_cloud")
126
            if kamaki_cloud == "":
127
                self.kamaki_cloud = None
128
        else:
129
            self.kamaki_cloud = None
130

    
131
        # Initialize variables
132
        self.fabric_installed = False
133
        self.kamaki_installed = False
134
        self.cyclades_client = None
135
        self.image_client = None
136

    
137
    def setup_kamaki(self):
138
        """Initialize kamaki
139

140
        Setup cyclades_client and image_client
141
        """
142

    
143
        config = kamaki_config.Config()
144
        if self.kamaki_cloud is None:
145
            self.kamaki_cloud = config.get_global("default_cloud")
146

    
147
        self.logger.info("Setup kamaki client, using cloud '%s'.." %
148
                         self.kamaki_cloud)
149
        auth_url = config.get_cloud(self.kamaki_cloud, "url")
150
        self.logger.debug("Authentication URL is %s" % _green(auth_url))
151
        token = config.get_cloud(self.kamaki_cloud, "token")
152
        #self.logger.debug("Token is %s" % _green(token))
153

    
154
        astakos_client = AstakosClient(auth_url, token)
155

    
156
        cyclades_url = \
157
            astakos_client.get_service_endpoints('compute')['publicURL']
158
        self.logger.debug("Cyclades API url is %s" % _green(cyclades_url))
159
        self.cyclades_client = CycladesClient(cyclades_url, token)
160
        self.cyclades_client.CONNECTION_RETRY_LIMIT = 2
161

    
162
        image_url = \
163
            astakos_client.get_service_endpoints('image')['publicURL']
164
        self.logger.debug("Images API url is %s" % _green(image_url))
165
        self.image_client = ImageClient(cyclades_url, token)
166
        self.image_client.CONNECTION_RETRY_LIMIT = 2
167

    
168
    def _wait_transition(self, server_id, current_status, new_status):
169
        """Wait for server to go from current_status to new_status"""
170
        self.logger.debug("Waiting for server to become %s" % new_status)
171
        timeout = self.config.getint('Global', 'build_timeout')
172
        sleep_time = 5
173
        while True:
174
            server = self.cyclades_client.get_server_details(server_id)
175
            if server['status'] == new_status:
176
                return server
177
            elif timeout < 0:
178
                self.logger.error(
179
                    "Waiting for server to become %s timed out" % new_status)
180
                self.destroy_server(False)
181
                sys.exit(-1)
182
            elif server['status'] == current_status:
183
                # Sleep for #n secs and continue
184
                timeout = timeout - sleep_time
185
                time.sleep(sleep_time)
186
            else:
187
                self.logger.error(
188
                    "Server failed with status %s" % server['status'])
189
                self.destroy_server(False)
190
                sys.exit(-1)
191

    
192
    @_check_kamaki
193
    def destroy_server(self, wait=True):
194
        """Destroy slave server"""
195
        server_id = self.config.getint('Temporary Options', 'server_id')
196
        self.logger.info("Destoying server with id %s " % server_id)
197
        self.cyclades_client.delete_server(server_id)
198
        if wait:
199
            self._wait_transition(server_id, "ACTIVE", "DELETED")
200

    
201
    @_check_kamaki
202
    def create_server(self, image_id=None, flavor_id=None):
203
        """Create slave server"""
204
        self.logger.info("Create a new server..")
205
        if image_id is None:
206
            image = self._find_image()
207
            self.logger.debug("Will use image \"%s\"" % _green(image['name']))
208
            image_id = image["id"]
209
        self.logger.debug("Image has id %s" % _green(image_id))
210
        if flavor_id is None:
211
            flavor_id = self.config.getint("Deployment", "flavor_id")
212
        server = self.cyclades_client.create_server(
213
            self.config.get('Deployment', 'server_name'),
214
            flavor_id,
215
            image_id)
216
        server_id = server['id']
217
        self.write_config('server_id', server_id)
218
        self.logger.debug("Server got id %s" % _green(server_id))
219
        server_user = server['metadata']['users']
220
        self.write_config('server_user', server_user)
221
        self.logger.debug("Server's admin user is %s" % _green(server_user))
222
        server_passwd = server['adminPass']
223
        self.write_config('server_passwd', server_passwd)
224

    
225
        server = self._wait_transition(server_id, "BUILD", "ACTIVE")
226
        self._get_server_ip_and_port(server)
227
        self._copy_ssh_keys()
228

    
229
        self.setup_fabric()
230
        self.logger.info("Setup firewall")
231
        accept_ssh_from = self.config.get('Global', 'filter_access_network')
232
        self.logger.debug("Block ssh except from %s" % accept_ssh_from)
233
        cmd = """
234
        local_ip=$(/sbin/ifconfig eth0 | grep 'inet addr:' | \
235
            cut -d':' -f2 | cut -d' ' -f1)
236
        iptables -A INPUT -s localhost -j ACCEPT
237
        iptables -A INPUT -s $local_ip -j ACCEPT
238
        iptables -A INPUT -s {0} -p tcp --dport 22 -j ACCEPT
239
        iptables -A INPUT -p tcp --dport 22 -j DROP
240
        """.format(accept_ssh_from)
241
        _run(cmd, False)
242

    
243
    def _find_image(self):
244
        """Find a suitable image to use
245

246
        It has to belong to the `system_uuid' user and
247
        contain the word `image_name'
248
        """
249
        system_uuid = self.config.get('Deployment', 'system_uuid')
250
        image_name = self.config.get('Deployment', 'image_name').lower()
251
        images = self.image_client.list_public(detail=True)['images']
252
        # Select images by `system_uuid' user
253
        images = [x for x in images if x['user_id'] == system_uuid]
254
        # Select images with `image_name' in their names
255
        images = \
256
            [x for x in images if x['name'].lower().find(image_name) != -1]
257
        # Let's select the first one
258
        return images[0]
259

    
260
    def _get_server_ip_and_port(self, server):
261
        """Compute server's IPv4 and ssh port number"""
262
        self.logger.info("Get server connection details..")
263
        server_ip = server['attachments'][0]['ipv4']
264
        if ".okeanos.io" in self.cyclades_client.base_url:
265
            tmp1 = int(server_ip.split(".")[2])
266
            tmp2 = int(server_ip.split(".")[3])
267
            server_ip = "gate.okeanos.io"
268
            server_port = 10000 + tmp1 * 256 + tmp2
269
        else:
270
            server_port = 22
271
        self.write_config('server_ip', server_ip)
272
        self.logger.debug("Server's IPv4 is %s" % _green(server_ip))
273
        self.write_config('server_port', server_port)
274
        self.logger.debug("Server's ssh port is %s" % _green(server_port))
275

    
276
    @_check_fabric
277
    def _copy_ssh_keys(self):
278
        authorized_keys = self.config.get("Deployment",
279
                                          "ssh_keys")
280
        if os.path.exists(authorized_keys):
281
            keyfile = '/tmp/%s.pub' % fabric.env.user
282
            _run('mkdir -p ~/.ssh && chmod 700 ~/.ssh', False)
283
            fabric.put(authorized_keys, keyfile)
284
            _run('cat %s >> ~/.ssh/authorized_keys' % keyfile, False)
285
            _run('rm %s' % keyfile, False)
286
            self.logger.debug("Uploaded ssh authorized keys")
287
        else:
288
            self.logger.debug("No ssh keys found")
289

    
290
    def write_config(self, option, value, section="Temporary Options"):
291
        """Write changes back to config file"""
292
        try:
293
            self.config.add_section(section)
294
        except DuplicateSectionError:
295
            pass
296
        self.config.set(section, option, str(value))
297
        temp_conf_file = self.config.get('Global', 'temporary_config')
298
        with open(temp_conf_file, 'wb') as tcf:
299
            self.config.write(tcf)
300

    
301
    def setup_fabric(self):
302
        """Setup fabric environment"""
303
        self.logger.info("Setup fabric parameters..")
304
        fabric.env.user = self.config.get('Temporary Options', 'server_user')
305
        fabric.env.host_string = \
306
            self.config.get('Temporary Options', 'server_ip')
307
        fabric.env.port = self.config.getint('Temporary Options',
308
                                             'server_port')
309
        fabric.env.password = self.config.get('Temporary Options',
310
                                              'server_passwd')
311
        fabric.env.connection_attempts = 10
312
        fabric.env.shell = "/bin/bash -c"
313
        fabric.env.disable_known_hosts = True
314
        fabric.env.output_prefix = None
315

    
316
    def _check_hash_sum(self, localfile, remotefile):
317
        """Check hash sums of two files"""
318
        self.logger.debug("Check hash sum for local file %s" % localfile)
319
        hash1 = os.popen("sha256sum %s" % localfile).read().split(' ')[0]
320
        self.logger.debug("Local file has sha256 hash %s" % hash1)
321
        self.logger.debug("Check hash sum for remote file %s" % remotefile)
322
        hash2 = _run("sha256sum %s" % remotefile, False)
323
        hash2 = hash2.split(' ')[0]
324
        self.logger.debug("Remote file has sha256 hash %s" % hash2)
325
        if hash1 != hash2:
326
            self.logger.error("Hashes differ.. aborting")
327
            sys.exit(-1)
328

    
329
    @_check_fabric
330
    def clone_repo(self):
331
        """Clone Synnefo repo from slave server"""
332
        self.logger.info("Configure repositories on remote server..")
333
        self.logger.debug("Setup apt, install curl and git")
334
        cmd = """
335
        echo 'APT::Install-Suggests "false";' >> /etc/apt/apt.conf
336
        apt-get update
337
        apt-get install curl git --yes
338
        echo -e "\n\ndeb {0}" >> /etc/apt/sources.list
339
        curl https://dev.grnet.gr/files/apt-grnetdev.pub | apt-key add -
340
        apt-get update
341
        git config --global user.name {1}
342
        git config --global user.mail {2}
343
        """.format(self.config.get('Global', 'apt_repo'),
344
                   self.config.get('Global', 'git_config_name'),
345
                   self.config.get('Global', 'git_config_mail'))
346
        _run(cmd, False)
347

    
348
        synnefo_repo = self.config.get('Global', 'synnefo_repo')
349
        synnefo_branch = self.config.get('Global', 'synnefo_branch')
350
        # Currently clonning synnefo can fail unexpectedly
351
        cloned = False
352
        for i in range(3):
353
            self.logger.debug("Clone synnefo from %s" % synnefo_repo)
354
            cmd = ("git clone --branch %s %s"
355
                   % (synnefo_branch, synnefo_repo))
356
            try:
357
                _run(cmd, False)
358
                cloned = True
359
                break
360
            except:
361
                self.logger.warning("Clonning synnefo failed.. retrying %s"
362
                                    % i)
363
        if not cloned:
364
            self.logger.error("Can not clone Synnefo repo.")
365
            sys.exit(-1)
366

    
367
        deploy_repo = self.config.get('Global', 'deploy_repo')
368
        self.logger.debug("Clone snf-deploy from %s" % deploy_repo)
369
        _run("git clone --depth 1 %s" % deploy_repo, False)
370

    
371
    @_check_fabric
372
    def build_synnefo(self):
373
        """Build Synnefo packages"""
374
        self.logger.info("Build Synnefo packages..")
375
        self.logger.debug("Install development packages")
376
        cmd = """
377
        apt-get update
378
        apt-get install zlib1g-dev dpkg-dev debhelper git-buildpackage \
379
                python-dev python-all python-pip --yes
380
        pip install devflow
381
        """
382
        _run(cmd, False)
383

    
384
        if self.config.get('Global', 'patch_pydist') == "True":
385
            self.logger.debug("Patch pydist.py module")
386
            cmd = r"""
387
            sed -r -i 's/(\(\?P<name>\[A-Za-z\]\[A-Za-z0-9_\.)/\1\\\-/' \
388
                /usr/share/python/debpython/pydist.py
389
            """
390
            _run(cmd, False)
391

392
        self.logger.debug("Build snf-deploy package")
393
        cmd = """
394
        git checkout -t origin/debian
395
        git-buildpackage --git-upstream-branch=master \
396
                --git-debian-branch=debian \
397
                --git-export-dir=../snf-deploy_build-area \
398
                -uc -us
399
        """
400
        with fabric.cd("snf-deploy"):
401
            _run(cmd, True)
402

403
        self.logger.debug("Install snf-deploy package")
404
        cmd = """
405
        dpkg -i snf-deploy*.deb
406
        apt-get -f install --yes
407
        """
408
        with fabric.cd("snf-deploy_build-area"):
409
            with fabric.settings(warn_only=True):
410
                _run(cmd, True)
411

412
        self.logger.debug("Build synnefo packages")
413
        cmd = """
414
        devflow-autopkg snapshot -b ~/synnefo_build-area --no-sign
415
        """
416
        with fabric.cd("synnefo"):
417
            _run(cmd, True)
418

419
        self.logger.debug("Copy synnefo debs to snf-deploy packages dir")
420
        cmd = """
421
        cp ~/synnefo_build-area/*.deb /var/lib/snf-deploy/packages/
422
        """
423
        _run(cmd, False)
424

425
    @_check_fabric
426
    def deploy_synnefo(self):
427
        """Deploy Synnefo using snf-deploy"""
428
        self.logger.info("Deploy Synnefo..")
429
        schema = self.config.get('Global', 'schema')
430
        schema_files = os.path.join(self.ci_dir, "schemas/%s/*" % schema)
431
        self.logger.debug("Will use %s schema" % schema)
432

433
        self.logger.debug("Upload schema files to server")
434
        with fabric.quiet():
435
            fabric.put(schema_files, "/etc/snf-deploy/")
436

437
        self.logger.debug("Change password in nodes.conf file")
438
        cmd = """
439
        sed -i 's/^password =.*/password = {0}/' /etc/snf-deploy/nodes.conf
440
        """.format(fabric.env.password)
441
        _run(cmd, False)
442

443
        self.logger.debug("Run snf-deploy")
444
        cmd = """
445
        snf-deploy all --autoconf
446
        """
447
        _run(cmd, True)
448

449
    @_check_fabric
450
    def unit_test(self):
451
        """Run Synnefo unit test suite"""
452
        self.logger.info("Run Synnefo unit test suite")
453
        component = self.config.get('Unit Tests', 'component')
454

455
        self.logger.debug("Install needed packages")
456
        cmd = """
457
        pip install mock
458
        pip install factory_boy
459
        """
460
        _run(cmd, False)
461

462
        self.logger.debug("Upload tests.sh file")
463
        unit_tests_file = os.path.join(self.ci_dir, "tests.sh")
464
        with fabric.quiet():
465
            fabric.put(unit_tests_file, ".")
466

467
        self.logger.debug("Run unit tests")
468
        cmd = """
469
        bash tests.sh {0}
470
        """.format(component)
471
        _run(cmd, True)
472

473
    @_check_fabric
474
    def run_burnin(self):
475
        """Run burnin functional test suite"""
476
        self.logger.info("Run Burnin functional test suite")
477
        cmd = """
478
        auth_url=$(grep -e '^url =' .kamakirc | cut -d' ' -f3)
479
        token=$(grep -e '^token =' .kamakirc | cut -d' ' -f3)
480
        images_user=$(kamaki image list -l | grep owner | \
481
                      cut -d':' -f2 | tr -d ' ')
482
        snf-burnin --auth-url=$auth_url --token=$token \
483
            --force-flavor=2 --image-id=all \
484
            --system-images-user=$images_user \
485
            {0}
486
        log_folder=$(ls -1d /var/log/burnin/* | tail -n1)
487
        for i in $(ls $log_folder/*/details*); do
488
            echo -e "\\n\\n"
489
            echo -e "***** $i\\n"
490
            cat $i
491
        done
492
        """.format(self.config.get('Burnin', 'cmd_options'))
493
        _run(cmd, True)
494

495
    @_check_fabric
496
    def fetch_packages(self):
497
        """Download Synnefo packages"""
498
        self.logger.info("Download Synnefo packages")
499
        self.logger.debug("Create tarball with packages")
500
        cmd = """
501
        tar czf synnefo_build-area.tgz synnefo_build-area
502
        """
503
        _run(cmd, False)
504

505
        pkgs_dir = self.config.get('Global', 'pkgs_dir')
506
        self.logger.debug("Fetch packages to local dir %s" % pkgs_dir)
507
        os.makedirs(pkgs_dir)
508
        with fabric.quiet():
509
            fabric.get("synnefo_build-area.tgz", pkgs_dir)
510

511
        pkgs_file = os.path.join(pkgs_dir, "synnefo_build-area.tgz")
512
        self._check_hash_sum(pkgs_file, "synnefo_build-area.tgz")
513

514
        self.logger.debug("Untar packages file %s" % pkgs_file)
515
        os.system("cd %s; tar xzf synnefo_build-area.tgz" % pkgs_dir)
516