Statistics
| Branch: | Tag: | Revision:

root / snf-deploy / snfdeploy / fabfile.py @ 080880d9

History | View | Annotate | Download (44.8 kB)

1
# Too many lines in module pylint: disable-msg=C0302
2
# Too many arguments (7/5) pylint: disable-msg=R0913
3
"""
4
Fabric file for snf-deploy
5

6
"""
7

    
8
from __future__ import with_statement
9
from fabric.api import hide, env, settings, local, roles
10
from fabric.operations import run, put, get
11
import fabric
12
import re
13
import os
14
import shutil
15
import tempfile
16
import ast
17
from snfdeploy.lib import debug, Conf, Env, disable_color
18
from snfdeploy import massedit
19

    
20

    
21
def setup_env(args):
22
    """Setup environment"""
23
    print("Loading configuration for synnefo...")
24

    
25
    conf = Conf(args)
26
    env.env = Env(conf)
27

    
28
    env.local = args.autoconf
29
    env.key_inject = args.key_inject
30
    env.password = env.env.password
31
    env.user = env.env.user
32
    env.shell = "/bin/bash -c"
33
    env.key_filename = args.ssh_key
34

    
35
    if args.disable_colors:
36
        disable_color()
37

    
38
    if env.env.cms.hostname in \
39
            [env.env.accounts.hostname, env.env.cyclades.hostname,
40
             env.env.pithos.hostname]:
41
        env.cms_pass = True
42
    else:
43
        env.cms_pass = False
44

    
45
    if env.env.accounts.hostname in \
46
            [env.env.cyclades.hostname, env.env.pithos.hostname]:
47
        env.csrf_disable = True
48
    else:
49
        env.csrf_disable = False
50

    
51
    env.roledefs = {
52
        "nodes": env.env.ips,
53
        "ips": env.env.ips,
54
        "accounts": [env.env.accounts.ip],
55
        "cyclades": [env.env.cyclades.ip],
56
        "pithos": [env.env.pithos.ip],
57
        "cms": [env.env.cms.ip],
58
        "mq": [env.env.mq.ip],
59
        "db": [env.env.db.ip],
60
        "mq": [env.env.mq.ip],
61
        "db": [env.env.db.ip],
62
        "ns": [env.env.ns.ip],
63
        "client": [env.env.client.ip],
64
        "router": [env.env.router.ip],
65
        "stats": [env.env.stats.ip],
66
    }
67

    
68
    env.enable_lvm = False
69
    env.enable_drbd = False
70
    if ast.literal_eval(env.env.create_extra_disk) and env.env.extra_disk:
71
        env.enable_lvm = True
72
        env.enable_drbd = True
73

    
74
    env.roledefs.update({
75
        "ganeti": env.env.cluster_ips,
76
        "master": [env.env.master.ip],
77
    })
78

    
79

    
80
def install_package(package):
81
    debug(env.host, " * Installing package %s..." % package)
82
    apt_get = "export DEBIAN_FRONTEND=noninteractive ;" + \
83
              "apt-get install -y --force-yes "
84

    
85
    host_info = env.env.ips_info[env.host]
86
    env.env.update_packages(host_info.os)
87
    if ast.literal_eval(env.env.use_local_packages):
88
        with settings(warn_only=True):
89
            deb = local("ls %s/%s*%s_*.deb"
90
                        % (env.env.packages, package, host_info.os),
91
                        capture=True)
92
            if deb:
93
                debug(env.host,
94
                      " * Package %s found in %s..."
95
                      % (package, env.env.packages))
96
                try_put(deb, "/tmp/")
97
                try_run("dpkg -i /tmp/%s || "
98
                        % os.path.basename(deb) + apt_get + "-f")
99
                try_run("rm /tmp/%s" % os.path.basename(deb))
100
                return
101

    
102
    info = getattr(env.env, package)
103
    if info in \
104
            ["squeeze-backports", "squeeze", "stable",
105
             "testing", "unstable", "wheezy"]:
106
        apt_get += " -t %s %s " % (info, package)
107
    elif info:
108
        apt_get += " %s=%s " % (package, info)
109
    else:
110
        apt_get += package
111

    
112
    try_run(apt_get)
113

    
114
    return
115

    
116

    
117
@roles("ns")
118
def update_ns_for_ganeti():
119
    debug(env.host,
120
          "Updating name server entries for backend %s..."
121
          % env.env.cluster.fqdn)
122
    update_arecord(env.env.cluster)
123
    update_ptrrecord(env.env.cluster)
124
    try_run("/etc/init.d/bind9 restart")
125

    
126

    
127
@roles("ns")
128
def update_ns_for_node(node):
129
    info = env.env.nodes_info.get(node)
130
    update_arecord(info)
131
    update_ptrrecord(info)
132
    try_run("/etc/init.d/bind9 restart")
133

    
134

    
135
@roles("ns")
136
def update_arecord(host):
137
    filename = "/etc/bind/zones/" + env.env.domain
138
    cmd = """
139
    echo '{0}' >> {1}
140
    """.format(host.arecord, filename)
141
    try_run(cmd)
142

    
143

    
144
@roles("ns")
145
def update_cnamerecord(host):
146
    filename = "/etc/bind/zones/" + env.env.domain
147
    cmd = """
148
    echo '{0}' >> {1}
149
    """.format(host.cnamerecord, filename)
150
    try_run(cmd)
151

    
152

    
153
@roles("ns")
154
def update_ptrrecord(host):
155
    filename = "/etc/bind/rev/synnefo.in-addr.arpa.zone"
156
    cmd = """
157
    echo '{0}' >> {1}
158
    """.format(host.ptrrecord, filename)
159
    try_run(cmd)
160

    
161

    
162
@roles("nodes")
163
def apt_get_update():
164
    debug(env.host, "apt-get update....")
165
    try_run("apt-get update")
166

    
167

    
168
@roles("ns")
169
def setup_ns():
170
    debug(env.host, "Setting up name server..")
171
    #WARNING: this should be remove after we are done
172
    # because gevent does pick randomly nameservers and google does
173
    # not know our setup!!!!!
174
    apt_get_update()
175
    install_package("bind9")
176
    tmpl = "/etc/bind/named.conf.local"
177
    replace = {
178
        "domain": env.env.domain,
179
    }
180
    custom = customize_settings_from_tmpl(tmpl, replace)
181
    try_put(custom, tmpl)
182

    
183
    try_run("mkdir -p /etc/bind/zones")
184
    tmpl = "/etc/bind/zones/example.com"
185
    replace = {
186
        "domain": env.env.domain,
187
        "ns_node_ip": env.env.ns.ip,
188
    }
189
    custom = customize_settings_from_tmpl(tmpl, replace)
190
    remote = "/etc/bind/zones/" + env.env.domain
191
    try_put(custom, remote)
192

    
193
    try_run("mkdir -p /etc/bind/rev")
194
    tmpl = "/etc/bind/rev/synnefo.in-addr.arpa.zone"
195
    replace = {
196
        "domain": env.env.domain,
197
    }
198
    custom = customize_settings_from_tmpl(tmpl, replace)
199
    try_put(custom, tmpl)
200

    
201
    tmpl = "/etc/bind/named.conf.options"
202
    replace = {
203
        "NODE_IPS": ";".join(env.env.ips),
204
    }
205
    custom = customize_settings_from_tmpl(tmpl, replace)
206
    try_put(custom, tmpl, mode=0644)
207

    
208
    for role, info in env.env.roles.iteritems():
209
        if role == "ns":
210
            continue
211
        update_cnamerecord(info)
212
    for node, info in env.env.nodes_info.iteritems():
213
        update_arecord(info)
214
        update_ptrrecord(info)
215

    
216
    try_run("/etc/init.d/bind9 restart")
217

    
218

    
219
@roles("nodes")
220
def check_dhcp():
221
    debug(env.host, "Checking IPs for synnefo..")
222
    for n, info in env.env.nodes_info.iteritems():
223
        try_run("ping -c 1 " + info.ip)
224

    
225

    
226
@roles("nodes")
227
def check_dns():
228
    debug(env.host, "Checking fqdns for synnefo..")
229
    for n, info in env.env.nodes_info.iteritems():
230
        try_run("ping -c 1 " + info.fqdn)
231

    
232
    for n, info in env.env.roles.iteritems():
233
        try_run("ping -c 1 " + info.fqdn)
234

    
235

    
236
@roles("nodes")
237
def check_connectivity():
238
    debug(env.host, "Checking internet connectivity..")
239
    try_run("ping -c 1 www.google.com")
240

    
241

    
242
@roles("nodes")
243
def check_ssh():
244
    debug(env.host, "Checking password-less ssh..")
245
    for n, info in env.env.nodes_info.iteritems():
246
        try_run("ssh " + info.fqdn + "  date")
247

    
248

    
249
@roles("ips")
250
def add_keys():
251
    if not env.key_inject:
252
        debug(env.host, "Skipping ssh keys injection..")
253
        return
254
    else:
255
        debug(env.host, "Adding rsa/dsa keys..")
256
    try_run("mkdir -p /root/.ssh")
257
    cmd = """
258
for f in $(ls /root/.ssh/*); do
259
  cp $f $f.bak
260
done
261
    """
262
    try_run(cmd)
263
    files = ["authorized_keys", "id_dsa", "id_dsa.pub",
264
             "id_rsa", "id_rsa.pub"]
265
    for f in files:
266
        tmpl = "/root/.ssh/" + f
267
        replace = {}
268
        custom = customize_settings_from_tmpl(tmpl, replace)
269
        try_put(custom, tmpl, mode=0600)
270

    
271
    cmd = """
272
if [ -e /root/.ssh/authorized_keys.bak ]; then
273
  cat /root/.ssh/authorized_keys.bak >> /root/.ssh/authorized_keys
274
fi
275
    """
276
    debug(env.host, "Updating exising authorized keys..")
277
    try_run(cmd)
278

    
279

    
280
@roles("ips")
281
def setup_resolv_conf():
282
    debug(env.host, "Tweak /etc/resolv.conf...")
283
    try_run("/etc/init.d/network-manager stop", abort=False)
284
    tmpl = "/etc/dhcp/dhclient-enter-hooks.d/nodnsupdate"
285
    replace = {}
286
    custom = customize_settings_from_tmpl(tmpl, replace)
287
    try_put(custom, tmpl, mode=0644)
288
    try_run("cp /etc/resolv.conf /etc/resolv.conf.bak")
289
    tmpl = "/etc/resolv.conf"
290
    replace = {
291
        "domain": env.env.domain,
292
        "ns_node_ip": env.env.ns.ip,
293
    }
294
    custom = customize_settings_from_tmpl(tmpl, replace)
295
    try:
296
        try_put(custom, tmpl)
297
        cmd = """
298
        echo "\
299
# This has been generated automatically by snf-deploy, at
300
# $(date).
301
# The immutable bit (+i attribute) has been used to avoid it being
302
# overwritten by software such as NetworkManager or resolvconf.
303
# Use lsattr/chattr to view or modify its file attributes.
304

305

306
$(cat {0})" > {0}
307
""".format(tmpl)
308
        try_run(cmd)
309
    except:
310
        pass
311
    try_run("chattr +i /etc/resolv.conf")
312

    
313

    
314
@roles("ips")
315
def setup_hosts():
316
    debug(env.host, "Tweaking /etc/hosts and ssh_config files...")
317
    try_run("echo StrictHostKeyChecking no >> /etc/ssh/ssh_config")
318
    cmd = "sed -i 's/^127.*$/127.0.0.1 localhost/g' /etc/hosts "
319
    try_run(cmd)
320
    host_info = env.env.ips_info[env.host]
321
    cmd = "hostname %s" % host_info.hostname
322
    try_run(cmd)
323
    cmd = "echo %s > /etc/hostname" % host_info.hostname
324
    try_run(cmd)
325

    
326

    
327
def try_run(cmd, abort=True):
328
    try:
329
        if env.local:
330
            return local(cmd, capture=True)
331
        else:
332
            return run(cmd)
333
    except BaseException as e:
334
        if abort:
335
            fabric.utils.abort(e)
336
        else:
337
            debug(env.host, "WARNING: command failed. Continuing anyway...")
338

    
339

    
340
def try_put(local_path=None, remote_path=None, abort=True, **kwargs):
341
    try:
342
        put(local_path=local_path, remote_path=remote_path, **kwargs)
343
    except BaseException as e:
344
        if abort:
345
            fabric.utils.abort(e)
346
        else:
347
            debug(env.host, "WARNING: command failed. Continuing anyway...")
348

    
349

    
350
def try_get(remote_path, local_path=None, abort=True, **kwargs):
351
    try:
352
        get(remote_path, local_path=local_path, **kwargs)
353
    except BaseException as e:
354
        if abort:
355
            fabric.utils.abort(e)
356
        else:
357
            debug(env.host, "WARNING: command failed. Continuing anyway...")
358

    
359

    
360
def create_bridges():
361
    debug(env.host, " * Creating bridges...")
362
    install_package("bridge-utils")
363
    cmd = """
364
    brctl addbr {0} ; ip link set {0} up
365
    """.format(env.env.common_bridge)
366
    try_run(cmd)
367

    
368

    
369
def connect_bridges():
370
    debug(env.host, " * Connecting bridges...")
371
    #cmd = """
372
    #brctl addif {0} {1}
373
    #""".format(env.env.common_bridge, env.env.public_iface)
374
    #try_run(cmd)
375

    
376

    
377
@roles("ganeti")
378
def setup_net_infra():
379
    debug(env.host, "Setup networking infrastracture..")
380
    create_bridges()
381
    connect_bridges()
382

    
383

    
384
@roles("ganeti")
385
def setup_lvm():
386
    debug(env.host, "create volume group %s for ganeti.." % env.env.vg)
387
    if env.enable_lvm:
388
        install_package("lvm2")
389
        cmd = """
390
        pvcreate {0}
391
        vgcreate {1} {0}
392
        """.format(env.env.extra_disk, env.env.vg)
393
        try_run(cmd)
394

    
395

    
396
def customize_settings_from_tmpl(tmpl, replace):
397
    debug(env.host, " * Customizing template %s..." % tmpl)
398
    local = env.env.templates + tmpl
399
    _, custom = tempfile.mkstemp()
400
    shutil.copyfile(local, custom)
401
    for k, v in replace.iteritems():
402
        regex = "re.sub('%{0}%', '{1}', line)".format(k.upper(), v)
403
        massedit.edit_files([custom], [regex], dry_run=False)
404

    
405
    return custom
406

    
407

    
408
@roles("nodes")
409
def setup_apt():
410
    debug(env.host, "Setting up apt sources...")
411
    install_package("curl")
412
    cmd = """
413
    echo 'APT::Install-Suggests "false";' >> /etc/apt/apt.conf
414
    curl -k https://dev.grnet.gr/files/apt-grnetdev.pub | apt-key add -
415
    """
416
    try_run(cmd)
417
    host_info = env.env.ips_info[env.host]
418
    if host_info.os == "squeeze":
419
        tmpl = "/etc/apt/sources.list.d/synnefo.squeeze.list"
420
    else:
421
        tmpl = "/etc/apt/sources.list.d/synnefo.wheezy.list"
422
    replace = {}
423
    custom = customize_settings_from_tmpl(tmpl, replace)
424
    try_put(custom, tmpl)
425
    apt_get_update()
426

    
427

    
428
@roles("cyclades", "cms", "pithos", "accounts")
429
def restart_services():
430
    debug(env.host, " * Restarting apache2 and gunicorn...")
431
    try_run("/etc/init.d/gunicorn restart")
432
    try_run("/etc/init.d/apache2 restart")
433

    
434

    
435
def setup_gunicorn():
436
    debug(env.host, " * Setting up gunicorn...")
437
    install_package("gunicorn")
438
    try_run("chown root.www-data /var/log/gunicorn")
439
    tmpl = "/etc/gunicorn.d/synnefo"
440
    replace = {}
441
    custom = customize_settings_from_tmpl(tmpl, replace)
442
    try_put(custom, tmpl, mode=0644)
443
    try_run("/etc/init.d/gunicorn restart")
444

    
445

    
446
def setup_apache():
447
    debug(env.host, " * Setting up apache2...")
448
    host_info = env.env.ips_info[env.host]
449
    install_package("apache2")
450
    tmpl = "/etc/apache2/sites-available/synnefo"
451
    replace = {
452
        "HOST": host_info.fqdn,
453
    }
454
    custom = customize_settings_from_tmpl(tmpl, replace)
455
    try_put(custom, tmpl)
456
    tmpl = "/etc/apache2/sites-available/synnefo-ssl"
457
    custom = customize_settings_from_tmpl(tmpl, replace)
458
    try_put(custom, tmpl)
459
    cmd = """
460
    a2enmod ssl
461
    a2enmod rewrite
462
    a2dissite default
463
    a2ensite synnefo
464
    a2ensite synnefo-ssl
465
    a2enmod headers
466
    a2enmod proxy_http
467
    a2dismod autoindex
468
    """
469
    try_run(cmd)
470
    try_run("/etc/init.d/apache2 restart")
471

    
472

    
473
@roles("mq")
474
def setup_mq():
475
    debug(env.host, "Setting up RabbitMQ...")
476
    install_package("rabbitmq-server")
477
    cmd = """
478
    rabbitmqctl add_user {0} {1}
479
    rabbitmqctl set_permissions {0} ".*" ".*" ".*"
480
    rabbitmqctl delete_user guest
481
    rabbitmqctl set_user_tags {0} administrator
482
    """.format(env.env.synnefo_user, env.env.synnefo_rabbitmq_passwd)
483
    try_run(cmd)
484
    try_run("/etc/init.d/rabbitmq-server restart")
485

    
486

    
487
@roles("db")
488
def allow_access_in_db(ip, user="all", method="md5"):
489
    cmd = """
490
    pg_hba=$(ls /etc/postgresql/*/main/pg_hba.conf)
491
    echo host all {0} {1}/32 {2} >> $pg_hba
492
    """.format(user, ip, method)
493
    try_run(cmd)
494
    cmd = """
495
    pg_hba=$(ls /etc/postgresql/*/main/pg_hba.conf)
496
    sed -i 's/\(host.*127.0.0.1.*\)md5/\\1trust/' $pg_hba
497
    """
498
    try_run(cmd)
499
    try_run("/etc/init.d/postgresql restart")
500

    
501

    
502
@roles("db")
503
def setup_db():
504
    debug(env.host, "Setting up DataBase server...")
505
    install_package("postgresql")
506

    
507
    tmpl = "/tmp/db-init.psql"
508
    replace = {
509
        "synnefo_user": env.env.synnefo_user,
510
        "synnefo_db_passwd": env.env.synnefo_db_passwd,
511
        }
512
    custom = customize_settings_from_tmpl(tmpl, replace)
513
    try_put(custom, tmpl)
514
    cmd = 'su - postgres -c "psql -w -f %s" ' % tmpl
515
    try_run(cmd)
516
    cmd = """
517
    conf=$(ls /etc/postgresql/*/main/postgresql.conf)
518
    echo "listen_addresses = '*'" >> $conf
519
    """
520
    try_run(cmd)
521

    
522
    if env.env.testing_vm:
523
        cmd = """
524
        conf=$(ls /etc/postgresql/*/main/postgresql.conf)
525
        echo "fsync=off\nsynchronous_commit=off\nfull_page_writes=off" >> $conf
526
        """
527
        try_run(cmd)
528

    
529
    allow_access_in_db(env.host, "all", "trust")
530
    try_run("/etc/init.d/postgresql restart")
531

    
532

    
533
@roles("db")
534
def destroy_db():
535
    try_run("""su - postgres -c ' psql -w -c "drop database snf_apps" '""")
536
    try_run("""su - postgres -c ' psql -w -c "drop database snf_pithos" '""")
537

    
538

    
539
def setup_webproject():
540
    debug(env.host, " * Setting up snf-webproject...")
541
    with settings(hide("everything")):
542
        try_run("ping -c1 " + env.env.db.ip)
543
    setup_common()
544
    install_package("snf-webproject")
545
    install_package("python-psycopg2")
546
    install_package("python-gevent")
547
    install_package("python-django")
548
    tmpl = "/etc/synnefo/webproject.conf"
549
    replace = {
550
        "synnefo_user": env.env.synnefo_user,
551
        "synnefo_db_passwd": env.env.synnefo_db_passwd,
552
        "db_node": env.env.db.ip,
553
        "domain": env.env.domain,
554
    }
555
    custom = customize_settings_from_tmpl(tmpl, replace)
556
    try_put(custom, tmpl, mode=0644)
557
    with settings(host_string=env.env.db.ip):
558
        host_info = env.env.ips_info[env.host]
559
        allow_access_in_db(host_info.ip, "all", "trust")
560
    try_run("/etc/init.d/gunicorn restart")
561

    
562

    
563
def setup_common():
564
    debug(env.host, " * Setting up snf-common...")
565
    host_info = env.env.ips_info[env.host]
566
    install_package("python-objpool")
567
    install_package("snf-common")
568
    install_package("python-astakosclient")
569
    install_package("snf-django-lib")
570
    install_package("snf-branding")
571
    tmpl = "/etc/synnefo/common.conf"
572
    replace = {
573
        #FIXME:
574
        "EMAIL_SUBJECT_PREFIX": env.host,
575
        "domain": env.env.domain,
576
        "HOST": host_info.fqdn,
577
        "MAIL_DIR": env.env.mail_dir,
578
    }
579
    custom = customize_settings_from_tmpl(tmpl, replace)
580
    try_put(custom, tmpl, mode=0644)
581
    try_run("mkdir -p {0}; chmod 777 {0}".format(env.env.mail_dir))
582
    try_run("/etc/init.d/gunicorn restart")
583

    
584

    
585
@roles("accounts")
586
def astakos_loaddata():
587
    debug(env.host, " * Loading initial data to astakos...")
588
    cmd = """
589
    snf-manage loaddata groups
590
    """
591
    try_run(cmd)
592

    
593

    
594
@roles("accounts")
595
def astakos_register_components():
596
    debug(env.host, " * Register services in astakos...")
597

    
598
    cyclades_base_url = "https://%s/cyclades" % env.env.cyclades.fqdn
599
    pithos_base_url = "https://%s/pithos" % env.env.pithos.fqdn
600
    astakos_base_url = "https://%s/astakos" % env.env.accounts.fqdn
601

    
602
    cmd = """
603
    snf-manage component-add "home" --ui-url https://{0}
604
    snf-manage component-add "cyclades" --base-url {1} --ui-url {1}/ui
605
    snf-manage component-add "pithos" --base-url {2} --ui-url {2}/ui
606
    snf-manage component-add "astakos" --base-url {3} --ui-url {3}/ui
607
    """.format(env.env.cms.fqdn, cyclades_base_url,
608
               pithos_base_url, astakos_base_url)
609
    try_run(cmd)
610

    
611

    
612
@roles("accounts")
613
def astakos_register_pithos_view():
614
    debug(env.host, " * Register pithos view as oauth2 client...")
615

    
616
    pithos_base_url = "https://%s/pithos" % env.env.pithos.fqdn
617

    
618
    cmd = """
619
    snf-manage oauth2-client-add pithos-view --secret={0} --is-trusted \
620
    --url {1}
621
    """.format(env.env.oa2_secret, '%s/ui/view' % pithos_base_url)
622
    try_run(cmd)
623

    
624

    
625
@roles("accounts")
626
def add_user():
627
    debug(env.host, " * adding user %s to astakos..." % env.env.user_email)
628
    email = env.env.user_email
629
    name = env.env.user_name
630
    lastname = env.env.user_lastname
631
    passwd = env.env.user_passwd
632
    cmd = """
633
    snf-manage user-add {0} {1} {2}
634
    """.format(email, name, lastname)
635
    try_run(cmd)
636
    with settings(host_string=env.env.db.ip):
637
        uid, user_auth_token, user_uuid = get_auth_token_from_db(email)
638
    cmd = """
639
    snf-manage user-modify --password {0} {1}
640
    """.format(passwd, uid)
641
    try_run(cmd)
642

    
643

    
644
@roles("accounts")
645
def activate_user(user_email=None):
646
    if not user_email:
647
        user_email = env.env.user_email
648
    debug(env.host, " * Activate user %s..." % user_email)
649
    with settings(host_string=env.env.db.ip):
650
        uid, user_auth_token, user_uuid = get_auth_token_from_db(user_email)
651

    
652
    cmd = """
653
    snf-manage user-modify --verify {0}
654
    snf-manage user-modify --accept {0}
655
    """.format(uid)
656
    try_run(cmd)
657

    
658

    
659
@roles("accounts")
660
def setup_astakos():
661
    debug(env.host, "Setting up snf-astakos-app...")
662
    setup_gunicorn()
663
    setup_apache()
664
    setup_webproject()
665
    install_package("python-django-south")
666
    install_package("snf-astakos-app")
667
    install_package("kamaki")
668

    
669
    tmpl = "/etc/synnefo/astakos.conf"
670
    replace = {
671
        "ACCOUNTS": env.env.accounts.fqdn,
672
        "domain": env.env.domain,
673
        "CYCLADES": env.env.cyclades.fqdn,
674
        "PITHOS": env.env.pithos.fqdn,
675
    }
676
    custom = customize_settings_from_tmpl(tmpl, replace)
677
    try_put(custom, tmpl, mode=0644)
678
    if env.csrf_disable:
679
        cmd = """
680
cat <<EOF >> /etc/synnefo/astakos.conf
681
try:
682
  MIDDLEWARE_CLASSES.remove('django.middleware.csrf.CsrfViewMiddleware')
683
except:
684
  pass
685
EOF
686
"""
687
        try_run(cmd)
688

    
689
    try_run("/etc/init.d/gunicorn restart")
690

    
691
    cmd = """
692
    snf-manage syncdb --noinput
693
    snf-manage migrate im --delete-ghost-migrations
694
    snf-manage migrate quotaholder_app
695
    snf-manage migrate oa2
696
    """
697
    try_run(cmd)
698

    
699

    
700
@roles("accounts")
701
def get_service_details(service="pithos"):
702
    debug(env.host,
703
          " * Getting registered details for %s service..." % service)
704
    result = try_run("snf-manage component-list -o id,name,token")
705
    r = re.compile(r".*%s.*" % service, re.M)
706
    service_id, _, service_token = r.search(result).group().split()
707
    # print("%s: %s %s" % (service, service_id, service_token))
708
    return (service_id, service_token)
709

    
710

    
711
@roles("db")
712
def get_auth_token_from_db(user_email=None):
713
    if not user_email:
714
        user_email = env.env.user_email
715
    debug(env.host,
716
          " * Getting authentication token and uuid for user %s..."
717
          % user_email)
718
    cmd = """
719
echo "select id, auth_token, uuid, email from auth_user, im_astakosuser \
720
where auth_user.id = im_astakosuser.user_ptr_id and auth_user.email = '{0}';" \
721
> /tmp/psqlcmd
722
su - postgres -c  "psql -w -d snf_apps -f /tmp/psqlcmd"
723
""".format(user_email)
724

    
725
    result = try_run(cmd)
726
    r = re.compile(r"(\d+)[ |]*(\S+)[ |]*(\S+)[ |]*" + user_email, re.M)
727
    match = r.search(result)
728
    uid, user_auth_token, user_uuid = match.groups()
729
    # print("%s: %s %s %s" % ( user_email, uid, user_auth_token, user_uuid))
730

    
731
    return (uid, user_auth_token, user_uuid)
732

    
733

    
734
@roles("cms")
735
def cms_loaddata():
736
    debug(env.host, " * Loading cms initial data...")
737
    if env.cms_pass:
738
        debug(env.host, "Aborting. Prerequisites not met.")
739
        return
740
    tmpl = "/tmp/sites.json"
741
    replace = {}
742
    custom = customize_settings_from_tmpl(tmpl, replace)
743
    try_put(custom, tmpl)
744

    
745
    tmpl = "/tmp/page.json"
746
    replace = {}
747
    custom = customize_settings_from_tmpl(tmpl, replace)
748
    try_put(custom, tmpl)
749

    
750
    cmd = """
751
    snf-manage loaddata /tmp/sites.json
752
    snf-manage loaddata /tmp/page.json
753
    snf-manage createsuperuser --username=admin --email=admin@{0} --noinput
754
    """.format(env.env.domain)
755
    try_run(cmd)
756

    
757

    
758
@roles("cms")
759
def setup_cms():
760
    debug(env.host, "Setting up cms...")
761
    if env.cms_pass:
762
        debug(env.host, "Aborting. Prerequisites not met.")
763
        return
764
    with settings(hide("everything")):
765
        try_run("ping -c1 accounts." + env.env.domain)
766
    setup_gunicorn()
767
    setup_apache()
768
    setup_webproject()
769
    install_package("snf-cloudcms")
770

    
771
    tmpl = "/etc/synnefo/cms.conf"
772
    replace = {
773
        "ACCOUNTS": env.env.accounts.fqdn,
774
        }
775
    custom = customize_settings_from_tmpl(tmpl, replace)
776
    try_put(custom, tmpl, mode=0644)
777
    try_run("/etc/init.d/gunicorn restart")
778

    
779
    cmd = """
780
    snf-manage syncdb
781
    snf-manage migrate --delete-ghost-migrations
782
    """.format(env.env.domain)
783
    try_run(cmd)
784

    
785

    
786
def setup_nfs_dirs():
787
    debug(env.host, " * Creating NFS mount point for pithos and ganeti...")
788
    cmd = """
789
    mkdir -p {0}
790
    cd {0}
791
    mkdir -p data
792
    chown www-data:www-data data
793
    chmod g+ws data
794
    mkdir -p {1}
795
    """.format(env.env.pithos_dir, env.env.image_dir)
796
    try_run(cmd)
797

    
798

    
799
@roles("nodes")
800
def setup_nfs_clients():
801
    if env.host == env.env.pithos.ip:
802
        return
803

    
804
    host_info = env.env.ips_info[env.host]
805
    debug(env.host, " * Mounting pithos NFS mount point...")
806
    with settings(hide("everything")):
807
        try_run("ping -c1 " + env.env.pithos.hostname)
808
    with settings(host_string=env.env.pithos.ip):
809
        update_nfs_exports(host_info.ip)
810

    
811
    install_package("nfs-common")
812
    for d in [env.env.pithos_dir, env.env.image_dir]:
813
        try_run("mkdir -p " + d)
814
        cmd = """
815
echo "{0}:{1} {1}  nfs defaults,rw,noatime,rsize=131072,\
816
wsize=131072,timeo=14,intr,noacl" >> /etc/fstab
817
""".format(env.env.pithos.ip, d)
818
        try_run(cmd)
819
        try_run("mount " + d)
820

    
821

    
822
@roles("pithos")
823
def update_nfs_exports(ip):
824
    tmpl = "/tmp/exports"
825
    replace = {
826
        "pithos_dir": env.env.pithos_dir,
827
        "image_dir": env.env.image_dir,
828
        "ip": ip,
829
    }
830
    custom = customize_settings_from_tmpl(tmpl, replace)
831
    try_put(custom, tmpl)
832
    try_run("cat %s >> /etc/exports" % tmpl)
833
    try_run("/etc/init.d/nfs-kernel-server restart")
834

    
835

    
836
@roles("pithos")
837
def setup_nfs_server():
838
    debug(env.host, " * Setting up NFS server for pithos...")
839
    setup_nfs_dirs()
840
    install_package("nfs-kernel-server")
841

    
842

    
843
@roles("pithos")
844
def setup_pithos():
845
    debug(env.host, "Setting up snf-pithos-app...")
846
    with settings(hide("everything")):
847
        try_run("ping -c1 accounts." + env.env.domain)
848
        try_run("ping -c1 " + env.env.db.ip)
849
    setup_gunicorn()
850
    setup_apache()
851
    setup_webproject()
852

    
853
    with settings(host_string=env.env.accounts.ip):
854
        service_id, service_token = get_service_details("pithos")
855

    
856
    install_package("kamaki")
857
    install_package("snf-pithos-backend")
858
    install_package("snf-pithos-app")
859
    tmpl = "/etc/synnefo/pithos.conf"
860
    replace = {
861
        "ACCOUNTS": env.env.accounts.fqdn,
862
        "PITHOS": env.env.pithos.fqdn,
863
        "db_node": env.env.db.ip,
864
        "synnefo_user": env.env.synnefo_user,
865
        "synnefo_db_passwd": env.env.synnefo_db_passwd,
866
        "pithos_dir": env.env.pithos_dir,
867
        "PITHOS_SERVICE_TOKEN": service_token,
868
        "oa2_secret": env.env.oa2_secret,
869
        }
870
    custom = customize_settings_from_tmpl(tmpl, replace)
871
    try_put(custom, tmpl, mode=0644)
872
    try_run("/etc/init.d/gunicorn restart")
873

    
874
    install_package("snf-pithos-webclient")
875
    tmpl = "/etc/synnefo/webclient.conf"
876
    replace = {
877
        "ACCOUNTS": env.env.accounts.fqdn,
878
        "PITHOS_UI_CLOUDBAR_ACTIVE_SERVICE": service_id,
879
        }
880
    custom = customize_settings_from_tmpl(tmpl, replace)
881
    try_put(custom, tmpl, mode=0644)
882

    
883
    try_run("/etc/init.d/gunicorn restart")
884
    #TOFIX: the previous command lets pithos-backend create blocks and maps
885
    #       with root owner
886
    try_run("chown -R www-data:www-data %s/data " % env.env.pithos_dir)
887
    try_run("pithos-migrate stamp head")
888
    #try_run("pithos-migrate upgrade head")
889

    
890

    
891
@roles("ganeti")
892
def setup_ganeti():
893
    debug(env.host, "Setting up snf-ganeti...")
894
    node_info = env.env.ips_info[env.host]
895
    with settings(hide("everything")):
896
        #if env.enable_lvm:
897
        #    try_run("vgs " + env.env.vg)
898
        try_run("getent hosts " + env.env.cluster.fqdn)
899
        try_run("getent hosts %s | grep -v ^127" % env.host)
900
        try_run("hostname -f | grep " + node_info.fqdn)
901
        #try_run("ip link show " + env.env.common_bridge)
902
        #try_run("ip link show " + env.env.common_bridge)
903
        #try_run("apt-get update")
904
    install_package("qemu-kvm")
905
    install_package("python-bitarray")
906
    install_package("ganeti-haskell")
907
    install_package("ganeti-htools")
908
    install_package("snf-ganeti")
909
    try_run("mkdir -p /srv/ganeti/file-storage/")
910
    cmd = """
911
cat <<EOF > /etc/ganeti/file-storage-paths
912
/srv/ganeti/file-storage
913
/srv/ganeti/shared-file-storage
914
EOF
915
"""
916
    try_run(cmd)
917

    
918

    
919
@roles("master")
920
def add_rapi_user():
921
    debug(env.host, " * Adding RAPI user to Ganeti backend...")
922
    cmd = """
923
    echo -n "{0}:Ganeti Remote API:{1}" | openssl md5 | sed 's/^.* //'
924
    """.format(env.env.synnefo_user, env.env.synnefo_rapi_passwd)
925
    result = try_run(cmd)
926
    if result.startswith("(stdin)= "):
927
        result = result.split("(stdin)= ")[1]
928
    cmd = """
929
    echo "{0} {1}{2} write" >> /var/lib/ganeti/rapi/users
930
    """.format(env.env.synnefo_user, '{ha1}', result)
931
    try_run(cmd)
932
    try_run("/etc/init.d/ganeti restart")
933

    
934

    
935
@roles("master")
936
def add_nodes():
937
    nodes = env.env.cluster_nodes.split(",")
938
    nodes.remove(env.env.master_node)
939
    debug(env.host, " * Adding nodes to Ganeti backend...")
940
    for n in nodes:
941
        add_node(n)
942

    
943

    
944
@roles("master")
945
def add_node(node):
946
    node_info = env.env.nodes_info[node]
947
    debug(env.host, " * Adding node %s to Ganeti backend..." % node_info.fqdn)
948
    cmd = "gnt-node add --no-ssh-key-check --master-capable=yes " + \
949
          "--vm-capable=yes " + node_info.fqdn
950
    try_run(cmd)
951

    
952

    
953
@roles("ganeti")
954
def enable_drbd():
955
    if env.enable_drbd:
956
        debug(env.host, " * Enabling DRBD...")
957
        install_package("drbd8-utils")
958
        try_run("modprobe drbd minor_count=255 usermode_helper=/bin/true")
959
        try_run("echo drbd minor_count=255 usermode_helper=/bin/true " +
960
                ">> /etc/modules")
961

    
962

    
963
@roles("master")
964
def setup_drbd_dparams():
965
    if env.enable_drbd:
966
        debug(env.host,
967
              " * Twicking drbd related disk parameters in Ganeti...")
968
        cmd = """
969
        gnt-cluster modify --disk-parameters=drbd:metavg={0}
970
        gnt-group modify --disk-parameters=drbd:metavg={0} default
971
        """.format(env.env.vg)
972
        try_run(cmd)
973

    
974

    
975
@roles("master")
976
def enable_lvm():
977
    if env.enable_lvm:
978
        debug(env.host, " * Enabling LVM...")
979
        cmd = """
980
        gnt-cluster modify --vg-name={0}
981
        """.format(env.env.vg)
982
        try_run(cmd)
983
    else:
984
        debug(env.host, " * Disabling LVM...")
985
        try_run("gnt-cluster modify --no-lvm-storage")
986

    
987

    
988
@roles("master")
989
def destroy_cluster():
990
    debug(env.host, " * Destroying Ganeti cluster...")
991
    #TODO: remove instances first
992
    allnodes = env.env.cluster_hostnames[:]
993
    allnodes.remove(env.host)
994
    for n in allnodes:
995
        host_info = env.env.ips_info[env.host]
996
        debug(env.host, " * Removing node %s..." % n)
997
        cmd = "gnt-node remove  " + host_info.fqdn
998
        try_run(cmd)
999
    try_run("gnt-cluster destroy --yes-do-it")
1000

    
1001

    
1002
@roles("master")
1003
def init_cluster():
1004
    debug(env.host, " * Initializing Ganeti backend...")
1005
    # extra = ""
1006
    # if env.enable_lvm:
1007
    #     extra += " --vg-name={0} ".format(env.env.vg)
1008
    # else:
1009
    #     extra += " --no-lvm-storage "
1010
    # if not env.enable_drbd:
1011
    #     extra += " --no-drbd-storage "
1012
    extra = " --no-lvm-storage --no-drbd-storage "
1013
    cmd = """
1014
    gnt-cluster init --enabled-hypervisors=kvm \
1015
        {0} \
1016
        --nic-parameters link={1},mode=bridged \
1017
        --master-netdev {2} \
1018
        --default-iallocator hail \
1019
        --specs-nic-count min=0,max=8 \
1020
        --hypervisor-parameters kvm:kernel_path=,vnc_bind_address=0.0.0.0 \
1021
        --no-ssh-init --no-etc-hosts \
1022
        {3}
1023
    """.format(extra, env.env.common_bridge,
1024
               env.env.cluster_netdev, env.env.cluster.fqdn)
1025
    try_run(cmd)
1026
    cmd = """gnt-cluster modify --enabled-disk-templates file,plain,ext"""
1027
    try_run(cmd)
1028

    
1029

    
1030
@roles("ganeti")
1031
def debootstrap():
1032
    install_package("ganeti-instance-debootstrap")
1033

    
1034

    
1035
@roles("ganeti")
1036
def setup_image_host():
1037
    debug(env.host, "Setting up snf-image...")
1038
    install_package("snf-pithos-backend")
1039
    install_package("snf-image")
1040
    try_run("mkdir -p %s" % env.env.image_dir)
1041
    tmpl = "/etc/default/snf-image"
1042
    replace = {
1043
        "synnefo_user": env.env.synnefo_user,
1044
        "synnefo_db_passwd": env.env.synnefo_db_passwd,
1045
        "pithos_dir": env.env.pithos_dir,
1046
        "db_node": env.env.db.ip,
1047
        "image_dir": env.env.image_dir,
1048
    }
1049
    custom = customize_settings_from_tmpl(tmpl, replace)
1050
    try_put(custom, tmpl)
1051

    
1052

    
1053
@roles("ganeti")
1054
def setup_image_helper():
1055
    debug(env.host, " * Updating helper image...")
1056
    cmd = """
1057
    snf-image-update-helper -y
1058
    """
1059
    try_run(cmd)
1060

    
1061

    
1062
@roles("ganeti")
1063
def setup_gtools():
1064
    debug(env.host, " * Setting up snf-cyclades-gtools...")
1065
    with settings(hide("everything")):
1066
        try_run("ping -c1 " + env.env.mq.ip)
1067
    setup_common()
1068
    install_package("snf-cyclades-gtools")
1069
    tmpl = "/etc/synnefo/gtools.conf"
1070
    replace = {
1071
        "synnefo_user": env.env.synnefo_user,
1072
        "synnefo_rabbitmq_passwd": env.env.synnefo_rabbitmq_passwd,
1073
        "mq_node": env.env.mq.ip,
1074
    }
1075
    custom = customize_settings_from_tmpl(tmpl, replace)
1076
    try_put(custom, tmpl)
1077

    
1078
    cmd = """
1079
    sed -i 's/false/true/' /etc/default/snf-ganeti-eventd
1080
    /etc/init.d/snf-ganeti-eventd start
1081
    """
1082
    try_run(cmd)
1083

    
1084

    
1085
@roles("ganeti")
1086
def setup_iptables():
1087
    debug(env.host, " * Setting up iptables to mangle DHCP requests...")
1088
    cmd = """
1089
    iptables -t mangle -A PREROUTING -i br+ -p udp -m udp --dport 67 \
1090
            -j NFQUEUE --queue-num 42
1091
    iptables -t mangle -A PREROUTING -i tap+ -p udp -m udp --dport 67 \
1092
            -j NFQUEUE --queue-num 42
1093
    iptables -t mangle -A PREROUTING -i prv+ -p udp -m udp --dport 67 \
1094
            -j NFQUEUE --queue-num 42
1095

1096
    ip6tables -t mangle -A PREROUTING -i br+ -p ipv6-icmp -m icmp6 \
1097
            --icmpv6-type 133 -j NFQUEUE --queue-num 43
1098
    ip6tables -t mangle -A PREROUTING -i br+ -p ipv6-icmp -m icmp6 \
1099
            --icmpv6-type 135 -j NFQUEUE --queue-num 44
1100
    """
1101
    try_run(cmd)
1102

    
1103

    
1104
@roles("ganeti")
1105
def setup_network():
1106
    debug(env.host,
1107
          "Setting up networking for Ganeti instances (nfdhcpd, etc.)...")
1108
    install_package("python-nfqueue")
1109
    install_package("nfdhcpd")
1110
    tmpl = "/etc/nfdhcpd/nfdhcpd.conf"
1111
    replace = {
1112
        "ns_node_ip": env.env.ns.ip
1113
    }
1114
    custom = customize_settings_from_tmpl(tmpl, replace)
1115
    try_put(custom, tmpl)
1116
    try_run("/etc/init.d/nfdhcpd restart")
1117

    
1118
    install_package("snf-network")
1119
    cmd = """
1120
sed -i 's/MAC_MASK.*/MAC_MASK = ff:ff:f0:00:00:00/' /etc/default/snf-network
1121
    """
1122
    try_run(cmd)
1123

    
1124

    
1125
@roles("router")
1126
def setup_router():
1127
    debug(env.host, " * Setting up internal router for NAT...")
1128
    cmd = """
1129
    echo 1 > /proc/sys/net/ipv4/ip_forward
1130
    iptables -t nat -A POSTROUTING -s {0} -o {3} -j MASQUERADE
1131
    ip addr add {1} dev {2}
1132
    ip route add {0} dev {2} src {1}
1133
    """.format(env.env.synnefo_public_network_subnet,
1134
               env.env.synnefo_public_network_gateway,
1135
               env.env.common_bridge, env.env.public_iface)
1136
    try_run(cmd)
1137

    
1138

    
1139
@roles("cyclades")
1140
def cyclades_loaddata():
1141
    debug(env.host, " * Loading initial data for cyclades...")
1142
    try_run("snf-manage flavor-create %s %s %s %s" % (env.env.flavor_cpu,
1143
                                                      env.env.flavor_ram,
1144
                                                      env.env.flavor_disk,
1145
                                                      env.env.flavor_storage))
1146
    #run("snf-manage loaddata flavors")
1147

    
1148

    
1149
@roles("ganeti", "stats")
1150
def setup_collectd():
1151
    install_package("collectd")
1152
    tmpl = "/etc/collectd/collectd.conf"
1153
    replace = {}
1154
    custom = customize_settings_from_tmpl(tmpl, replace)
1155
    try_put(custom, tmpl, mode=0644)
1156

    
1157

    
1158
@roles("ganeti")
1159
def setup_ganeti_collectd():
1160
    setup_collectd()
1161

    
1162
    tmpl = "/etc/collectd/passwd"
1163
    replace = {}
1164
    custom = customize_settings_from_tmpl(tmpl, replace)
1165
    try_put(custom, tmpl, mode=0644)
1166

    
1167
    tmpl = "/etc/collectd/synnefo-ganeti.conf"
1168
    replace = {
1169
        "STATS": env.env.stats.fqdn,
1170
        }
1171
    custom = customize_settings_from_tmpl(tmpl, replace)
1172
    try_put(custom, tmpl, mode=0644)
1173

    
1174
    try_run("/etc/init.d/collectd restart")
1175

    
1176

    
1177
@roles("stats")
1178
def setup_stats_collectd():
1179
    setup_collectd()
1180
    tmpl = "/etc/collectd/synnefo-stats.conf"
1181

    
1182
    replace = {
1183
        "STATS": env.env.stats.fqdn,
1184
        }
1185
    custom = customize_settings_from_tmpl(tmpl, replace)
1186
    try_put(custom, tmpl, mode=0644)
1187
    try_run("/etc/init.d/collectd restart")
1188

    
1189

    
1190
@roles("stats")
1191
def setup_stats():
1192
    debug(env.host, "Setting up snf-stats-app...")
1193
    setup_stats_collectd()
1194
    setup_gunicorn()
1195
    setup_apache()
1196
    setup_webproject()
1197
    install_package("snf-stats-app")
1198
    cmd = """
1199
    mkdir /var/cache/snf-stats-app/
1200
    chown www-data:www-data /var/cache/snf-stats-app/
1201
    """
1202
    try_run(cmd)
1203
    tmpl = "/etc/synnefo/stats.conf"
1204

    
1205
    replace = {
1206
        "STATS": env.env.stats.fqdn,
1207
        }
1208
    custom = customize_settings_from_tmpl(tmpl, replace)
1209
    try_put(custom, tmpl, mode=0644)
1210
    try_run("/etc/init.d/gunicorn restart")
1211

    
1212

    
1213
@roles("cyclades")
1214
def setup_cyclades():
1215
    debug(env.host, "Setting up snf-cyclades-app...")
1216
    with settings(hide("everything")):
1217
        try_run("ping -c1 accounts." + env.env.domain)
1218
        try_run("ping -c1 " + env.env.db.ip)
1219
        try_run("ping -c1 " + env.env.mq.ip)
1220
    setup_gunicorn()
1221
    setup_apache()
1222
    setup_webproject()
1223
    install_package("memcached")
1224
    install_package("python-memcache")
1225
    install_package("snf-pithos-backend")
1226
    install_package("kamaki")
1227
    install_package("snf-cyclades-app")
1228
    install_package("python-django-south")
1229
    tmpl = "/etc/synnefo/cyclades.conf"
1230

    
1231
    with settings(host_string=env.env.accounts.ip):
1232
        service_id, service_token = get_service_details("cyclades")
1233

    
1234
    replace = {
1235
        "ACCOUNTS": env.env.accounts.fqdn,
1236
        "CYCLADES": env.env.cyclades.fqdn,
1237
        "mq_node": env.env.mq.ip,
1238
        "db_node": env.env.db.ip,
1239
        "synnefo_user": env.env.synnefo_user,
1240
        "synnefo_db_passwd": env.env.synnefo_db_passwd,
1241
        "synnefo_rabbitmq_passwd": env.env.synnefo_rabbitmq_passwd,
1242
        "pithos_dir": env.env.pithos_dir,
1243
        "common_bridge": env.env.common_bridge,
1244
        "HOST": env.env.cyclades.ip,
1245
        "domain": env.env.domain,
1246
        "CYCLADES_SERVICE_TOKEN": service_token,
1247
        'STATS': env.env.stats.fqdn,
1248
        'CYCLADES_NODE_IP': env.env.cyclades.ip,
1249
        }
1250
    custom = customize_settings_from_tmpl(tmpl, replace)
1251
    try_put(custom, tmpl, mode=0644)
1252
    try_run("/etc/init.d/gunicorn restart")
1253

    
1254
    cmd = """
1255
    sed -i 's/false/true/' /etc/default/snf-dispatcher
1256
    /etc/init.d/snf-dispatcher start
1257
    """
1258
    try_run(cmd)
1259

    
1260
    try_run("snf-manage syncdb")
1261
    try_run("snf-manage migrate --delete-ghost-migrations")
1262

    
1263

    
1264
@roles("cyclades")
1265
def get_backend_id(cluster_name="ganeti1.synnefo.deploy.local"):
1266
    backend_id = try_run("snf-manage backend-list 2>/dev/null " +
1267
                         "| grep %s | awk '{print $1}'" % cluster_name)
1268
    return backend_id
1269

    
1270

    
1271
@roles("cyclades")
1272
def add_backend():
1273
    debug(env.host,
1274
          "adding %s ganeti backend to cyclades..." % env.env.cluster.fqdn)
1275
    with settings(hide("everything")):
1276
        try_run("ping -c1 " + env.env.cluster.fqdn)
1277
    cmd = """
1278
    snf-manage backend-add --clustername={0} --user={1} --pass={2}
1279
    """.format(env.env.cluster.fqdn, env.env.synnefo_user,
1280
               env.env.synnefo_rapi_passwd)
1281
    try_run(cmd)
1282
    backend_id = get_backend_id(env.env.cluster.fqdn)
1283
    try_run("snf-manage backend-modify --drained=False " + backend_id)
1284

    
1285

    
1286
@roles("cyclades")
1287
def pin_user_to_backend(user_email):
1288
    backend_id = get_backend_id(env.env.cluster.fqdn)
1289
    # pin user to backend
1290
    cmd = """
1291
cat <<EOF >> /etc/synnefo/cyclades.conf
1292

1293
BACKEND_PER_USER = {
1294
  '{0}': {1},
1295
}
1296

1297
EOF
1298
/etc/init.d/gunicorn restart
1299
""".format(user_email, backend_id)
1300
    try_run(cmd)
1301

    
1302

    
1303
@roles("cyclades")
1304
def add_pools():
1305
    debug(env.host,
1306
          " * Creating pools of resources (brigdes, mac prefixes) " +
1307
          "in cyclades...")
1308
    try_run("snf-manage pool-create --type=mac-prefix " +
1309
            "--base=aa:00:0 --size=65536")
1310
    try_run("snf-manage pool-create --type=bridge --base=prv --size=20")
1311

    
1312

    
1313
@roles("accounts", "cyclades", "pithos")
1314
def export_services():
1315
    debug(env.host, " * Exporting services...")
1316
    host = env.host
1317
    services = []
1318
    if host == env.env.cyclades.ip:
1319
        services.append("cyclades")
1320
    if host == env.env.pithos.ip:
1321
        services.append("pithos")
1322
    if host == env.env.accounts.ip:
1323
        services.append("astakos")
1324
    for service in services:
1325
        filename = "%s_services.json" % service
1326
        cmd = "snf-manage service-export-%s > %s" % (service, filename)
1327
        run(cmd)
1328
        try_get(filename, filename+".local")
1329

    
1330

    
1331
@roles("accounts")
1332
def import_services():
1333
    debug(env.host, " * Registering services to astakos...")
1334
    for service in ["cyclades", "pithos", "astakos"]:
1335
        filename = "%s_services.json" % service
1336
        try_put(filename + ".local", filename)
1337
        cmd = "snf-manage service-import --json=%s" % filename
1338
        run(cmd)
1339

    
1340
    debug(env.host, " * Setting default quota...")
1341
    cmd = """
1342
    snf-manage resource-modify --default-quota 40G pithos.diskspace
1343
    snf-manage resource-modify --default-quota 2 astakos.pending_app
1344
    snf-manage resource-modify --default-quota 4 cyclades.vm
1345
    snf-manage resource-modify --default-quota 40G cyclades.disk
1346
    snf-manage resource-modify --default-quota 16G cyclades.total_ram
1347
    snf-manage resource-modify --default-quota 8G cyclades.ram
1348
    snf-manage resource-modify --default-quota 32 cyclades.total_cpu
1349
    snf-manage resource-modify --default-quota 16 cyclades.cpu
1350
    snf-manage resource-modify --default-quota 4 cyclades.network.private
1351
    snf-manage resource-modify --default-quota 4 cyclades.floating_ip
1352
    """
1353
    try_run(cmd)
1354

    
1355

    
1356
@roles("accounts")
1357
def set_user_quota():
1358
    debug(env.host, " * Setting user quota...")
1359
    cmd = """
1360
    snf-manage user-modify -f --all --base-quota pithos.diskspace 40G
1361
    snf-manage user-modify -f --all --base-quota astakos.pending_app 2
1362
    snf-manage user-modify -f --all --base-quota cyclades.vm 4
1363
    snf-manage user-modify -f --all --base-quota cyclades.disk 40G
1364
    snf-manage user-modify -f --all --base-quota cyclades.total_ram 16G
1365
    snf-manage user-modify -f --all --base-quota cyclades.ram 8G
1366
    snf-manage user-modify -f --all --base-quota cyclades.total_cpu 32
1367
    snf-manage user-modify -f --all --base-quota cyclades.cpu 16
1368
    snf-manage user-modify -f --all --base-quota cyclades.network.private 4
1369
    snf-manage user-modify -f --all --base-quota cyclades.floating_ip 4
1370
    """
1371
    try_run(cmd)
1372

    
1373

    
1374
@roles("cyclades")
1375
def add_network():
1376
    debug(env.host, " * Adding public network in cyclades...")
1377
    cmd = """
1378
    snf-manage network-create --subnet={0} --gateway={1} --public \
1379
        --dhcp=True --flavor={2} --mode=bridged --link={3} --name=Internet \
1380
        --floating-ip-pool=True
1381
    """.format(env.env.synnefo_public_network_subnet,
1382
               env.env.synnefo_public_network_gateway,
1383
               env.env.synnefo_public_network_type,
1384
               env.env.common_bridge)
1385
    try_run(cmd)
1386
    if env.env.testing_vm:
1387
        cmd = ("snf-manage network-create --subnet6=babe::/64"
1388
               " --gateway6=babe::1 --public --flavor={0} --mode=bridged"
1389
               " --link={1} --name=IPv6PublicNetwork"
1390
               .format(env.env.synnefo_public_network_type,
1391
                       env.env.common_bridge))
1392
        try_run(cmd)
1393

    
1394

    
1395
@roles("cyclades")
1396
def setup_vncauthproxy():
1397
    debug(env.host, " * Setting up vncauthproxy...")
1398
    user = "synnefo"
1399
    salt = "$6$7FUdSvFcWAs3hfVj$"
1400
    passhash = "ZwvnvpQclTrDYWEwBvZDMRJZNgb6ZUKT1vNsh9NzUIxMpzBuGgMqYxCDTYF"\
1401
               "6OZcbunDZb88pjL2EIBnzrGMQW1"
1402
    cmd = """
1403
    mkdir /var/lib/vncauthproxy
1404
    echo '%s:%s%s' > /var/lib/vncauthproxy/users
1405
    """ % (user, salt, passhash)
1406
    try_run(cmd)
1407
    install_package("snf-vncauthproxy")
1408

    
1409

    
1410
@roles("client")
1411
def setup_kamaki():
1412
    debug(env.host, "Setting up kamaki client...")
1413
    with settings(hide("everything")):
1414
        try_run("ping -c1 accounts." + env.env.domain)
1415
        try_run("ping -c1 cyclades." + env.env.domain)
1416
        try_run("ping -c1 pithos." + env.env.domain)
1417

    
1418
    with settings(host_string=env.env.db.ip):
1419
        uid, user_auth_token, user_uuid = \
1420
            get_auth_token_from_db(env.env.user_email)
1421

    
1422
    install_package("python-progress")
1423
    install_package("kamaki")
1424
    cmd = """
1425
    kamaki config set cloud.default.url "https://{0}/astakos/identity/v2.0"
1426
    kamaki config set cloud.default.token {1}
1427
    """.format(env.env.accounts.fqdn, user_auth_token)
1428
    try_run(cmd)
1429
    try_run("kamaki container create images")
1430

    
1431

    
1432
@roles("client")
1433
def upload_image(image="debian_base.diskdump"):
1434
    debug(env.host, " * Uploading initial image to pithos...")
1435
    image = "debian_base.diskdump"
1436
    try_run("wget {0} -O /tmp/{1}".format(env.env.debian_base_url, image))
1437
    try_run("kamaki file upload --container images /tmp/{0} {0}".format(image))
1438

    
1439

    
1440
@roles("client")
1441
def register_image(image="debian_base.diskdump"):
1442
    debug(env.host, " * Register image to plankton...")
1443
    # with settings(host_string=env.env.db.ip):
1444
    #     uid, user_auth_token, user_uuid = \
1445
    #        get_auth_token_from_db(env.env.user_email)
1446

    
1447
    image_location = "/images/{0}".format(image)
1448
    cmd = """
1449
    sleep 5
1450
    kamaki image register --name="Debian Base" --location={0} --public \
1451
            --disk-format=diskdump \
1452
            --property OSFAMILY=linux --property ROOT_PARTITION=1 \
1453
            --property description="Debian Squeeze Base System" \
1454
            --property size=450M --property kernel=2.6.32 \
1455
            --property GUI="No GUI" --property sortorder=1 \
1456
            --property USERS=root --property OS=debian
1457
    """.format(image_location)
1458
    try_run(cmd)
1459

    
1460

    
1461
@roles("client")
1462
def setup_burnin():
1463
    debug(env.host, "Setting up burnin testing tool...")
1464
    install_package("kamaki")
1465
    install_package("snf-tools")
1466

    
1467

    
1468
@roles("pithos")
1469
def add_image_locally():
1470
    debug(env.host,
1471
          " * Getting image locally in order snf-image to use it directly..")
1472
    image = "debian_base.diskdump"
1473
    try_run("wget {0} -O {1}/{2}".format(
1474
            env.env.debian_base_url, env.env.image_dir, image))
1475

    
1476

    
1477
@roles("master")
1478
def gnt_instance_add(name="test"):
1479
    debug(env.host, " * Adding test instance to Ganeti...")
1480
    osp = """img_passwd=gamwtosecurity,\
1481
img_format=diskdump,img_id=debian_base,\
1482
img_properties='{"OSFAMILY":"linux"\,"ROOT_PARTITION":"1"}'"""
1483
    cmd = """
1484
    gnt-instance add  -o snf-image+default --os-parameters {0} \
1485
            -t plain --disk 0:size=1G --no-name-check --no-ip-check \
1486
            --net 0:ip=pool,network=test --no-install \
1487
            --hypervisor-parameters kvm:machine_version=pc-1.0 {1}
1488
    """.format(osp, name)
1489
    try_run(cmd)
1490

    
1491

    
1492
@roles("master")
1493
def gnt_network_add(name="test", subnet="10.0.0.0/26", gw="10.0.0.1",
1494
                    mode="bridged", link="br0"):
1495
    debug(env.host, " * Adding test network to Ganeti...")
1496
    cmd = """
1497
    gnt-network add --network={1} --gateway={2} {0}
1498
    gnt-network connect {0} {3} {4}
1499
    """.format(name, subnet, gw, mode, link)
1500
    try_run(cmd)
1501

    
1502

    
1503
@roles("ips")
1504
def test():
1505
    debug(env.host, "Testing...")
1506
    try_run("hostname && date")