Statistics
| Branch: | Tag: | Revision:

root / snf-deploy / snfdeploy / fabfile.py @ c6e90126

History | View | Annotate | Download (44.7 kB)

1
# Too many lines in module pylint: disable-msg=C0302
2
# Too many arguments (7/5) pylint: disable-msg=R0913
3
"""
4
Fabric file for snf-deploy
5

6
"""
7

    
8
from __future__ import with_statement
9
from fabric.api import hide, env, settings, local, roles
10
from fabric.operations import run, put, get
11
import fabric
12
import re
13
import os
14
import shutil
15
import tempfile
16
import ast
17
from snfdeploy.lib import debug, Conf, Env, disable_color
18
from snfdeploy import massedit
19

    
20

    
21
def setup_env(args):
22
    """Setup environment"""
23
    print("Loading configuration for synnefo...")
24

    
25
    conf = Conf(args)
26
    env.env = Env(conf)
27

    
28
    env.local = args.autoconf
29
    env.key_inject = args.key_inject
30
    env.password = env.env.password
31
    env.user = env.env.user
32
    env.shell = "/bin/bash -c"
33
    env.key_filename = args.ssh_key
34

    
35
    if args.disable_colors:
36
        disable_color()
37

    
38
    if env.env.cms.hostname in \
39
            [env.env.accounts.hostname, env.env.cyclades.hostname,
40
             env.env.pithos.hostname]:
41
        env.cms_pass = True
42
    else:
43
        env.cms_pass = False
44

    
45
    if env.env.accounts.hostname in \
46
            [env.env.cyclades.hostname, env.env.pithos.hostname]:
47
        env.csrf_disable = True
48
    else:
49
        env.csrf_disable = False
50

    
51
    env.roledefs = {
52
        "nodes": env.env.ips,
53
        "ips": env.env.ips,
54
        "accounts": [env.env.accounts.ip],
55
        "cyclades": [env.env.cyclades.ip],
56
        "pithos": [env.env.pithos.ip],
57
        "cms": [env.env.cms.ip],
58
        "mq": [env.env.mq.ip],
59
        "db": [env.env.db.ip],
60
        "mq": [env.env.mq.ip],
61
        "db": [env.env.db.ip],
62
        "ns": [env.env.ns.ip],
63
        "client": [env.env.client.ip],
64
        "router": [env.env.router.ip],
65
        "stats": [env.env.stats.ip],
66
    }
67

    
68
    env.enable_lvm = False
69
    env.enable_drbd = False
70
    if ast.literal_eval(env.env.create_extra_disk) and env.env.extra_disk:
71
        env.enable_lvm = True
72
        env.enable_drbd = True
73

    
74
    env.roledefs.update({
75
        "ganeti": env.env.cluster_ips,
76
        "master": [env.env.master.ip],
77
    })
78

    
79

    
80
def install_package(package):
81
    debug(env.host, " * Installing package %s..." % package)
82
    apt_get = "export DEBIAN_FRONTEND=noninteractive ;" + \
83
              "apt-get install -y --force-yes "
84

    
85
    host_info = env.env.ips_info[env.host]
86
    env.env.update_packages(host_info.os)
87
    if ast.literal_eval(env.env.use_local_packages):
88
        with settings(warn_only=True):
89
            deb = local("ls %s/%s*%s_*.deb"
90
                        % (env.env.packages, package, host_info.os),
91
                        capture=True)
92
            if deb:
93
                debug(env.host,
94
                      " * Package %s found in %s..."
95
                      % (package, env.env.packages))
96
                try_put(deb, "/tmp/")
97
                try_run("dpkg -i /tmp/%s || "
98
                        % os.path.basename(deb) + apt_get + "-f")
99
                try_run("rm /tmp/%s" % os.path.basename(deb))
100
                return
101

    
102
    info = getattr(env.env, package)
103
    if info in \
104
            ["squeeze-backports", "squeeze", "stable",
105
             "testing", "unstable", "wheezy"]:
106
        apt_get += " -t %s %s " % (info, package)
107
    elif info:
108
        apt_get += " %s=%s " % (package, info)
109
    else:
110
        apt_get += package
111

    
112
    try_run(apt_get)
113

    
114
    return
115

    
116

    
117
@roles("ns")
118
def update_ns_for_ganeti():
119
    debug(env.host,
120
          "Updating name server entries for backend %s..."
121
          % env.env.cluster.fqdn)
122
    update_arecord(env.env.cluster)
123
    update_ptrrecord(env.env.cluster)
124
    try_run("/etc/init.d/bind9 restart")
125

    
126

    
127
@roles("ns")
128
def update_ns_for_node(node):
129
    info = env.env.nodes_info.get(node)
130
    update_arecord(info)
131
    update_ptrrecord(info)
132
    try_run("/etc/init.d/bind9 restart")
133

    
134

    
135
@roles("ns")
136
def update_arecord(host):
137
    filename = "/etc/bind/zones/" + env.env.domain
138
    cmd = """
139
    echo '{0}' >> {1}
140
    """.format(host.arecord, filename)
141
    try_run(cmd)
142

    
143

    
144
@roles("ns")
145
def update_cnamerecord(host):
146
    filename = "/etc/bind/zones/" + env.env.domain
147
    cmd = """
148
    echo '{0}' >> {1}
149
    """.format(host.cnamerecord, filename)
150
    try_run(cmd)
151

    
152

    
153
@roles("ns")
154
def update_ptrrecord(host):
155
    filename = "/etc/bind/rev/synnefo.in-addr.arpa.zone"
156
    cmd = """
157
    echo '{0}' >> {1}
158
    """.format(host.ptrrecord, filename)
159
    try_run(cmd)
160

    
161

    
162
@roles("nodes")
163
def apt_get_update():
164
    debug(env.host, "apt-get update....")
165
    try_run("apt-get update")
166

    
167

    
168
@roles("ns")
169
def setup_ns():
170
    debug(env.host, "Setting up name server..")
171
    #WARNING: this should be remove after we are done
172
    # because gevent does pick randomly nameservers and google does
173
    # not know our setup!!!!!
174
    apt_get_update()
175
    install_package("bind9")
176
    tmpl = "/etc/bind/named.conf.local"
177
    replace = {
178
        "domain": env.env.domain,
179
    }
180
    custom = customize_settings_from_tmpl(tmpl, replace)
181
    try_put(custom, tmpl)
182

    
183
    try_run("mkdir -p /etc/bind/zones")
184
    tmpl = "/etc/bind/zones/example.com"
185
    replace = {
186
        "domain": env.env.domain,
187
        "ns_node_ip": env.env.ns.ip,
188
    }
189
    custom = customize_settings_from_tmpl(tmpl, replace)
190
    remote = "/etc/bind/zones/" + env.env.domain
191
    try_put(custom, remote)
192

    
193
    try_run("mkdir -p /etc/bind/rev")
194
    tmpl = "/etc/bind/rev/synnefo.in-addr.arpa.zone"
195
    replace = {
196
        "domain": env.env.domain,
197
    }
198
    custom = customize_settings_from_tmpl(tmpl, replace)
199
    try_put(custom, tmpl)
200

    
201
    tmpl = "/etc/bind/named.conf.options"
202
    replace = {
203
        "NODE_IPS": ";".join(env.env.ips),
204
    }
205
    custom = customize_settings_from_tmpl(tmpl, replace)
206
    try_put(custom, tmpl, mode=0644)
207

    
208
    for role, info in env.env.roles.iteritems():
209
        if role == "ns":
210
            continue
211
        update_cnamerecord(info)
212
    for node, info in env.env.nodes_info.iteritems():
213
        update_arecord(info)
214
        update_ptrrecord(info)
215

    
216
    try_run("/etc/init.d/bind9 restart")
217

    
218

    
219
@roles("nodes")
220
def check_dhcp():
221
    debug(env.host, "Checking IPs for synnefo..")
222
    for n, info in env.env.nodes_info.iteritems():
223
        try_run("ping -c 1 " + info.ip)
224

    
225

    
226
@roles("nodes")
227
def check_dns():
228
    debug(env.host, "Checking fqdns for synnefo..")
229
    for n, info in env.env.nodes_info.iteritems():
230
        try_run("ping -c 1 " + info.fqdn)
231

    
232
    for n, info in env.env.roles.iteritems():
233
        try_run("ping -c 1 " + info.fqdn)
234

    
235

    
236
@roles("nodes")
237
def check_connectivity():
238
    debug(env.host, "Checking internet connectivity..")
239
    try_run("ping -c 1 www.google.com")
240

    
241

    
242
@roles("nodes")
243
def check_ssh():
244
    debug(env.host, "Checking password-less ssh..")
245
    for n, info in env.env.nodes_info.iteritems():
246
        try_run("ssh " + info.fqdn + "  date")
247

    
248

    
249
@roles("ips")
250
def add_keys():
251
    if not env.key_inject:
252
        debug(env.host, "Skipping ssh keys injection..")
253
        return
254
    else:
255
        debug(env.host, "Adding rsa/dsa keys..")
256
    try_run("mkdir -p /root/.ssh")
257
    cmd = """
258
for f in $(ls /root/.ssh/*); do
259
  cp $f $f.bak
260
done
261
    """
262
    try_run(cmd)
263
    files = ["authorized_keys", "id_dsa", "id_dsa.pub",
264
             "id_rsa", "id_rsa.pub"]
265
    for f in files:
266
        tmpl = "/root/.ssh/" + f
267
        replace = {}
268
        custom = customize_settings_from_tmpl(tmpl, replace)
269
        try_put(custom, tmpl, mode=0600)
270

    
271
    cmd = """
272
if [ -e /root/.ssh/authorized_keys.bak ]; then
273
  cat /root/.ssh/authorized_keys.bak >> /root/.ssh/authorized_keys
274
fi
275
    """
276
    debug(env.host, "Updating exising authorized keys..")
277
    try_run(cmd)
278

    
279

    
280
@roles("ips")
281
def setup_resolv_conf():
282
    debug(env.host, "Tweak /etc/resolv.conf...")
283
    try_run("/etc/init.d/network-manager stop", abort=False)
284
    tmpl = "/etc/dhcp/dhclient-enter-hooks.d/nodnsupdate"
285
    replace = {}
286
    custom = customize_settings_from_tmpl(tmpl, replace)
287
    try_put(custom, tmpl, mode=0644)
288
    try_run("cp /etc/resolv.conf /etc/resolv.conf.bak")
289
    tmpl = "/etc/resolv.conf"
290
    replace = {
291
        "domain": env.env.domain,
292
        "ns_node_ip": env.env.ns.ip,
293
    }
294
    custom = customize_settings_from_tmpl(tmpl, replace)
295
    try:
296
        try_put(custom, tmpl)
297
        cmd = """
298
        echo "\
299
# This has been generated automatically by snf-deploy, at
300
# $(date).
301
# The immutable bit (+i attribute) has been used to avoid it being
302
# overwritten by software such as NetworkManager or resolvconf.
303
# Use lsattr/chattr to view or modify its file attributes.
304

305

306
$(cat {0})" > {0}
307
""".format(tmpl)
308
        try_run(cmd)
309
    except:
310
        pass
311
    try_run("chattr +i /etc/resolv.conf")
312

    
313

    
314
@roles("ips")
315
def setup_hosts():
316
    debug(env.host, "Tweaking /etc/hosts and ssh_config files...")
317
    try_run("echo StrictHostKeyChecking no >> /etc/ssh/ssh_config")
318
    cmd = "sed -i 's/^127.*$/127.0.0.1 localhost/g' /etc/hosts "
319
    try_run(cmd)
320
    host_info = env.env.ips_info[env.host]
321
    cmd = "hostname %s" % host_info.hostname
322
    try_run(cmd)
323
    cmd = "echo %s > /etc/hostname" % host_info.hostname
324
    try_run(cmd)
325

    
326

    
327
def try_run(cmd, abort=True):
328
    try:
329
        if env.local:
330
            return local(cmd, capture=True)
331
        else:
332
            return run(cmd)
333
    except BaseException as e:
334
        if abort:
335
            fabric.utils.abort(e)
336
        else:
337
            debug(env.host, "WARNING: command failed. Continuing anyway...")
338

    
339

    
340
def try_put(local_path=None, remote_path=None, abort=True, **kwargs):
341
    try:
342
        put(local_path=local_path, remote_path=remote_path, **kwargs)
343
    except BaseException as e:
344
        if abort:
345
            fabric.utils.abort(e)
346
        else:
347
            debug(env.host, "WARNING: command failed. Continuing anyway...")
348

    
349

    
350
def try_get(remote_path, local_path=None, abort=True, **kwargs):
351
    try:
352
        get(remote_path, local_path=local_path, **kwargs)
353
    except BaseException as e:
354
        if abort:
355
            fabric.utils.abort(e)
356
        else:
357
            debug(env.host, "WARNING: command failed. Continuing anyway...")
358

    
359

    
360
def create_bridges():
361
    debug(env.host, " * Creating bridges...")
362
    install_package("bridge-utils")
363
    cmd = """
364
    brctl addbr {0} ; ip link set {0} up
365
    """.format(env.env.common_bridge)
366
    try_run(cmd)
367

    
368

    
369
def connect_bridges():
370
    debug(env.host, " * Connecting bridges...")
371
    #cmd = """
372
    #brctl addif {0} {1}
373
    #""".format(env.env.common_bridge, env.env.public_iface)
374
    #try_run(cmd)
375

    
376

    
377
@roles("ganeti")
378
def setup_net_infra():
379
    debug(env.host, "Setup networking infrastracture..")
380
    create_bridges()
381
    connect_bridges()
382

    
383

    
384
@roles("ganeti")
385
def setup_lvm():
386
    debug(env.host, "create volume group %s for ganeti.." % env.env.vg)
387
    if env.enable_lvm:
388
        install_package("lvm2")
389
        cmd = """
390
        pvcreate {0}
391
        vgcreate {1} {0}
392
        """.format(env.env.extra_disk, env.env.vg)
393
        try_run(cmd)
394

    
395

    
396
def customize_settings_from_tmpl(tmpl, replace):
397
    debug(env.host, " * Customizing template %s..." % tmpl)
398
    local = env.env.templates + tmpl
399
    _, custom = tempfile.mkstemp()
400
    shutil.copyfile(local, custom)
401
    for k, v in replace.iteritems():
402
        regex = "re.sub('%{0}%', '{1}', line)".format(k.upper(), v)
403
        massedit.edit_files([custom], [regex], dry_run=False)
404

    
405
    return custom
406

    
407

    
408
@roles("nodes")
409
def setup_apt():
410
    debug(env.host, "Setting up apt sources...")
411
    install_package("curl")
412
    cmd = """
413
    echo 'APT::Install-Suggests "false";' >> /etc/apt/apt.conf
414
    curl -k https://dev.grnet.gr/files/apt-grnetdev.pub | apt-key add -
415
    """
416
    try_run(cmd)
417
    host_info = env.env.ips_info[env.host]
418
    if host_info.os == "squeeze":
419
        tmpl = "/etc/apt/sources.list.d/synnefo.squeeze.list"
420
    else:
421
        tmpl = "/etc/apt/sources.list.d/synnefo.wheezy.list"
422
    replace = {}
423
    custom = customize_settings_from_tmpl(tmpl, replace)
424
    try_put(custom, tmpl)
425
    apt_get_update()
426

    
427

    
428
@roles("cyclades", "cms", "pithos", "accounts")
429
def restart_services():
430
    debug(env.host, " * Restarting apache2 and gunicorn...")
431
    try_run("/etc/init.d/gunicorn restart")
432
    try_run("/etc/init.d/apache2 restart")
433

    
434

    
435
def setup_gunicorn():
436
    debug(env.host, " * Setting up gunicorn...")
437
    install_package("gunicorn")
438
    try_run("chown root.www-data /var/log/gunicorn")
439
    tmpl = "/etc/gunicorn.d/synnefo"
440
    replace = {}
441
    custom = customize_settings_from_tmpl(tmpl, replace)
442
    try_put(custom, tmpl, mode=0644)
443
    try_run("/etc/init.d/gunicorn restart")
444

    
445

    
446
def setup_apache():
447
    debug(env.host, " * Setting up apache2...")
448
    host_info = env.env.ips_info[env.host]
449
    install_package("apache2")
450
    tmpl = "/etc/apache2/sites-available/synnefo"
451
    replace = {
452
        "HOST": host_info.fqdn,
453
    }
454
    custom = customize_settings_from_tmpl(tmpl, replace)
455
    try_put(custom, tmpl)
456
    tmpl = "/etc/apache2/sites-available/synnefo-ssl"
457
    custom = customize_settings_from_tmpl(tmpl, replace)
458
    try_put(custom, tmpl)
459
    cmd = """
460
    a2enmod ssl
461
    a2enmod rewrite
462
    a2dissite default
463
    a2ensite synnefo
464
    a2ensite synnefo-ssl
465
    a2enmod headers
466
    a2enmod proxy_http
467
    a2dismod autoindex
468
    """
469
    try_run(cmd)
470
    try_run("/etc/init.d/apache2 restart")
471

    
472

    
473
@roles("mq")
474
def setup_mq():
475
    debug(env.host, "Setting up RabbitMQ...")
476
    install_package("rabbitmq-server")
477
    cmd = """
478
    rabbitmqctl add_user {0} {1}
479
    rabbitmqctl set_permissions {0} ".*" ".*" ".*"
480
    rabbitmqctl delete_user guest
481
    rabbitmqctl set_user_tags {0} administrator
482
    """.format(env.env.synnefo_user, env.env.synnefo_rabbitmq_passwd)
483
    try_run(cmd)
484
    try_run("/etc/init.d/rabbitmq-server restart")
485

    
486

    
487
@roles("db")
488
def allow_access_in_db(ip, user="all", method="md5"):
489
    cmd = """
490
    pg_hba=$(ls /etc/postgresql/*/main/pg_hba.conf)
491
    echo host all {0} {1}/32 {2} >> $pg_hba
492
    """.format(user, ip, method)
493
    try_run(cmd)
494
    cmd = """
495
    pg_hba=$(ls /etc/postgresql/*/main/pg_hba.conf)
496
    sed -i 's/\(host.*127.0.0.1.*\)md5/\\1trust/' $pg_hba
497
    """
498
    try_run(cmd)
499
    try_run("/etc/init.d/postgresql restart")
500

    
501

    
502
@roles("db")
503
def setup_db():
504
    debug(env.host, "Setting up DataBase server...")
505
    install_package("postgresql")
506

    
507
    tmpl = "/tmp/db-init.psql"
508
    replace = {
509
        "synnefo_user": env.env.synnefo_user,
510
        "synnefo_db_passwd": env.env.synnefo_db_passwd,
511
        }
512
    custom = customize_settings_from_tmpl(tmpl, replace)
513
    try_put(custom, tmpl)
514
    cmd = 'su - postgres -c "psql -w -f %s" ' % tmpl
515
    try_run(cmd)
516
    cmd = """
517
    conf=$(ls /etc/postgresql/*/main/postgresql.conf)
518
    echo "listen_addresses = '*'" >> $conf
519
    """
520
    try_run(cmd)
521

    
522
    if env.env.testing_vm:
523
        cmd = """
524
        conf=$(ls /etc/postgresql/*/main/postgresql.conf)
525
        echo "fsync=off\nsynchronous_commit=off\nfull_page_writes=off" >> $conf
526
        """
527
        try_run(cmd)
528

    
529
    allow_access_in_db(env.host, "all", "trust")
530
    try_run("/etc/init.d/postgresql restart")
531

    
532

    
533
@roles("db")
534
def destroy_db():
535
    try_run("""su - postgres -c ' psql -w -c "drop database snf_apps" '""")
536
    try_run("""su - postgres -c ' psql -w -c "drop database snf_pithos" '""")
537

    
538

    
539
def setup_webproject():
540
    debug(env.host, " * Setting up snf-webproject...")
541
    with settings(hide("everything")):
542
        try_run("ping -c1 " + env.env.db.ip)
543
    setup_common()
544
    install_package("snf-webproject")
545
    install_package("python-psycopg2")
546
    install_package("python-gevent")
547
    install_package("python-django")
548
    tmpl = "/etc/synnefo/webproject.conf"
549
    replace = {
550
        "synnefo_user": env.env.synnefo_user,
551
        "synnefo_db_passwd": env.env.synnefo_db_passwd,
552
        "db_node": env.env.db.ip,
553
        "domain": env.env.domain,
554
    }
555
    custom = customize_settings_from_tmpl(tmpl, replace)
556
    try_put(custom, tmpl, mode=0644)
557
    with settings(host_string=env.env.db.ip):
558
        host_info = env.env.ips_info[env.host]
559
        allow_access_in_db(host_info.ip, "all", "trust")
560
    try_run("/etc/init.d/gunicorn restart")
561

    
562

    
563
def setup_common():
564
    debug(env.host, " * Setting up snf-common...")
565
    host_info = env.env.ips_info[env.host]
566
    install_package("python-objpool")
567
    install_package("snf-common")
568
    install_package("python-astakosclient")
569
    install_package("snf-django-lib")
570
    install_package("snf-branding")
571
    tmpl = "/etc/synnefo/common.conf"
572
    replace = {
573
        #FIXME:
574
        "EMAIL_SUBJECT_PREFIX": env.host,
575
        "domain": env.env.domain,
576
        "HOST": host_info.fqdn,
577
        "MAIL_DIR": env.env.mail_dir,
578
    }
579
    custom = customize_settings_from_tmpl(tmpl, replace)
580
    try_put(custom, tmpl, mode=0644)
581
    try_run("mkdir -p {0}; chmod 777 {0}".format(env.env.mail_dir))
582
    try_run("/etc/init.d/gunicorn restart")
583

    
584

    
585
@roles("accounts")
586
def astakos_loaddata():
587
    debug(env.host, " * Loading initial data to astakos...")
588
    cmd = """
589
    snf-manage loaddata groups
590
    """
591
    try_run(cmd)
592

    
593

    
594
@roles("accounts")
595
def astakos_register_components():
596
    debug(env.host, " * Register services in astakos...")
597

    
598
    cyclades_base_url = "https://%s/cyclades" % env.env.cyclades.fqdn
599
    pithos_base_url = "https://%s/pithos" % env.env.pithos.fqdn
600
    astakos_base_url = "https://%s/astakos" % env.env.accounts.fqdn
601

    
602
    cmd = """
603
    snf-manage component-add "home" --ui-url https://{0}
604
    snf-manage component-add "cyclades" --base-url {1} --ui-url {1}/ui
605
    snf-manage component-add "pithos" --base-url {2} --ui-url {2}/ui
606
    snf-manage component-add "astakos" --base-url {3} --ui-url {3}/ui
607
    """.format(env.env.cms.fqdn, cyclades_base_url,
608
               pithos_base_url, astakos_base_url)
609
    try_run(cmd)
610

    
611

    
612
@roles("accounts")
613
def astakos_register_pithos_view():
614
    debug(env.host, " * Register pithos view as oauth2 client...")
615

    
616
    pithos_base_url = "https://%s/pithos" % env.env.pithos.fqdn
617

    
618
    cmd = """
619
    snf-manage oauth2-client-add pithos-view --secret=12345 --is-trusted \
620
    --url {0}
621
    """.format('%s/ui/view' % pithos_base_url)
622
    try_run(cmd)
623

    
624

    
625
@roles("accounts")
626
def add_user():
627
    debug(env.host, " * adding user %s to astakos..." % env.env.user_email)
628
    email = env.env.user_email
629
    name = env.env.user_name
630
    lastname = env.env.user_lastname
631
    passwd = env.env.user_passwd
632
    cmd = """
633
    snf-manage user-add {0} {1} {2}
634
    """.format(email, name, lastname)
635
    try_run(cmd)
636
    with settings(host_string=env.env.db.ip):
637
        uid, user_auth_token, user_uuid = get_auth_token_from_db(email)
638
    cmd = """
639
    snf-manage user-modify --password {0} {1}
640
    """.format(passwd, uid)
641
    try_run(cmd)
642

    
643

    
644
@roles("accounts")
645
def activate_user(user_email=None):
646
    if not user_email:
647
        user_email = env.env.user_email
648
    debug(env.host, " * Activate user %s..." % user_email)
649
    with settings(host_string=env.env.db.ip):
650
        uid, user_auth_token, user_uuid = get_auth_token_from_db(user_email)
651

    
652
    cmd = """
653
    snf-manage user-modify --verify {0}
654
    snf-manage user-modify --accept {0}
655
    """.format(uid)
656
    try_run(cmd)
657

    
658

    
659
@roles("accounts")
660
def setup_astakos():
661
    debug(env.host, "Setting up snf-astakos-app...")
662
    setup_gunicorn()
663
    setup_apache()
664
    setup_webproject()
665
    install_package("python-django-south")
666
    install_package("snf-astakos-app")
667
    install_package("kamaki")
668

    
669
    tmpl = "/etc/synnefo/astakos.conf"
670
    replace = {
671
        "ACCOUNTS": env.env.accounts.fqdn,
672
        "domain": env.env.domain,
673
        "CYCLADES": env.env.cyclades.fqdn,
674
        "PITHOS": env.env.pithos.fqdn,
675
    }
676
    custom = customize_settings_from_tmpl(tmpl, replace)
677
    try_put(custom, tmpl, mode=0644)
678
    if env.csrf_disable:
679
        cmd = """
680
cat <<EOF >> /etc/synnefo/astakos.conf
681
try:
682
  MIDDLEWARE_CLASSES.remove('django.middleware.csrf.CsrfViewMiddleware')
683
except:
684
  pass
685
EOF
686
"""
687
        try_run(cmd)
688

    
689
    try_run("/etc/init.d/gunicorn restart")
690

    
691
    cmd = """
692
    snf-manage syncdb --noinput
693
    snf-manage migrate im --delete-ghost-migrations
694
    snf-manage migrate quotaholder_app
695
    snf-manage migrate oa2
696
    """
697
    try_run(cmd)
698

    
699

    
700
@roles("accounts")
701
def get_service_details(service="pithos"):
702
    debug(env.host,
703
          " * Getting registered details for %s service..." % service)
704
    result = try_run("snf-manage component-list -o id,name,token")
705
    r = re.compile(r".*%s.*" % service, re.M)
706
    service_id, _, service_token = r.search(result).group().split()
707
    # print("%s: %s %s" % (service, service_id, service_token))
708
    return (service_id, service_token)
709

    
710

    
711
@roles("db")
712
def get_auth_token_from_db(user_email=None):
713
    if not user_email:
714
        user_email = env.env.user_email
715
    debug(env.host,
716
          " * Getting authentication token and uuid for user %s..."
717
          % user_email)
718
    cmd = """
719
echo "select id, auth_token, uuid, email from auth_user, im_astakosuser \
720
where auth_user.id = im_astakosuser.user_ptr_id and auth_user.email = '{0}';" \
721
> /tmp/psqlcmd
722
su - postgres -c  "psql -w -d snf_apps -f /tmp/psqlcmd"
723
""".format(user_email)
724

    
725
    result = try_run(cmd)
726
    r = re.compile(r"(\d+)[ |]*(\S+)[ |]*(\S+)[ |]*" + user_email, re.M)
727
    match = r.search(result)
728
    uid, user_auth_token, user_uuid = match.groups()
729
    # print("%s: %s %s %s" % ( user_email, uid, user_auth_token, user_uuid))
730

    
731
    return (uid, user_auth_token, user_uuid)
732

    
733

    
734
@roles("cms")
735
def cms_loaddata():
736
    debug(env.host, " * Loading cms initial data...")
737
    if env.cms_pass:
738
        debug(env.host, "Aborting. Prerequisites not met.")
739
        return
740
    tmpl = "/tmp/sites.json"
741
    replace = {}
742
    custom = customize_settings_from_tmpl(tmpl, replace)
743
    try_put(custom, tmpl)
744

    
745
    tmpl = "/tmp/page.json"
746
    replace = {}
747
    custom = customize_settings_from_tmpl(tmpl, replace)
748
    try_put(custom, tmpl)
749

    
750
    cmd = """
751
    snf-manage loaddata /tmp/sites.json
752
    snf-manage loaddata /tmp/page.json
753
    snf-manage createsuperuser --username=admin --email=admin@{0} --noinput
754
    """.format(env.env.domain)
755
    try_run(cmd)
756

    
757

    
758
@roles("cms")
759
def setup_cms():
760
    debug(env.host, "Setting up cms...")
761
    if env.cms_pass:
762
        debug(env.host, "Aborting. Prerequisites not met.")
763
        return
764
    with settings(hide("everything")):
765
        try_run("ping -c1 accounts." + env.env.domain)
766
    setup_gunicorn()
767
    setup_apache()
768
    setup_webproject()
769
    install_package("snf-cloudcms")
770

    
771
    tmpl = "/etc/synnefo/cms.conf"
772
    replace = {
773
        "ACCOUNTS": env.env.accounts.fqdn,
774
        }
775
    custom = customize_settings_from_tmpl(tmpl, replace)
776
    try_put(custom, tmpl, mode=0644)
777
    try_run("/etc/init.d/gunicorn restart")
778

    
779
    cmd = """
780
    snf-manage syncdb
781
    snf-manage migrate --delete-ghost-migrations
782
    """.format(env.env.domain)
783
    try_run(cmd)
784

    
785

    
786
def setup_nfs_dirs():
787
    debug(env.host, " * Creating NFS mount point for pithos and ganeti...")
788
    cmd = """
789
    mkdir -p {0}
790
    cd {0}
791
    mkdir -p data
792
    chown www-data:www-data data
793
    chmod g+ws data
794
    mkdir -p {1}
795
    """.format(env.env.pithos_dir, env.env.image_dir)
796
    try_run(cmd)
797

    
798

    
799
@roles("nodes")
800
def setup_nfs_clients():
801
    if env.host == env.env.pithos.ip:
802
        return
803

    
804
    host_info = env.env.ips_info[env.host]
805
    debug(env.host, " * Mounting pithos NFS mount point...")
806
    with settings(hide("everything")):
807
        try_run("ping -c1 " + env.env.pithos.hostname)
808
    with settings(host_string=env.env.pithos.ip):
809
        update_nfs_exports(host_info.ip)
810

    
811
    install_package("nfs-common")
812
    for d in [env.env.pithos_dir, env.env.image_dir]:
813
        try_run("mkdir -p " + d)
814
        cmd = """
815
echo "{0}:{1} {1}  nfs defaults,rw,noatime,rsize=131072,\
816
wsize=131072,timeo=14,intr,noacl" >> /etc/fstab
817
""".format(env.env.pithos.ip, d)
818
        try_run(cmd)
819
        try_run("mount " + d)
820

    
821

    
822
@roles("pithos")
823
def update_nfs_exports(ip):
824
    tmpl = "/tmp/exports"
825
    replace = {
826
        "pithos_dir": env.env.pithos_dir,
827
        "image_dir": env.env.image_dir,
828
        "ip": ip,
829
    }
830
    custom = customize_settings_from_tmpl(tmpl, replace)
831
    try_put(custom, tmpl)
832
    try_run("cat %s >> /etc/exports" % tmpl)
833
    try_run("/etc/init.d/nfs-kernel-server restart")
834

    
835

    
836
@roles("pithos")
837
def setup_nfs_server():
838
    debug(env.host, " * Setting up NFS server for pithos...")
839
    setup_nfs_dirs()
840
    install_package("nfs-kernel-server")
841

    
842

    
843
@roles("pithos")
844
def setup_pithos():
845
    debug(env.host, "Setting up snf-pithos-app...")
846
    with settings(hide("everything")):
847
        try_run("ping -c1 accounts." + env.env.domain)
848
        try_run("ping -c1 " + env.env.db.ip)
849
    setup_gunicorn()
850
    setup_apache()
851
    setup_webproject()
852

    
853
    with settings(host_string=env.env.accounts.ip):
854
        service_id, service_token = get_service_details("pithos")
855

    
856
    install_package("kamaki")
857
    install_package("snf-pithos-backend")
858
    install_package("snf-pithos-app")
859
    tmpl = "/etc/synnefo/pithos.conf"
860
    replace = {
861
        "ACCOUNTS": env.env.accounts.fqdn,
862
        "PITHOS": env.env.pithos.fqdn,
863
        "db_node": env.env.db.ip,
864
        "synnefo_user": env.env.synnefo_user,
865
        "synnefo_db_passwd": env.env.synnefo_db_passwd,
866
        "pithos_dir": env.env.pithos_dir,
867
        "PITHOS_SERVICE_TOKEN": service_token,
868
        }
869
    custom = customize_settings_from_tmpl(tmpl, replace)
870
    try_put(custom, tmpl, mode=0644)
871
    try_run("/etc/init.d/gunicorn restart")
872

    
873
    install_package("snf-pithos-webclient")
874
    tmpl = "/etc/synnefo/webclient.conf"
875
    replace = {
876
        "ACCOUNTS": env.env.accounts.fqdn,
877
        "PITHOS_UI_CLOUDBAR_ACTIVE_SERVICE": service_id,
878
        }
879
    custom = customize_settings_from_tmpl(tmpl, replace)
880
    try_put(custom, tmpl, mode=0644)
881

    
882
    try_run("/etc/init.d/gunicorn restart")
883
    #TOFIX: the previous command lets pithos-backend create blocks and maps
884
    #       with root owner
885
    try_run("chown -R www-data:www-data %s/data " % env.env.pithos_dir)
886
    #try_run("pithos-migrate stamp 4c8ccdc58192")
887
    #try_run("pithos-migrate upgrade head")
888

    
889

    
890
@roles("ganeti")
891
def setup_ganeti():
892
    debug(env.host, "Setting up snf-ganeti...")
893
    node_info = env.env.ips_info[env.host]
894
    with settings(hide("everything")):
895
        #if env.enable_lvm:
896
        #    try_run("vgs " + env.env.vg)
897
        try_run("getent hosts " + env.env.cluster.fqdn)
898
        try_run("getent hosts %s | grep -v ^127" % env.host)
899
        try_run("hostname -f | grep " + node_info.fqdn)
900
        #try_run("ip link show " + env.env.common_bridge)
901
        #try_run("ip link show " + env.env.common_bridge)
902
        #try_run("apt-get update")
903
    install_package("qemu-kvm")
904
    install_package("python-bitarray")
905
    install_package("ganeti-haskell")
906
    install_package("ganeti-htools")
907
    install_package("snf-ganeti")
908
    try_run("mkdir -p /srv/ganeti/file-storage/")
909
    cmd = """
910
cat <<EOF > /etc/ganeti/file-storage-paths
911
/srv/ganeti/file-storage
912
/srv/ganeti/shared-file-storage
913
EOF
914
"""
915
    try_run(cmd)
916

    
917

    
918
@roles("master")
919
def add_rapi_user():
920
    debug(env.host, " * Adding RAPI user to Ganeti backend...")
921
    cmd = """
922
    echo -n "{0}:Ganeti Remote API:{1}" | openssl md5 | sed 's/^.* //'
923
    """.format(env.env.synnefo_user, env.env.synnefo_rapi_passwd)
924
    result = try_run(cmd)
925
    if result.startswith("(stdin)= "):
926
        result = result.split("(stdin)= ")[1]
927
    cmd = """
928
    echo "{0} {1}{2} write" >> /var/lib/ganeti/rapi/users
929
    """.format(env.env.synnefo_user, '{ha1}', result)
930
    try_run(cmd)
931
    try_run("/etc/init.d/ganeti restart")
932

    
933

    
934
@roles("master")
935
def add_nodes():
936
    nodes = env.env.cluster_nodes.split(",")
937
    nodes.remove(env.env.master_node)
938
    debug(env.host, " * Adding nodes to Ganeti backend...")
939
    for n in nodes:
940
        add_node(n)
941

    
942

    
943
@roles("master")
944
def add_node(node):
945
    node_info = env.env.nodes_info[node]
946
    debug(env.host, " * Adding node %s to Ganeti backend..." % node_info.fqdn)
947
    cmd = "gnt-node add --no-ssh-key-check --master-capable=yes " + \
948
          "--vm-capable=yes " + node_info.fqdn
949
    try_run(cmd)
950

    
951

    
952
@roles("ganeti")
953
def enable_drbd():
954
    if env.enable_drbd:
955
        debug(env.host, " * Enabling DRBD...")
956
        install_package("drbd8-utils")
957
        try_run("modprobe drbd minor_count=255 usermode_helper=/bin/true")
958
        try_run("echo drbd minor_count=255 usermode_helper=/bin/true " +
959
                ">> /etc/modules")
960

    
961

    
962
@roles("master")
963
def setup_drbd_dparams():
964
    if env.enable_drbd:
965
        debug(env.host,
966
              " * Twicking drbd related disk parameters in Ganeti...")
967
        cmd = """
968
        gnt-cluster modify --disk-parameters=drbd:metavg={0}
969
        gnt-group modify --disk-parameters=drbd:metavg={0} default
970
        """.format(env.env.vg)
971
        try_run(cmd)
972

    
973

    
974
@roles("master")
975
def enable_lvm():
976
    if env.enable_lvm:
977
        debug(env.host, " * Enabling LVM...")
978
        cmd = """
979
        gnt-cluster modify --vg-name={0}
980
        """.format(env.env.vg)
981
        try_run(cmd)
982
    else:
983
        debug(env.host, " * Disabling LVM...")
984
        try_run("gnt-cluster modify --no-lvm-storage")
985

    
986

    
987
@roles("master")
988
def destroy_cluster():
989
    debug(env.host, " * Destroying Ganeti cluster...")
990
    #TODO: remove instances first
991
    allnodes = env.env.cluster_hostnames[:]
992
    allnodes.remove(env.host)
993
    for n in allnodes:
994
        host_info = env.env.ips_info[env.host]
995
        debug(env.host, " * Removing node %s..." % n)
996
        cmd = "gnt-node remove  " + host_info.fqdn
997
        try_run(cmd)
998
    try_run("gnt-cluster destroy --yes-do-it")
999

    
1000

    
1001
@roles("master")
1002
def init_cluster():
1003
    debug(env.host, " * Initializing Ganeti backend...")
1004
    # extra = ""
1005
    # if env.enable_lvm:
1006
    #     extra += " --vg-name={0} ".format(env.env.vg)
1007
    # else:
1008
    #     extra += " --no-lvm-storage "
1009
    # if not env.enable_drbd:
1010
    #     extra += " --no-drbd-storage "
1011
    extra = " --no-lvm-storage --no-drbd-storage "
1012
    cmd = """
1013
    gnt-cluster init --enabled-hypervisors=kvm \
1014
        {0} \
1015
        --nic-parameters link={1},mode=bridged \
1016
        --master-netdev {2} \
1017
        --default-iallocator hail \
1018
        --specs-nic-count min=0,max=8 \
1019
        --hypervisor-parameters kvm:kernel_path=,vnc_bind_address=0.0.0.0 \
1020
        --no-ssh-init --no-etc-hosts \
1021
        {3}
1022
    """.format(extra, env.env.common_bridge,
1023
               env.env.cluster_netdev, env.env.cluster.fqdn)
1024
    try_run(cmd)
1025
    cmd = """gnt-cluster modify --enabled-disk-templates file,plain,ext"""
1026
    try_run(cmd)
1027

    
1028

    
1029
@roles("ganeti")
1030
def debootstrap():
1031
    install_package("ganeti-instance-debootstrap")
1032

    
1033

    
1034
@roles("ganeti")
1035
def setup_image_host():
1036
    debug(env.host, "Setting up snf-image...")
1037
    install_package("snf-pithos-backend")
1038
    install_package("snf-image")
1039
    try_run("mkdir -p %s" % env.env.image_dir)
1040
    tmpl = "/etc/default/snf-image"
1041
    replace = {
1042
        "synnefo_user": env.env.synnefo_user,
1043
        "synnefo_db_passwd": env.env.synnefo_db_passwd,
1044
        "pithos_dir": env.env.pithos_dir,
1045
        "db_node": env.env.db.ip,
1046
        "image_dir": env.env.image_dir,
1047
    }
1048
    custom = customize_settings_from_tmpl(tmpl, replace)
1049
    try_put(custom, tmpl)
1050

    
1051

    
1052
@roles("ganeti")
1053
def setup_image_helper():
1054
    debug(env.host, " * Updating helper image...")
1055
    cmd = """
1056
    snf-image-update-helper -y
1057
    """
1058
    try_run(cmd)
1059

    
1060

    
1061
@roles("ganeti")
1062
def setup_gtools():
1063
    debug(env.host, " * Setting up snf-cyclades-gtools...")
1064
    with settings(hide("everything")):
1065
        try_run("ping -c1 " + env.env.mq.ip)
1066
    setup_common()
1067
    install_package("snf-cyclades-gtools")
1068
    tmpl = "/etc/synnefo/gtools.conf"
1069
    replace = {
1070
        "synnefo_user": env.env.synnefo_user,
1071
        "synnefo_rabbitmq_passwd": env.env.synnefo_rabbitmq_passwd,
1072
        "mq_node": env.env.mq.ip,
1073
    }
1074
    custom = customize_settings_from_tmpl(tmpl, replace)
1075
    try_put(custom, tmpl)
1076

    
1077
    cmd = """
1078
    sed -i 's/false/true/' /etc/default/snf-ganeti-eventd
1079
    /etc/init.d/snf-ganeti-eventd start
1080
    """
1081
    try_run(cmd)
1082

    
1083

    
1084
@roles("ganeti")
1085
def setup_iptables():
1086
    debug(env.host, " * Setting up iptables to mangle DHCP requests...")
1087
    cmd = """
1088
    iptables -t mangle -A PREROUTING -i br+ -p udp -m udp --dport 67 \
1089
            -j NFQUEUE --queue-num 42
1090
    iptables -t mangle -A PREROUTING -i tap+ -p udp -m udp --dport 67 \
1091
            -j NFQUEUE --queue-num 42
1092
    iptables -t mangle -A PREROUTING -i prv+ -p udp -m udp --dport 67 \
1093
            -j NFQUEUE --queue-num 42
1094

1095
    ip6tables -t mangle -A PREROUTING -i br+ -p ipv6-icmp -m icmp6 \
1096
            --icmpv6-type 133 -j NFQUEUE --queue-num 43
1097
    ip6tables -t mangle -A PREROUTING -i br+ -p ipv6-icmp -m icmp6 \
1098
            --icmpv6-type 135 -j NFQUEUE --queue-num 44
1099
    """
1100
    try_run(cmd)
1101

    
1102

    
1103
@roles("ganeti")
1104
def setup_network():
1105
    debug(env.host,
1106
          "Setting up networking for Ganeti instances (nfdhcpd, etc.)...")
1107
    install_package("python-nfqueue")
1108
    install_package("nfdhcpd")
1109
    tmpl = "/etc/nfdhcpd/nfdhcpd.conf"
1110
    replace = {
1111
        "ns_node_ip": env.env.ns.ip
1112
    }
1113
    custom = customize_settings_from_tmpl(tmpl, replace)
1114
    try_put(custom, tmpl)
1115
    try_run("/etc/init.d/nfdhcpd restart")
1116

    
1117
    install_package("snf-network")
1118
    cmd = """
1119
sed -i 's/MAC_MASK.*/MAC_MASK = ff:ff:f0:00:00:00/' /etc/default/snf-network
1120
    """
1121
    try_run(cmd)
1122

    
1123

    
1124
@roles("router")
1125
def setup_router():
1126
    debug(env.host, " * Setting up internal router for NAT...")
1127
    cmd = """
1128
    echo 1 > /proc/sys/net/ipv4/ip_forward
1129
    iptables -t nat -A POSTROUTING -s {0} -o {3} -j MASQUERADE
1130
    ip addr add {1} dev {2}
1131
    ip route add {0} dev {2} src {1}
1132
    """.format(env.env.synnefo_public_network_subnet,
1133
               env.env.synnefo_public_network_gateway,
1134
               env.env.common_bridge, env.env.public_iface)
1135
    try_run(cmd)
1136

    
1137

    
1138
@roles("cyclades")
1139
def cyclades_loaddata():
1140
    debug(env.host, " * Loading initial data for cyclades...")
1141
    try_run("snf-manage flavor-create %s %s %s %s" % (env.env.flavor_cpu,
1142
                                                      env.env.flavor_ram,
1143
                                                      env.env.flavor_disk,
1144
                                                      env.env.flavor_storage))
1145
    #run("snf-manage loaddata flavors")
1146

    
1147

    
1148
@roles("ganeti", "stats")
1149
def setup_collectd():
1150
    install_package("collectd")
1151
    tmpl = "/etc/collectd/collectd.conf"
1152
    replace = {}
1153
    custom = customize_settings_from_tmpl(tmpl, replace)
1154
    try_put(custom, tmpl, mode=0644)
1155

    
1156

    
1157
@roles("ganeti")
1158
def setup_ganeti_collectd():
1159
    setup_collectd()
1160

    
1161
    tmpl = "/etc/collectd/passwd"
1162
    replace = {}
1163
    custom = customize_settings_from_tmpl(tmpl, replace)
1164
    try_put(custom, tmpl, mode=0644)
1165

    
1166
    tmpl = "/etc/collectd/synnefo-ganeti.conf"
1167
    replace = {
1168
        "STATS": env.env.stats.fqdn,
1169
        }
1170
    custom = customize_settings_from_tmpl(tmpl, replace)
1171
    try_put(custom, tmpl, mode=0644)
1172

    
1173
    try_run("/etc/init.d/collectd restart")
1174

    
1175

    
1176
@roles("stats")
1177
def setup_stats_collectd():
1178
    setup_collectd()
1179
    tmpl = "/etc/collectd/synnefo-stats.conf"
1180

    
1181
    replace = {
1182
        "STATS": env.env.stats.fqdn,
1183
        }
1184
    custom = customize_settings_from_tmpl(tmpl, replace)
1185
    try_put(custom, tmpl, mode=0644)
1186
    try_run("/etc/init.d/collectd restart")
1187

    
1188

    
1189
@roles("stats")
1190
def setup_stats():
1191
    debug(env.host, "Setting up snf-stats-app...")
1192
    setup_stats_collectd()
1193
    setup_gunicorn()
1194
    setup_apache()
1195
    setup_webproject()
1196
    install_package("snf-stats-app")
1197
    cmd = """
1198
    mkdir /var/cache/snf-stats-app/
1199
    chown www-data:www-data /var/cache/snf-stats-app/
1200
    """
1201
    try_run(cmd)
1202
    tmpl = "/etc/synnefo/stats.conf"
1203

    
1204
    replace = {
1205
        "STATS": env.env.stats.fqdn,
1206
        }
1207
    custom = customize_settings_from_tmpl(tmpl, replace)
1208
    try_put(custom, tmpl, mode=0644)
1209
    try_run("/etc/init.d/gunicorn restart")
1210

    
1211

    
1212
@roles("cyclades")
1213
def setup_cyclades():
1214
    debug(env.host, "Setting up snf-cyclades-app...")
1215
    with settings(hide("everything")):
1216
        try_run("ping -c1 accounts." + env.env.domain)
1217
        try_run("ping -c1 " + env.env.db.ip)
1218
        try_run("ping -c1 " + env.env.mq.ip)
1219
    setup_gunicorn()
1220
    setup_apache()
1221
    setup_webproject()
1222
    install_package("memcached")
1223
    install_package("python-memcache")
1224
    install_package("snf-pithos-backend")
1225
    install_package("kamaki")
1226
    install_package("snf-cyclades-app")
1227
    install_package("python-django-south")
1228
    tmpl = "/etc/synnefo/cyclades.conf"
1229

    
1230
    with settings(host_string=env.env.accounts.ip):
1231
        service_id, service_token = get_service_details("cyclades")
1232

    
1233
    replace = {
1234
        "ACCOUNTS": env.env.accounts.fqdn,
1235
        "CYCLADES": env.env.cyclades.fqdn,
1236
        "mq_node": env.env.mq.ip,
1237
        "db_node": env.env.db.ip,
1238
        "synnefo_user": env.env.synnefo_user,
1239
        "synnefo_db_passwd": env.env.synnefo_db_passwd,
1240
        "synnefo_rabbitmq_passwd": env.env.synnefo_rabbitmq_passwd,
1241
        "pithos_dir": env.env.pithos_dir,
1242
        "common_bridge": env.env.common_bridge,
1243
        "HOST": env.env.cyclades.ip,
1244
        "domain": env.env.domain,
1245
        "CYCLADES_SERVICE_TOKEN": service_token,
1246
        'STATS': env.env.stats.fqdn,
1247
        }
1248
    custom = customize_settings_from_tmpl(tmpl, replace)
1249
    try_put(custom, tmpl, mode=0644)
1250
    try_run("/etc/init.d/gunicorn restart")
1251

    
1252
    cmd = """
1253
    sed -i 's/false/true/' /etc/default/snf-dispatcher
1254
    /etc/init.d/snf-dispatcher start
1255
    """
1256
    try_run(cmd)
1257

    
1258
    try_run("snf-manage syncdb")
1259
    try_run("snf-manage migrate --delete-ghost-migrations")
1260

    
1261

    
1262
@roles("cyclades")
1263
def get_backend_id(cluster_name="ganeti1.synnefo.deploy.local"):
1264
    backend_id = try_run("snf-manage backend-list 2>/dev/null " +
1265
                         "| grep %s | awk '{print $1}'" % cluster_name)
1266
    return backend_id
1267

    
1268

    
1269
@roles("cyclades")
1270
def add_backend():
1271
    debug(env.host,
1272
          "adding %s ganeti backend to cyclades..." % env.env.cluster.fqdn)
1273
    with settings(hide("everything")):
1274
        try_run("ping -c1 " + env.env.cluster.fqdn)
1275
    cmd = """
1276
    snf-manage backend-add --clustername={0} --user={1} --pass={2}
1277
    """.format(env.env.cluster.fqdn, env.env.synnefo_user,
1278
               env.env.synnefo_rapi_passwd)
1279
    try_run(cmd)
1280
    backend_id = get_backend_id(env.env.cluster.fqdn)
1281
    try_run("snf-manage backend-modify --drained=False " + backend_id)
1282

    
1283

    
1284
@roles("cyclades")
1285
def pin_user_to_backend(user_email):
1286
    backend_id = get_backend_id(env.env.cluster.fqdn)
1287
    # pin user to backend
1288
    cmd = """
1289
cat <<EOF >> /etc/synnefo/cyclades.conf
1290

1291
BACKEND_PER_USER = {
1292
  '{0}': {1},
1293
}
1294

1295
EOF
1296
/etc/init.d/gunicorn restart
1297
""".format(user_email, backend_id)
1298
    try_run(cmd)
1299

    
1300

    
1301
@roles("cyclades")
1302
def add_pools():
1303
    debug(env.host,
1304
          " * Creating pools of resources (brigdes, mac prefixes) " +
1305
          "in cyclades...")
1306
    try_run("snf-manage pool-create --type=mac-prefix " +
1307
            "--base=aa:00:0 --size=65536")
1308
    try_run("snf-manage pool-create --type=bridge --base=prv --size=20")
1309

    
1310

    
1311
@roles("accounts", "cyclades", "pithos")
1312
def export_services():
1313
    debug(env.host, " * Exporting services...")
1314
    host = env.host
1315
    services = []
1316
    if host == env.env.cyclades.ip:
1317
        services.append("cyclades")
1318
    if host == env.env.pithos.ip:
1319
        services.append("pithos")
1320
    if host == env.env.accounts.ip:
1321
        services.append("astakos")
1322
    for service in services:
1323
        filename = "%s_services.json" % service
1324
        cmd = "snf-manage service-export-%s > %s" % (service, filename)
1325
        run(cmd)
1326
        try_get(filename, filename+".local")
1327

    
1328

    
1329
@roles("accounts")
1330
def import_services():
1331
    debug(env.host, " * Registering services to astakos...")
1332
    for service in ["cyclades", "pithos", "astakos"]:
1333
        filename = "%s_services.json" % service
1334
        try_put(filename + ".local", filename)
1335
        cmd = "snf-manage service-import --json=%s" % filename
1336
        run(cmd)
1337

    
1338
    debug(env.host, " * Setting default quota...")
1339
    cmd = """
1340
    snf-manage resource-modify --default-quota 40G pithos.diskspace
1341
    snf-manage resource-modify --default-quota 2 astakos.pending_app
1342
    snf-manage resource-modify --default-quota 4 cyclades.vm
1343
    snf-manage resource-modify --default-quota 40G cyclades.disk
1344
    snf-manage resource-modify --default-quota 16G cyclades.total_ram
1345
    snf-manage resource-modify --default-quota 8G cyclades.ram
1346
    snf-manage resource-modify --default-quota 32 cyclades.total_cpu
1347
    snf-manage resource-modify --default-quota 16 cyclades.cpu
1348
    snf-manage resource-modify --default-quota 4 cyclades.network.private
1349
    snf-manage resource-modify --default-quota 4 cyclades.floating_ip
1350
    """
1351
    try_run(cmd)
1352

    
1353

    
1354
@roles("accounts")
1355
def set_user_quota():
1356
    debug(env.host, " * Setting user quota...")
1357
    cmd = """
1358
    snf-manage user-modify -f --all --base-quota pithos.diskspace 40G
1359
    snf-manage user-modify -f --all --base-quota astakos.pending_app 2
1360
    snf-manage user-modify -f --all --base-quota cyclades.vm 4
1361
    snf-manage user-modify -f --all --base-quota cyclades.disk 40G
1362
    snf-manage user-modify -f --all --base-quota cyclades.total_ram 16G
1363
    snf-manage user-modify -f --all --base-quota cyclades.ram 8G
1364
    snf-manage user-modify -f --all --base-quota cyclades.total_cpu 32
1365
    snf-manage user-modify -f --all --base-quota cyclades.cpu 16
1366
    snf-manage user-modify -f --all --base-quota cyclades.network.private 4
1367
    snf-manage user-modify -f --all --base-quota cyclades.floating_ip 4
1368
    """
1369
    try_run(cmd)
1370

    
1371

    
1372
@roles("cyclades")
1373
def add_network():
1374
    debug(env.host, " * Adding public network in cyclades...")
1375
    cmd = """
1376
    snf-manage network-create --subnet={0} --gateway={1} --public \
1377
        --dhcp=True --flavor={2} --mode=bridged --link={3} --name=Internet \
1378
        --floating-ip-pool=True
1379
    """.format(env.env.synnefo_public_network_subnet,
1380
               env.env.synnefo_public_network_gateway,
1381
               env.env.synnefo_public_network_type,
1382
               env.env.common_bridge)
1383
    try_run(cmd)
1384
    if env.env.testing_vm:
1385
        cmd = ("snf-manage network-create --subnet6=babe::/64"
1386
               " --gateway6=babe::1 --public --flavor={0} --mode=bridged"
1387
               " --link={1} --name=IPv6PublicNetwork"
1388
               .format(env.env.synnefo_public_network_type,
1389
                       env.env.common_bridge))
1390
        try_run(cmd)
1391

    
1392

    
1393
@roles("cyclades")
1394
def setup_vncauthproxy():
1395
    debug(env.host, " * Setting up vncauthproxy...")
1396
    user = "synnefo"
1397
    salt = "$6$7FUdSvFcWAs3hfVj$"
1398
    passhash = "ZwvnvpQclTrDYWEwBvZDMRJZNgb6ZUKT1vNsh9NzUIxMpzBuGgMqYxCDTYF"\
1399
               "6OZcbunDZb88pjL2EIBnzrGMQW1"
1400
    cmd = """
1401
    mkdir /var/lib/vncauthproxy
1402
    echo '%s:%s%s' > /var/lib/vncauthproxy/users
1403
    """ % (user, salt, passhash)
1404
    try_run(cmd)
1405
    install_package("snf-vncauthproxy")
1406

    
1407

    
1408
@roles("client")
1409
def setup_kamaki():
1410
    debug(env.host, "Setting up kamaki client...")
1411
    with settings(hide("everything")):
1412
        try_run("ping -c1 accounts." + env.env.domain)
1413
        try_run("ping -c1 cyclades." + env.env.domain)
1414
        try_run("ping -c1 pithos." + env.env.domain)
1415

    
1416
    with settings(host_string=env.env.db.ip):
1417
        uid, user_auth_token, user_uuid = \
1418
            get_auth_token_from_db(env.env.user_email)
1419

    
1420
    install_package("python-progress")
1421
    install_package("kamaki")
1422
    cmd = """
1423
    kamaki config set cloud.default.url "https://{0}/astakos/identity/v2.0"
1424
    kamaki config set cloud.default.token {1}
1425
    """.format(env.env.accounts.fqdn, user_auth_token)
1426
    try_run(cmd)
1427
    try_run("kamaki container create images")
1428

    
1429

    
1430
@roles("client")
1431
def upload_image(image="debian_base.diskdump"):
1432
    debug(env.host, " * Uploading initial image to pithos...")
1433
    image = "debian_base.diskdump"
1434
    try_run("wget {0} -O /tmp/{1}".format(env.env.debian_base_url, image))
1435
    try_run("kamaki file upload --container images /tmp/{0} {0}".format(image))
1436

    
1437

    
1438
@roles("client")
1439
def register_image(image="debian_base.diskdump"):
1440
    debug(env.host, " * Register image to plankton...")
1441
    # with settings(host_string=env.env.db.ip):
1442
    #     uid, user_auth_token, user_uuid = \
1443
    #        get_auth_token_from_db(env.env.user_email)
1444

    
1445
    image_location = "images:{0}".format(image)
1446
    cmd = """
1447
    sleep 5
1448
    kamaki image register "Debian Base" {0} --public --disk-format=diskdump \
1449
            --property OSFAMILY=linux --property ROOT_PARTITION=1 \
1450
            --property description="Debian Squeeze Base System" \
1451
            --property size=450M --property kernel=2.6.32 \
1452
            --property GUI="No GUI" --property sortorder=1 \
1453
            --property USERS=root --property OS=debian
1454
    """.format(image_location)
1455
    try_run(cmd)
1456

    
1457

    
1458
@roles("client")
1459
def setup_burnin():
1460
    debug(env.host, "Setting up burnin testing tool...")
1461
    install_package("kamaki")
1462
    install_package("snf-tools")
1463

    
1464

    
1465
@roles("pithos")
1466
def add_image_locally():
1467
    debug(env.host,
1468
          " * Getting image locally in order snf-image to use it directly..")
1469
    image = "debian_base.diskdump"
1470
    try_run("wget {0} -O {1}/{2}".format(
1471
            env.env.debian_base_url, env.env.image_dir, image))
1472

    
1473

    
1474
@roles("master")
1475
def gnt_instance_add(name="test"):
1476
    debug(env.host, " * Adding test instance to Ganeti...")
1477
    osp = """img_passwd=gamwtosecurity,\
1478
img_format=diskdump,img_id=debian_base,\
1479
img_properties='{"OSFAMILY":"linux"\,"ROOT_PARTITION":"1"}'"""
1480
    cmd = """
1481
    gnt-instance add  -o snf-image+default --os-parameters {0} \
1482
            -t plain --disk 0:size=1G --no-name-check --no-ip-check \
1483
            --net 0:ip=pool,network=test --no-install \
1484
            --hypervisor-parameters kvm:machine_version=pc-1.0 {1}
1485
    """.format(osp, name)
1486
    try_run(cmd)
1487

    
1488

    
1489
@roles("master")
1490
def gnt_network_add(name="test", subnet="10.0.0.0/26", gw="10.0.0.1",
1491
                    mode="bridged", link="br0"):
1492
    debug(env.host, " * Adding test network to Ganeti...")
1493
    cmd = """
1494
    gnt-network add --network={1} --gateway={2} {0}
1495
    gnt-network connect {0} {3} {4}
1496
    """.format(name, subnet, gw, mode, link)
1497
    try_run(cmd)
1498

    
1499

    
1500
@roles("ips")
1501
def test():
1502
    debug(env.host, "Testing...")
1503
    try_run("hostname && date")