Statistics
| Branch: | Tag: | Revision:

root / snf-deploy / snfdeploy / fabfile.py @ 952a28c0

History | View | Annotate | Download (44.6 kB)

1
# Too many lines in module pylint: disable-msg=C0302
2
# Too many arguments (7/5) pylint: disable-msg=R0913
3
"""
4
Fabric file for snf-deploy
5

6
"""
7

    
8
from __future__ import with_statement
9
from fabric.api import hide, env, settings, local, roles
10
from fabric.operations import run, put, get
11
import fabric
12
import re
13
import os
14
import shutil
15
import tempfile
16
import ast
17
from snfdeploy.lib import debug, Conf, Env, disable_color
18
from snfdeploy import massedit
19

    
20

    
21
def setup_env(args):
22
    """Setup environment"""
23
    print("Loading configuration for synnefo...")
24

    
25
    conf = Conf(args)
26
    env.env = Env(conf)
27

    
28
    env.local = args.autoconf
29
    env.key_inject = args.key_inject
30
    env.password = env.env.password
31
    env.user = env.env.user
32
    env.shell = "/bin/bash -c"
33
    env.key_filename = args.ssh_key
34

    
35
    if args.disable_colors:
36
        disable_color()
37

    
38
    if env.env.cms.hostname in \
39
            [env.env.accounts.hostname, env.env.cyclades.hostname,
40
             env.env.pithos.hostname]:
41
        env.cms_pass = True
42
    else:
43
        env.cms_pass = False
44

    
45
    if env.env.accounts.hostname in \
46
            [env.env.cyclades.hostname, env.env.pithos.hostname]:
47
        env.csrf_disable = True
48
    else:
49
        env.csrf_disable = False
50

    
51
    env.roledefs = {
52
        "nodes": env.env.ips,
53
        "ips": env.env.ips,
54
        "accounts": [env.env.accounts.ip],
55
        "cyclades": [env.env.cyclades.ip],
56
        "pithos": [env.env.pithos.ip],
57
        "cms": [env.env.cms.ip],
58
        "mq": [env.env.mq.ip],
59
        "db": [env.env.db.ip],
60
        "mq": [env.env.mq.ip],
61
        "db": [env.env.db.ip],
62
        "ns": [env.env.ns.ip],
63
        "client": [env.env.client.ip],
64
        "router": [env.env.router.ip],
65
        "stats": [env.env.stats.ip],
66
    }
67

    
68
    env.enable_lvm = False
69
    env.enable_drbd = False
70
    if ast.literal_eval(env.env.create_extra_disk) and env.env.extra_disk:
71
        env.enable_lvm = True
72
        env.enable_drbd = True
73

    
74
    env.roledefs.update({
75
        "ganeti": env.env.cluster_ips,
76
        "master": [env.env.master.ip],
77
    })
78

    
79

    
80
def install_package(package):
81
    debug(env.host, " * Installing package %s..." % package)
82
    apt_get = "export DEBIAN_FRONTEND=noninteractive ;" + \
83
              "apt-get install -y --force-yes "
84

    
85
    host_info = env.env.ips_info[env.host]
86
    env.env.update_packages(host_info.os)
87
    if ast.literal_eval(env.env.use_local_packages):
88
        with settings(warn_only=True):
89
            deb = local("ls %s/%s*%s_*.deb"
90
                        % (env.env.packages, package, host_info.os),
91
                        capture=True)
92
            if deb:
93
                debug(env.host,
94
                      " * Package %s found in %s..."
95
                      % (package, env.env.packages))
96
                try_put(deb, "/tmp/")
97
                try_run("dpkg -i /tmp/%s || "
98
                        % os.path.basename(deb) + apt_get + "-f")
99
                try_run("rm /tmp/%s" % os.path.basename(deb))
100
                return
101

    
102
    info = getattr(env.env, package)
103
    if info in \
104
            ["squeeze-backports", "squeeze", "stable",
105
             "testing", "unstable", "wheezy"]:
106
        apt_get += " -t %s %s " % (info, package)
107
    elif info:
108
        apt_get += " %s=%s " % (package, info)
109
    else:
110
        apt_get += package
111

    
112
    try_run(apt_get)
113

    
114
    return
115

    
116

    
117
@roles("ns")
118
def update_ns_for_ganeti():
119
    debug(env.host,
120
          "Updating name server entries for backend %s..."
121
          % env.env.cluster.fqdn)
122
    update_arecord(env.env.cluster)
123
    update_ptrrecord(env.env.cluster)
124
    try_run("/etc/init.d/bind9 restart")
125

    
126

    
127
@roles("ns")
128
def update_ns_for_node(node):
129
    info = env.env.nodes_info.get(node)
130
    update_arecord(info)
131
    update_ptrrecord(info)
132
    try_run("/etc/init.d/bind9 restart")
133

    
134

    
135
@roles("ns")
136
def update_arecord(host):
137
    filename = "/etc/bind/zones/" + env.env.domain
138
    cmd = """
139
    echo '{0}' >> {1}
140
    """.format(host.arecord, filename)
141
    try_run(cmd)
142

    
143

    
144
@roles("ns")
145
def update_cnamerecord(host):
146
    filename = "/etc/bind/zones/" + env.env.domain
147
    cmd = """
148
    echo '{0}' >> {1}
149
    """.format(host.cnamerecord, filename)
150
    try_run(cmd)
151

    
152

    
153
@roles("ns")
154
def update_ptrrecord(host):
155
    filename = "/etc/bind/rev/synnefo.in-addr.arpa.zone"
156
    cmd = """
157
    echo '{0}' >> {1}
158
    """.format(host.ptrrecord, filename)
159
    try_run(cmd)
160

    
161

    
162
@roles("nodes")
163
def apt_get_update():
164
    debug(env.host, "apt-get update....")
165
    try_run("apt-get update")
166

    
167

    
168
@roles("ns")
169
def setup_ns():
170
    debug(env.host, "Setting up name server..")
171
    #WARNING: this should be remove after we are done
172
    # because gevent does pick randomly nameservers and google does
173
    # not know our setup!!!!!
174
    apt_get_update()
175
    install_package("bind9")
176
    tmpl = "/etc/bind/named.conf.local"
177
    replace = {
178
        "domain": env.env.domain,
179
    }
180
    custom = customize_settings_from_tmpl(tmpl, replace)
181
    try_put(custom, tmpl)
182

    
183
    try_run("mkdir -p /etc/bind/zones")
184
    tmpl = "/etc/bind/zones/example.com"
185
    replace = {
186
        "domain": env.env.domain,
187
        "ns_node_ip": env.env.ns.ip,
188
    }
189
    custom = customize_settings_from_tmpl(tmpl, replace)
190
    remote = "/etc/bind/zones/" + env.env.domain
191
    try_put(custom, remote)
192

    
193
    try_run("mkdir -p /etc/bind/rev")
194
    tmpl = "/etc/bind/rev/synnefo.in-addr.arpa.zone"
195
    replace = {
196
        "domain": env.env.domain,
197
    }
198
    custom = customize_settings_from_tmpl(tmpl, replace)
199
    try_put(custom, tmpl)
200

    
201
    tmpl = "/etc/bind/named.conf.options"
202
    replace = {
203
        "NODE_IPS": ";".join(env.env.ips),
204
    }
205
    custom = customize_settings_from_tmpl(tmpl, replace)
206
    try_put(custom, tmpl, mode=0644)
207

    
208
    for role, info in env.env.roles.iteritems():
209
        if role == "ns":
210
            continue
211
        update_cnamerecord(info)
212
    for node, info in env.env.nodes_info.iteritems():
213
        update_arecord(info)
214
        update_ptrrecord(info)
215

    
216
    try_run("/etc/init.d/bind9 restart")
217

    
218

    
219
@roles("nodes")
220
def check_dhcp():
221
    debug(env.host, "Checking IPs for synnefo..")
222
    for n, info in env.env.nodes_info.iteritems():
223
        try_run("ping -c 1 " + info.ip)
224

    
225

    
226
@roles("nodes")
227
def check_dns():
228
    debug(env.host, "Checking fqdns for synnefo..")
229
    for n, info in env.env.nodes_info.iteritems():
230
        try_run("ping -c 1 " + info.fqdn)
231

    
232
    for n, info in env.env.roles.iteritems():
233
        try_run("ping -c 1 " + info.fqdn)
234

    
235

    
236
@roles("nodes")
237
def check_connectivity():
238
    debug(env.host, "Checking internet connectivity..")
239
    try_run("ping -c 1 www.google.com")
240

    
241

    
242
@roles("nodes")
243
def check_ssh():
244
    debug(env.host, "Checking password-less ssh..")
245
    for n, info in env.env.nodes_info.iteritems():
246
        try_run("ssh " + info.fqdn + "  date")
247

    
248

    
249
@roles("ips")
250
def add_keys():
251
    if not env.key_inject:
252
        debug(env.host, "Skipping ssh keys injection..")
253
        return
254
    else:
255
        debug(env.host, "Adding rsa/dsa keys..")
256
    try_run("mkdir -p /root/.ssh")
257
    cmd = """
258
for f in $(ls /root/.ssh/*); do
259
  cp $f $f.bak
260
done
261
    """
262
    try_run(cmd)
263
    files = ["authorized_keys", "id_dsa", "id_dsa.pub",
264
             "id_rsa", "id_rsa.pub"]
265
    for f in files:
266
        tmpl = "/root/.ssh/" + f
267
        replace = {}
268
        custom = customize_settings_from_tmpl(tmpl, replace)
269
        try_put(custom, tmpl, mode=0600)
270

    
271
    cmd = """
272
if [ -e /root/.ssh/authorized_keys.bak ]; then
273
  cat /root/.ssh/authorized_keys.bak >> /root/.ssh/authorized_keys
274
fi
275
    """
276
    debug(env.host, "Updating exising authorized keys..")
277
    try_run(cmd)
278

    
279

    
280
@roles("ips")
281
def setup_resolv_conf():
282
    debug(env.host, "Tweak /etc/resolv.conf...")
283
    try_run("/etc/init.d/network-manager stop", abort=False)
284
    tmpl = "/etc/dhcp/dhclient-enter-hooks.d/nodnsupdate"
285
    replace = {}
286
    custom = customize_settings_from_tmpl(tmpl, replace)
287
    try_put(custom, tmpl, mode=0644)
288
    try_run("cp /etc/resolv.conf /etc/resolv.conf.bak")
289
    tmpl = "/etc/resolv.conf"
290
    replace = {
291
        "domain": env.env.domain,
292
        "ns_node_ip": env.env.ns.ip,
293
    }
294
    custom = customize_settings_from_tmpl(tmpl, replace)
295
    try:
296
        try_put(custom, tmpl)
297
        cmd = """
298
        echo "\
299
# This has been generated automatically by snf-deploy, at
300
# $(date).
301
# The immutable bit (+i attribute) has been used to avoid it being
302
# overwritten by software such as NetworkManager or resolvconf.
303
# Use lsattr/chattr to view or modify its file attributes.
304

305

306
$(cat {0})" > {0}
307
""".format(tmpl)
308
        try_run(cmd)
309
    except:
310
        pass
311
    try_run("chattr +i /etc/resolv.conf")
312

    
313

    
314
@roles("ips")
315
def setup_hosts():
316
    debug(env.host, "Tweaking /etc/hosts and ssh_config files...")
317
    try_run("echo StrictHostKeyChecking no >> /etc/ssh/ssh_config")
318
    cmd = "sed -i 's/^127.*$/127.0.0.1 localhost/g' /etc/hosts "
319
    try_run(cmd)
320
    host_info = env.env.ips_info[env.host]
321
    cmd = "hostname %s" % host_info.hostname
322
    try_run(cmd)
323
    cmd = "echo %s > /etc/hostname" % host_info.hostname
324
    try_run(cmd)
325

    
326

    
327
def try_run(cmd, abort=True):
328
    try:
329
        if env.local:
330
            return local(cmd, capture=True)
331
        else:
332
            return run(cmd)
333
    except BaseException as e:
334
        if abort:
335
            fabric.utils.abort(e)
336
        else:
337
            debug(env.host, "WARNING: command failed. Continuing anyway...")
338

    
339

    
340
def try_put(local_path=None, remote_path=None, abort=True, **kwargs):
341
    try:
342
        put(local_path=local_path, remote_path=remote_path, **kwargs)
343
    except BaseException as e:
344
        if abort:
345
            fabric.utils.abort(e)
346
        else:
347
            debug(env.host, "WARNING: command failed. Continuing anyway...")
348

    
349

    
350
def try_get(remote_path, local_path=None, abort=True, **kwargs):
351
    try:
352
        get(remote_path, local_path=local_path, **kwargs)
353
    except BaseException as e:
354
        if abort:
355
            fabric.utils.abort(e)
356
        else:
357
            debug(env.host, "WARNING: command failed. Continuing anyway...")
358

    
359

    
360
def create_bridges():
361
    debug(env.host, " * Creating bridges...")
362
    install_package("bridge-utils")
363
    cmd = """
364
    brctl addbr {0} ; ip link set {0} up
365
    """.format(env.env.common_bridge)
366
    try_run(cmd)
367

    
368

    
369
def connect_bridges():
370
    debug(env.host, " * Connecting bridges...")
371
    #cmd = """
372
    #brctl addif {0} {1}
373
    #""".format(env.env.common_bridge, env.env.public_iface)
374
    #try_run(cmd)
375

    
376

    
377
@roles("ganeti")
378
def setup_net_infra():
379
    debug(env.host, "Setup networking infrastracture..")
380
    create_bridges()
381
    connect_bridges()
382

    
383

    
384
@roles("ganeti")
385
def setup_lvm():
386
    debug(env.host, "create volume group %s for ganeti.." % env.env.vg)
387
    if env.enable_lvm:
388
        install_package("lvm2")
389
        cmd = """
390
        pvcreate {0}
391
        vgcreate {1} {0}
392
        """.format(env.env.extra_disk, env.env.vg)
393
        try_run(cmd)
394

    
395

    
396
def customize_settings_from_tmpl(tmpl, replace):
397
    debug(env.host, " * Customizing template %s..." % tmpl)
398
    local = env.env.templates + tmpl
399
    _, custom = tempfile.mkstemp()
400
    shutil.copyfile(local, custom)
401
    for k, v in replace.iteritems():
402
        regex = "re.sub('%{0}%', '{1}', line)".format(k.upper(), v)
403
        massedit.edit_files([custom], [regex], dry_run=False)
404

    
405
    return custom
406

    
407

    
408
@roles("nodes")
409
def setup_apt():
410
    debug(env.host, "Setting up apt sources...")
411
    install_package("curl")
412
    cmd = """
413
    echo 'APT::Install-Suggests "false";' >> /etc/apt/apt.conf
414
    curl -k https://dev.grnet.gr/files/apt-grnetdev.pub | apt-key add -
415
    """
416
    try_run(cmd)
417
    host_info = env.env.ips_info[env.host]
418
    if host_info.os == "squeeze":
419
        tmpl = "/etc/apt/sources.list.d/synnefo.squeeze.list"
420
    else:
421
        tmpl = "/etc/apt/sources.list.d/synnefo.wheezy.list"
422
    replace = {}
423
    custom = customize_settings_from_tmpl(tmpl, replace)
424
    try_put(custom, tmpl)
425
    apt_get_update()
426

    
427

    
428
@roles("cyclades", "cms", "pithos", "accounts")
429
def restart_services():
430
    debug(env.host, " * Restarting apache2 and gunicorn...")
431
    try_run("/etc/init.d/gunicorn restart")
432
    try_run("/etc/init.d/apache2 restart")
433

    
434

    
435
def setup_gunicorn():
436
    debug(env.host, " * Setting up gunicorn...")
437
    install_package("gunicorn")
438
    try_run("chown root.www-data /var/log/gunicorn")
439
    tmpl = "/etc/gunicorn.d/synnefo"
440
    replace = {}
441
    custom = customize_settings_from_tmpl(tmpl, replace)
442
    try_put(custom, tmpl, mode=0644)
443
    try_run("/etc/init.d/gunicorn restart")
444

    
445

    
446
def setup_apache():
447
    debug(env.host, " * Setting up apache2...")
448
    host_info = env.env.ips_info[env.host]
449
    install_package("apache2")
450
    tmpl = "/etc/apache2/sites-available/synnefo"
451
    replace = {
452
        "HOST": host_info.fqdn,
453
    }
454
    custom = customize_settings_from_tmpl(tmpl, replace)
455
    try_put(custom, tmpl)
456
    tmpl = "/etc/apache2/sites-available/synnefo-ssl"
457
    custom = customize_settings_from_tmpl(tmpl, replace)
458
    try_put(custom, tmpl)
459
    cmd = """
460
    a2enmod ssl
461
    a2enmod rewrite
462
    a2dissite default
463
    a2ensite synnefo
464
    a2ensite synnefo-ssl
465
    a2enmod headers
466
    a2enmod proxy_http
467
    a2dismod autoindex
468
    """
469
    try_run(cmd)
470
    try_run("/etc/init.d/apache2 restart")
471

    
472

    
473
@roles("mq")
474
def setup_mq():
475
    debug(env.host, "Setting up RabbitMQ...")
476
    install_package("rabbitmq-server")
477
    cmd = """
478
    rabbitmqctl add_user {0} {1}
479
    rabbitmqctl set_permissions {0} ".*" ".*" ".*"
480
    rabbitmqctl delete_user guest
481
    rabbitmqctl set_user_tags {0} administrator
482
    """.format(env.env.synnefo_user, env.env.synnefo_rabbitmq_passwd)
483
    try_run(cmd)
484
    try_run("/etc/init.d/rabbitmq-server restart")
485

    
486

    
487
@roles("db")
488
def allow_access_in_db(ip, user="all", method="md5"):
489
    cmd = """
490
    pg_hba=$(ls /etc/postgresql/*/main/pg_hba.conf)
491
    echo host all {0} {1}/32 {2} >> $pg_hba
492
    """.format(user, ip, method)
493
    try_run(cmd)
494
    cmd = """
495
    pg_hba=$(ls /etc/postgresql/*/main/pg_hba.conf)
496
    sed -i 's/\(host.*127.0.0.1.*\)md5/\\1trust/' $pg_hba
497
    """
498
    try_run(cmd)
499
    try_run("/etc/init.d/postgresql restart")
500

    
501

    
502
@roles("db")
503
def setup_db():
504
    debug(env.host, "Setting up DataBase server...")
505
    install_package("postgresql")
506

    
507
    tmpl = "/tmp/db-init.psql"
508
    replace = {
509
        "synnefo_user": env.env.synnefo_user,
510
        "synnefo_db_passwd": env.env.synnefo_db_passwd,
511
        }
512
    custom = customize_settings_from_tmpl(tmpl, replace)
513
    try_put(custom, tmpl)
514
    cmd = 'su - postgres -c "psql -w -f %s" ' % tmpl
515
    try_run(cmd)
516
    cmd = """
517
    conf=$(ls /etc/postgresql/*/main/postgresql.conf)
518
    echo "listen_addresses = '*'" >> $conf
519
    """
520
    try_run(cmd)
521

    
522
    if env.env.testing_vm:
523
        cmd = """
524
        conf=$(ls /etc/postgresql/*/main/postgresql.conf)
525
        echo "fsync=off\nsynchronous_commit=off\nfull_page_writes=off" >> $conf
526
        """
527
        try_run(cmd)
528

    
529
    allow_access_in_db(env.host, "all", "trust")
530
    try_run("/etc/init.d/postgresql restart")
531

    
532

    
533
@roles("db")
534
def destroy_db():
535
    try_run("""su - postgres -c ' psql -w -c "drop database snf_apps" '""")
536
    try_run("""su - postgres -c ' psql -w -c "drop database snf_pithos" '""")
537

    
538

    
539
def setup_webproject():
540
    debug(env.host, " * Setting up snf-webproject...")
541
    with settings(hide("everything")):
542
        try_run("ping -c1 " + env.env.db.ip)
543
    setup_common()
544
    install_package("snf-webproject")
545
    install_package("python-psycopg2")
546
    install_package("python-gevent")
547
    install_package("python-django")
548
    tmpl = "/etc/synnefo/webproject.conf"
549
    replace = {
550
        "synnefo_user": env.env.synnefo_user,
551
        "synnefo_db_passwd": env.env.synnefo_db_passwd,
552
        "db_node": env.env.db.ip,
553
        "domain": env.env.domain,
554
    }
555
    custom = customize_settings_from_tmpl(tmpl, replace)
556
    try_put(custom, tmpl, mode=0644)
557
    with settings(host_string=env.env.db.ip):
558
        host_info = env.env.ips_info[env.host]
559
        allow_access_in_db(host_info.ip, "all", "trust")
560
    try_run("/etc/init.d/gunicorn restart")
561

    
562

    
563
def setup_common():
564
    debug(env.host, " * Setting up snf-common...")
565
    host_info = env.env.ips_info[env.host]
566
    install_package("python-objpool")
567
    install_package("snf-common")
568
    install_package("python-astakosclient")
569
    install_package("snf-django-lib")
570
    install_package("snf-branding")
571
    tmpl = "/etc/synnefo/common.conf"
572
    replace = {
573
        #FIXME:
574
        "EMAIL_SUBJECT_PREFIX": env.host,
575
        "domain": env.env.domain,
576
        "HOST": host_info.fqdn,
577
        "MAIL_DIR": env.env.mail_dir,
578
    }
579
    custom = customize_settings_from_tmpl(tmpl, replace)
580
    try_put(custom, tmpl, mode=0644)
581
    try_run("mkdir -p {0}; chmod 777 {0}".format(env.env.mail_dir))
582
    try_run("/etc/init.d/gunicorn restart")
583

    
584

    
585
@roles("accounts")
586
def astakos_loaddata():
587
    debug(env.host, " * Loading initial data to astakos...")
588
    cmd = """
589
    snf-manage loaddata groups
590
    """
591
    try_run(cmd)
592

    
593

    
594
@roles("accounts")
595
def astakos_register_components():
596
    debug(env.host, " * Register services in astakos...")
597

    
598
    cyclades_base_url = "https://%s/cyclades" % env.env.cyclades.fqdn
599
    pithos_base_url = "https://%s/pithos" % env.env.pithos.fqdn
600
    astakos_base_url = "https://%s/astakos" % env.env.accounts.fqdn
601

    
602
    cmd = """
603
    snf-manage component-add "home" --ui-url https://{0}
604
    snf-manage component-add "cyclades" --base-url {1} --ui-url {1}/ui
605
    snf-manage component-add "pithos" --base-url {2} --ui-url {2}/ui
606
    snf-manage component-add "astakos" --base-url {3} --ui-url {3}/ui
607
    """.format(env.env.cms.fqdn, cyclades_base_url,
608
               pithos_base_url, astakos_base_url)
609
    try_run(cmd)
610

    
611

    
612
@roles("accounts")
613
def astakos_register_pithos_view():
614
    debug(env.host, " * Register pithos view as oauth2 client...")
615

    
616
    pithos_base_url = "https://%s/pithos" % env.env.pithos.fqdn
617

    
618
    cmd = """
619
    snf-manage oauth2-client-add pithos-view --secret=12345 --is-trusted \
620
    --url {0}
621
    """.format('%s/ui/view' % pithos_base_url)
622
    try_run(cmd)
623

    
624

    
625
@roles("accounts")
626
def add_user():
627
    debug(env.host, " * adding user %s to astakos..." % env.env.user_email)
628
    email = env.env.user_email
629
    name = env.env.user_name
630
    lastname = env.env.user_lastname
631
    passwd = env.env.user_passwd
632
    cmd = """
633
    snf-manage user-add {0} {1} {2}
634
    """.format(email, name, lastname)
635
    try_run(cmd)
636
    with settings(host_string=env.env.db.ip):
637
        uid, user_auth_token, user_uuid = get_auth_token_from_db(email)
638
    cmd = """
639
    snf-manage user-modify --password {0} {1}
640
    """.format(passwd, uid)
641
    try_run(cmd)
642

    
643

    
644
@roles("accounts")
645
def activate_user(user_email=None):
646
    if not user_email:
647
        user_email = env.env.user_email
648
    debug(env.host, " * Activate user %s..." % user_email)
649
    with settings(host_string=env.env.db.ip):
650
        uid, user_auth_token, user_uuid = get_auth_token_from_db(user_email)
651

    
652
    cmd = """
653
    snf-manage user-modify --verify {0}
654
    snf-manage user-modify --accept {0}
655
    """.format(uid)
656
    try_run(cmd)
657

    
658

    
659
@roles("accounts")
660
def setup_astakos():
661
    debug(env.host, "Setting up snf-astakos-app...")
662
    setup_gunicorn()
663
    setup_apache()
664
    setup_webproject()
665
    install_package("python-django-south")
666
    install_package("snf-astakos-app")
667
    install_package("kamaki")
668

    
669
    tmpl = "/etc/synnefo/astakos.conf"
670
    replace = {
671
        "ACCOUNTS": env.env.accounts.fqdn,
672
        "domain": env.env.domain,
673
        "CYCLADES": env.env.cyclades.fqdn,
674
        "PITHOS": env.env.pithos.fqdn,
675
    }
676
    custom = customize_settings_from_tmpl(tmpl, replace)
677
    try_put(custom, tmpl, mode=0644)
678
    if env.csrf_disable:
679
        cmd = """
680
cat <<EOF >> /etc/synnefo/astakos.conf
681
try:
682
  MIDDLEWARE_CLASSES.remove('django.middleware.csrf.CsrfViewMiddleware')
683
except:
684
  pass
685
EOF
686
"""
687
        try_run(cmd)
688

    
689
    try_run("/etc/init.d/gunicorn restart")
690

    
691
    cmd = """
692
    snf-manage syncdb --noinput
693
    snf-manage migrate im --delete-ghost-migrations
694
    snf-manage migrate quotaholder_app
695
    snf-manage migrate oa2
696
    """
697
    try_run(cmd)
698

    
699

    
700
@roles("accounts")
701
def get_service_details(service="pithos"):
702
    debug(env.host,
703
          " * Getting registered details for %s service..." % service)
704
    result = try_run("snf-manage component-list -o id,name,token")
705
    r = re.compile(r".*%s.*" % service, re.M)
706
    service_id, _, service_token = r.search(result).group().split()
707
    # print("%s: %s %s" % (service, service_id, service_token))
708
    return (service_id, service_token)
709

    
710

    
711
@roles("db")
712
def get_auth_token_from_db(user_email=None):
713
    if not user_email:
714
        user_email = env.env.user_email
715
    debug(env.host,
716
          " * Getting authentication token and uuid for user %s..."
717
          % user_email)
718
    cmd = """
719
echo "select id, auth_token, uuid, email from auth_user, im_astakosuser \
720
where auth_user.id = im_astakosuser.user_ptr_id and auth_user.email = '{0}';" \
721
> /tmp/psqlcmd
722
su - postgres -c  "psql -w -d snf_apps -f /tmp/psqlcmd"
723
""".format(user_email)
724

    
725
    result = try_run(cmd)
726
    r = re.compile(r"(\d+)[ |]*(\S+)[ |]*(\S+)[ |]*" + user_email, re.M)
727
    match = r.search(result)
728
    uid, user_auth_token, user_uuid = match.groups()
729
    # print("%s: %s %s %s" % ( user_email, uid, user_auth_token, user_uuid))
730

    
731
    return (uid, user_auth_token, user_uuid)
732

    
733

    
734
@roles("cms")
735
def cms_loaddata():
736
    debug(env.host, " * Loading cms initial data...")
737
    if env.cms_pass:
738
        debug(env.host, "Aborting. Prerequisites not met.")
739
        return
740
    tmpl = "/tmp/sites.json"
741
    replace = {}
742
    custom = customize_settings_from_tmpl(tmpl, replace)
743
    try_put(custom, tmpl)
744

    
745
    tmpl = "/tmp/page.json"
746
    replace = {}
747
    custom = customize_settings_from_tmpl(tmpl, replace)
748
    try_put(custom, tmpl)
749

    
750
    cmd = """
751
    snf-manage loaddata /tmp/sites.json
752
    snf-manage loaddata /tmp/page.json
753
    snf-manage createsuperuser --username=admin --email=admin@{0} --noinput
754
    """.format(env.env.domain)
755
    try_run(cmd)
756

    
757

    
758
@roles("cms")
759
def setup_cms():
760
    debug(env.host, "Setting up cms...")
761
    if env.cms_pass:
762
        debug(env.host, "Aborting. Prerequisites not met.")
763
        return
764
    with settings(hide("everything")):
765
        try_run("ping -c1 accounts." + env.env.domain)
766
    setup_gunicorn()
767
    setup_apache()
768
    setup_webproject()
769
    install_package("snf-cloudcms")
770

    
771
    tmpl = "/etc/synnefo/cms.conf"
772
    replace = {
773
        "ACCOUNTS": env.env.accounts.fqdn,
774
        }
775
    custom = customize_settings_from_tmpl(tmpl, replace)
776
    try_put(custom, tmpl, mode=0644)
777
    try_run("/etc/init.d/gunicorn restart")
778

    
779
    cmd = """
780
    snf-manage syncdb
781
    snf-manage migrate --delete-ghost-migrations
782
    """.format(env.env.domain)
783
    try_run(cmd)
784

    
785

    
786
def setup_nfs_dirs():
787
    debug(env.host, " * Creating NFS mount point for pithos and ganeti...")
788
    cmd = """
789
    mkdir -p {0}
790
    cd {0}
791
    mkdir -p data
792
    chown www-data:www-data data
793
    chmod g+ws data
794
    mkdir -p {1}
795
    """.format(env.env.pithos_dir, env.env.image_dir)
796
    try_run(cmd)
797

    
798

    
799
@roles("nodes")
800
def setup_nfs_clients():
801
    if env.host == env.env.pithos.ip:
802
        return
803

    
804
    host_info = env.env.ips_info[env.host]
805
    debug(env.host, " * Mounting pithos NFS mount point...")
806
    with settings(hide("everything")):
807
        try_run("ping -c1 " + env.env.pithos.hostname)
808
    with settings(host_string=env.env.pithos.ip):
809
        update_nfs_exports(host_info.ip)
810

    
811
    install_package("nfs-common")
812
    for d in [env.env.pithos_dir, env.env.image_dir]:
813
        try_run("mkdir -p " + d)
814
        cmd = """
815
echo "{0}:{1} {1}  nfs defaults,rw,noatime,rsize=131072,\
816
wsize=131072,timeo=14,intr,noacl" >> /etc/fstab
817
""".format(env.env.pithos.ip, d)
818
        try_run(cmd)
819
        try_run("mount " + d)
820

    
821

    
822
@roles("pithos")
823
def update_nfs_exports(ip):
824
    tmpl = "/tmp/exports"
825
    replace = {
826
        "pithos_dir": env.env.pithos_dir,
827
        "image_dir": env.env.image_dir,
828
        "ip": ip,
829
    }
830
    custom = customize_settings_from_tmpl(tmpl, replace)
831
    try_put(custom, tmpl)
832
    try_run("cat %s >> /etc/exports" % tmpl)
833
    try_run("/etc/init.d/nfs-kernel-server restart")
834

    
835

    
836
@roles("pithos")
837
def setup_nfs_server():
838
    debug(env.host, " * Setting up NFS server for pithos...")
839
    setup_nfs_dirs()
840
    install_package("nfs-kernel-server")
841

    
842

    
843
@roles("pithos")
844
def setup_pithos():
845
    debug(env.host, "Setting up snf-pithos-app...")
846
    with settings(hide("everything")):
847
        try_run("ping -c1 accounts." + env.env.domain)
848
        try_run("ping -c1 " + env.env.db.ip)
849
    setup_gunicorn()
850
    setup_apache()
851
    setup_webproject()
852

    
853
    with settings(host_string=env.env.accounts.ip):
854
        service_id, service_token = get_service_details("pithos")
855

    
856
    install_package("kamaki")
857
    install_package("snf-pithos-backend")
858
    install_package("snf-pithos-app")
859
    tmpl = "/etc/synnefo/pithos.conf"
860
    replace = {
861
        "ACCOUNTS": env.env.accounts.fqdn,
862
        "PITHOS": env.env.pithos.fqdn,
863
        "db_node": env.env.db.ip,
864
        "synnefo_user": env.env.synnefo_user,
865
        "synnefo_db_passwd": env.env.synnefo_db_passwd,
866
        "pithos_dir": env.env.pithos_dir,
867
        "PITHOS_SERVICE_TOKEN": service_token,
868
        }
869
    custom = customize_settings_from_tmpl(tmpl, replace)
870
    try_put(custom, tmpl, mode=0644)
871
    try_run("/etc/init.d/gunicorn restart")
872

    
873
    install_package("snf-pithos-webclient")
874
    tmpl = "/etc/synnefo/webclient.conf"
875
    replace = {
876
        "ACCOUNTS": env.env.accounts.fqdn,
877
        "PITHOS_UI_CLOUDBAR_ACTIVE_SERVICE": service_id,
878
        }
879
    custom = customize_settings_from_tmpl(tmpl, replace)
880
    try_put(custom, tmpl, mode=0644)
881

    
882
    try_run("/etc/init.d/gunicorn restart")
883
    #TOFIX: the previous command lets pithos-backend create blocks and maps
884
    #       with root owner
885
    try_run("chown -R www-data:www-data %s/data " % env.env.pithos_dir)
886
    #try_run("pithos-migrate stamp 4c8ccdc58192")
887
    #try_run("pithos-migrate upgrade head")
888

    
889

    
890
@roles("ganeti")
891
def setup_ganeti():
892
    debug(env.host, "Setting up snf-ganeti...")
893
    node_info = env.env.ips_info[env.host]
894
    with settings(hide("everything")):
895
        #if env.enable_lvm:
896
        #    try_run("vgs " + env.env.vg)
897
        try_run("getent hosts " + env.env.cluster.fqdn)
898
        try_run("getent hosts %s | grep -v ^127" % env.host)
899
        try_run("hostname -f | grep " + node_info.fqdn)
900
        #try_run("ip link show " + env.env.common_bridge)
901
        #try_run("ip link show " + env.env.common_bridge)
902
        #try_run("apt-get update")
903
    install_package("qemu-kvm")
904
    install_package("python-bitarray")
905
    install_package("ganeti-haskell")
906
    install_package("ganeti-htools")
907
    install_package("snf-ganeti")
908
    try_run("mkdir -p /srv/ganeti/file-storage/")
909
    cmd = """
910
cat <<EOF > /etc/ganeti/file-storage-paths
911
/srv/ganeti/file-storage
912
/srv/ganeti/shared-file-storage
913
EOF
914
"""
915
    try_run(cmd)
916

    
917

    
918
@roles("master")
919
def add_rapi_user():
920
    debug(env.host, " * Adding RAPI user to Ganeti backend...")
921
    cmd = """
922
    echo -n "{0}:Ganeti Remote API:{1}" | openssl md5 | sed 's/^.* //'
923
    """.format(env.env.synnefo_user, env.env.synnefo_rapi_passwd)
924
    result = try_run(cmd)
925
    if result.startswith("(stdin)= "):
926
        result = result.split("(stdin)= ")[1]
927
    cmd = """
928
    echo "{0} {1}{2} write" >> /var/lib/ganeti/rapi/users
929
    """.format(env.env.synnefo_user, '{ha1}', result)
930
    try_run(cmd)
931
    try_run("/etc/init.d/ganeti restart")
932

    
933

    
934
@roles("master")
935
def add_nodes():
936
    nodes = env.env.cluster_nodes.split(",")
937
    nodes.remove(env.env.master_node)
938
    debug(env.host, " * Adding nodes to Ganeti backend...")
939
    for n in nodes:
940
        add_node(n)
941

    
942

    
943
@roles("master")
944
def add_node(node):
945
    node_info = env.env.nodes_info[node]
946
    debug(env.host, " * Adding node %s to Ganeti backend..." % node_info.fqdn)
947
    cmd = "gnt-node add --no-ssh-key-check --master-capable=yes " + \
948
          "--vm-capable=yes " + node_info.fqdn
949
    try_run(cmd)
950

    
951

    
952
@roles("ganeti")
953
def enable_drbd():
954
    if env.enable_drbd:
955
        debug(env.host, " * Enabling DRBD...")
956
        install_package("drbd8-utils")
957
        try_run("modprobe drbd minor_count=255 usermode_helper=/bin/true")
958
        try_run("echo drbd minor_count=255 usermode_helper=/bin/true " +
959
                ">> /etc/modules")
960

    
961

    
962
@roles("master")
963
def setup_drbd_dparams():
964
    if env.enable_drbd:
965
        debug(env.host,
966
              " * Twicking drbd related disk parameters in Ganeti...")
967
        cmd = """
968
        gnt-cluster modify --disk-parameters=drbd:metavg={0}
969
        gnt-group modify --disk-parameters=drbd:metavg={0} default
970
        """.format(env.env.vg)
971
        try_run(cmd)
972

    
973

    
974
@roles("master")
975
def enable_lvm():
976
    if env.enable_lvm:
977
        debug(env.host, " * Enabling LVM...")
978
        cmd = """
979
        gnt-cluster modify --vg-name={0}
980
        """.format(env.env.vg)
981
        try_run(cmd)
982
    else:
983
        debug(env.host, " * Disabling LVM...")
984
        try_run("gnt-cluster modify --no-lvm-storage")
985

    
986

    
987
@roles("master")
988
def destroy_cluster():
989
    debug(env.host, " * Destroying Ganeti cluster...")
990
    #TODO: remove instances first
991
    allnodes = env.env.cluster_hostnames[:]
992
    allnodes.remove(env.host)
993
    for n in allnodes:
994
        host_info = env.env.ips_info[env.host]
995
        debug(env.host, " * Removing node %s..." % n)
996
        cmd = "gnt-node remove  " + host_info.fqdn
997
        try_run(cmd)
998
    try_run("gnt-cluster destroy --yes-do-it")
999

    
1000

    
1001
@roles("master")
1002
def init_cluster():
1003
    debug(env.host, " * Initializing Ganeti backend...")
1004
    # extra = ""
1005
    # if env.enable_lvm:
1006
    #     extra += " --vg-name={0} ".format(env.env.vg)
1007
    # else:
1008
    #     extra += " --no-lvm-storage "
1009
    # if not env.enable_drbd:
1010
    #     extra += " --no-drbd-storage "
1011
    extra = " --no-lvm-storage --no-drbd-storage "
1012
    cmd = """
1013
    gnt-cluster init --enabled-hypervisors=kvm \
1014
        {0} \
1015
        --nic-parameters link={1},mode=bridged \
1016
        --master-netdev {2} \
1017
        --default-iallocator hail \
1018
        --hypervisor-parameters kvm:kernel_path=,vnc_bind_address=0.0.0.0 \
1019
        --no-ssh-init --no-etc-hosts \
1020
        {3}
1021
    """.format(extra, env.env.common_bridge,
1022
               env.env.cluster_netdev, env.env.cluster.fqdn)
1023
    try_run(cmd)
1024
    cmd = """gnt-cluster modify --enabled-disk-templates file,plain,ext"""
1025
    try_run(cmd)
1026

    
1027

    
1028
@roles("ganeti")
1029
def debootstrap():
1030
    install_package("ganeti-instance-debootstrap")
1031

    
1032

    
1033
@roles("ganeti")
1034
def setup_image_host():
1035
    debug(env.host, "Setting up snf-image...")
1036
    install_package("snf-pithos-backend")
1037
    install_package("snf-image")
1038
    try_run("mkdir -p %s" % env.env.image_dir)
1039
    tmpl = "/etc/default/snf-image"
1040
    replace = {
1041
        "synnefo_user": env.env.synnefo_user,
1042
        "synnefo_db_passwd": env.env.synnefo_db_passwd,
1043
        "pithos_dir": env.env.pithos_dir,
1044
        "db_node": env.env.db.ip,
1045
        "image_dir": env.env.image_dir,
1046
    }
1047
    custom = customize_settings_from_tmpl(tmpl, replace)
1048
    try_put(custom, tmpl)
1049

    
1050

    
1051
@roles("ganeti")
1052
def setup_image_helper():
1053
    debug(env.host, " * Updating helper image...")
1054
    cmd = """
1055
    snf-image-update-helper -y
1056
    """
1057
    try_run(cmd)
1058

    
1059

    
1060
@roles("ganeti")
1061
def setup_gtools():
1062
    debug(env.host, " * Setting up snf-cyclades-gtools...")
1063
    with settings(hide("everything")):
1064
        try_run("ping -c1 " + env.env.mq.ip)
1065
    setup_common()
1066
    install_package("snf-cyclades-gtools")
1067
    tmpl = "/etc/synnefo/gtools.conf"
1068
    replace = {
1069
        "synnefo_user": env.env.synnefo_user,
1070
        "synnefo_rabbitmq_passwd": env.env.synnefo_rabbitmq_passwd,
1071
        "mq_node": env.env.mq.ip,
1072
    }
1073
    custom = customize_settings_from_tmpl(tmpl, replace)
1074
    try_put(custom, tmpl)
1075

    
1076
    cmd = """
1077
    sed -i 's/false/true/' /etc/default/snf-ganeti-eventd
1078
    /etc/init.d/snf-ganeti-eventd start
1079
    """
1080
    try_run(cmd)
1081

    
1082

    
1083
@roles("ganeti")
1084
def setup_iptables():
1085
    debug(env.host, " * Setting up iptables to mangle DHCP requests...")
1086
    cmd = """
1087
    iptables -t mangle -A PREROUTING -i br+ -p udp -m udp --dport 67 \
1088
            -j NFQUEUE --queue-num 42
1089
    iptables -t mangle -A PREROUTING -i tap+ -p udp -m udp --dport 67 \
1090
            -j NFQUEUE --queue-num 42
1091
    iptables -t mangle -A PREROUTING -i prv+ -p udp -m udp --dport 67 \
1092
            -j NFQUEUE --queue-num 42
1093

1094
    ip6tables -t mangle -A PREROUTING -i br+ -p ipv6-icmp -m icmp6 \
1095
            --icmpv6-type 133 -j NFQUEUE --queue-num 43
1096
    ip6tables -t mangle -A PREROUTING -i br+ -p ipv6-icmp -m icmp6 \
1097
            --icmpv6-type 135 -j NFQUEUE --queue-num 44
1098
    """
1099
    try_run(cmd)
1100

    
1101

    
1102
@roles("ganeti")
1103
def setup_network():
1104
    debug(env.host,
1105
          "Setting up networking for Ganeti instances (nfdhcpd, etc.)...")
1106
    install_package("python-nfqueue")
1107
    install_package("nfdhcpd")
1108
    tmpl = "/etc/nfdhcpd/nfdhcpd.conf"
1109
    replace = {
1110
        "ns_node_ip": env.env.ns.ip
1111
    }
1112
    custom = customize_settings_from_tmpl(tmpl, replace)
1113
    try_put(custom, tmpl)
1114
    try_run("/etc/init.d/nfdhcpd restart")
1115

    
1116
    install_package("snf-network")
1117
    cmd = """
1118
sed -i 's/MAC_MASK.*/MAC_MASK = ff:ff:f0:00:00:00/' /etc/default/snf-network
1119
    """
1120
    try_run(cmd)
1121

    
1122

    
1123
@roles("router")
1124
def setup_router():
1125
    debug(env.host, " * Setting up internal router for NAT...")
1126
    cmd = """
1127
    echo 1 > /proc/sys/net/ipv4/ip_forward
1128
    iptables -t nat -A POSTROUTING -s {0} -o {3} -j MASQUERADE
1129
    ip addr add {1} dev {2}
1130
    ip route add {0} dev {2} src {1}
1131
    """.format(env.env.synnefo_public_network_subnet,
1132
               env.env.synnefo_public_network_gateway,
1133
               env.env.common_bridge, env.env.public_iface)
1134
    try_run(cmd)
1135

    
1136

    
1137
@roles("cyclades")
1138
def cyclades_loaddata():
1139
    debug(env.host, " * Loading initial data for cyclades...")
1140
    try_run("snf-manage flavor-create %s %s %s %s" % (env.env.flavor_cpu,
1141
                                                      env.env.flavor_ram,
1142
                                                      env.env.flavor_disk,
1143
                                                      env.env.flavor_storage))
1144
    #run("snf-manage loaddata flavors")
1145

    
1146

    
1147
@roles("ganeti", "stats")
1148
def setup_collectd():
1149
    install_package("collectd")
1150
    tmpl = "/etc/collectd/collectd.conf"
1151
    replace = {}
1152
    custom = customize_settings_from_tmpl(tmpl, replace)
1153
    try_put(custom, tmpl, mode=0644)
1154

    
1155

    
1156
@roles("ganeti")
1157
def setup_ganeti_collectd():
1158
    setup_collectd()
1159

    
1160
    tmpl = "/etc/collectd/passwd"
1161
    replace = {}
1162
    custom = customize_settings_from_tmpl(tmpl, replace)
1163
    try_put(custom, tmpl, mode=0644)
1164

    
1165
    tmpl = "/etc/collectd/synnefo-ganeti.conf"
1166
    replace = {
1167
        "STATS": env.env.stats.fqdn,
1168
        }
1169
    custom = customize_settings_from_tmpl(tmpl, replace)
1170
    try_put(custom, tmpl, mode=0644)
1171

    
1172
    try_run("/etc/init.d/collectd restart")
1173

    
1174

    
1175
@roles("stats")
1176
def setup_stats_collectd():
1177
    setup_collectd()
1178
    tmpl = "/etc/collectd/synnefo-stats.conf"
1179

    
1180
    replace = {
1181
        "STATS": env.env.stats.fqdn,
1182
        }
1183
    custom = customize_settings_from_tmpl(tmpl, replace)
1184
    try_put(custom, tmpl, mode=0644)
1185
    try_run("/etc/init.d/collectd restart")
1186

    
1187

    
1188
@roles("stats")
1189
def setup_stats():
1190
    debug(env.host, "Setting up snf-stats-app...")
1191
    setup_stats_collectd()
1192
    setup_gunicorn()
1193
    setup_apache()
1194
    setup_webproject()
1195
    install_package("snf-stats-app")
1196
    cmd = """
1197
    mkdir /var/cache/snf-stats-app/
1198
    chown www-data:www-data /var/cache/snf-stats-app/
1199
    """
1200
    try_run(cmd)
1201
    tmpl = "/etc/synnefo/stats.conf"
1202

    
1203
    replace = {
1204
        "STATS": env.env.stats.fqdn,
1205
        }
1206
    custom = customize_settings_from_tmpl(tmpl, replace)
1207
    try_put(custom, tmpl, mode=0644)
1208
    try_run("/etc/init.d/gunicorn restart")
1209

    
1210

    
1211
@roles("cyclades")
1212
def setup_cyclades():
1213
    debug(env.host, "Setting up snf-cyclades-app...")
1214
    with settings(hide("everything")):
1215
        try_run("ping -c1 accounts." + env.env.domain)
1216
        try_run("ping -c1 " + env.env.db.ip)
1217
        try_run("ping -c1 " + env.env.mq.ip)
1218
    setup_gunicorn()
1219
    setup_apache()
1220
    setup_webproject()
1221
    install_package("memcached")
1222
    install_package("python-memcache")
1223
    install_package("snf-pithos-backend")
1224
    install_package("kamaki")
1225
    install_package("snf-cyclades-app")
1226
    install_package("python-django-south")
1227
    tmpl = "/etc/synnefo/cyclades.conf"
1228

    
1229
    with settings(host_string=env.env.accounts.ip):
1230
        service_id, service_token = get_service_details("cyclades")
1231

    
1232
    replace = {
1233
        "ACCOUNTS": env.env.accounts.fqdn,
1234
        "CYCLADES": env.env.cyclades.fqdn,
1235
        "mq_node": env.env.mq.ip,
1236
        "db_node": env.env.db.ip,
1237
        "synnefo_user": env.env.synnefo_user,
1238
        "synnefo_db_passwd": env.env.synnefo_db_passwd,
1239
        "synnefo_rabbitmq_passwd": env.env.synnefo_rabbitmq_passwd,
1240
        "pithos_dir": env.env.pithos_dir,
1241
        "common_bridge": env.env.common_bridge,
1242
        "HOST": env.env.cyclades.ip,
1243
        "domain": env.env.domain,
1244
        "CYCLADES_SERVICE_TOKEN": service_token,
1245
        'STATS': env.env.stats.fqdn,
1246
        }
1247
    custom = customize_settings_from_tmpl(tmpl, replace)
1248
    try_put(custom, tmpl, mode=0644)
1249
    try_run("/etc/init.d/gunicorn restart")
1250

    
1251
    cmd = """
1252
    sed -i 's/false/true/' /etc/default/snf-dispatcher
1253
    /etc/init.d/snf-dispatcher start
1254
    """
1255
    try_run(cmd)
1256

    
1257
    try_run("snf-manage syncdb")
1258
    try_run("snf-manage migrate --delete-ghost-migrations")
1259

    
1260

    
1261
@roles("cyclades")
1262
def get_backend_id(cluster_name="ganeti1.synnefo.deploy.local"):
1263
    backend_id = try_run("snf-manage backend-list 2>/dev/null " +
1264
                         "| grep %s | awk '{print $1}'" % cluster_name)
1265
    return backend_id
1266

    
1267

    
1268
@roles("cyclades")
1269
def add_backend():
1270
    debug(env.host,
1271
          "adding %s ganeti backend to cyclades..." % env.env.cluster.fqdn)
1272
    with settings(hide("everything")):
1273
        try_run("ping -c1 " + env.env.cluster.fqdn)
1274
    cmd = """
1275
    snf-manage backend-add --clustername={0} --user={1} --pass={2}
1276
    """.format(env.env.cluster.fqdn, env.env.synnefo_user,
1277
               env.env.synnefo_rapi_passwd)
1278
    try_run(cmd)
1279
    backend_id = get_backend_id(env.env.cluster.fqdn)
1280
    try_run("snf-manage backend-modify --drained=False " + backend_id)
1281

    
1282

    
1283
@roles("cyclades")
1284
def pin_user_to_backend(user_email):
1285
    backend_id = get_backend_id(env.env.cluster.fqdn)
1286
    # pin user to backend
1287
    cmd = """
1288
cat <<EOF >> /etc/synnefo/cyclades.conf
1289

1290
BACKEND_PER_USER = {
1291
  '{0}': {1},
1292
}
1293

1294
EOF
1295
/etc/init.d/gunicorn restart
1296
""".format(user_email, backend_id)
1297
    try_run(cmd)
1298

    
1299

    
1300
@roles("cyclades")
1301
def add_pools():
1302
    debug(env.host,
1303
          " * Creating pools of resources (brigdes, mac prefixes) " +
1304
          "in cyclades...")
1305
    try_run("snf-manage pool-create --type=mac-prefix " +
1306
            "--base=aa:00:0 --size=65536")
1307
    try_run("snf-manage pool-create --type=bridge --base=prv --size=20")
1308

    
1309

    
1310
@roles("accounts", "cyclades", "pithos")
1311
def export_services():
1312
    debug(env.host, " * Exporting services...")
1313
    host = env.host
1314
    services = []
1315
    if host == env.env.cyclades.ip:
1316
        services.append("cyclades")
1317
    if host == env.env.pithos.ip:
1318
        services.append("pithos")
1319
    if host == env.env.accounts.ip:
1320
        services.append("astakos")
1321
    for service in services:
1322
        filename = "%s_services.json" % service
1323
        cmd = "snf-manage service-export-%s > %s" % (service, filename)
1324
        run(cmd)
1325
        try_get(filename, filename+".local")
1326

    
1327

    
1328
@roles("accounts")
1329
def import_services():
1330
    debug(env.host, " * Registering services to astakos...")
1331
    for service in ["cyclades", "pithos", "astakos"]:
1332
        filename = "%s_services.json" % service
1333
        try_put(filename + ".local", filename)
1334
        cmd = "snf-manage service-import --json=%s" % filename
1335
        run(cmd)
1336

    
1337
    debug(env.host, " * Setting default quota...")
1338
    cmd = """
1339
    snf-manage resource-modify --default-quota 40G pithos.diskspace
1340
    snf-manage resource-modify --default-quota 2 astakos.pending_app
1341
    snf-manage resource-modify --default-quota 4 cyclades.vm
1342
    snf-manage resource-modify --default-quota 40G cyclades.disk
1343
    snf-manage resource-modify --default-quota 16G cyclades.total_ram
1344
    snf-manage resource-modify --default-quota 8G cyclades.ram
1345
    snf-manage resource-modify --default-quota 32 cyclades.total_cpu
1346
    snf-manage resource-modify --default-quota 16 cyclades.cpu
1347
    snf-manage resource-modify --default-quota 4 cyclades.network.private
1348
    snf-manage resource-modify --default-quota 4 cyclades.floating_ip
1349
    """
1350
    try_run(cmd)
1351

    
1352

    
1353
@roles("accounts")
1354
def set_user_quota():
1355
    debug(env.host, " * Setting user quota...")
1356
    cmd = """
1357
    snf-manage user-modify -f --all --base-quota pithos.diskspace 40G
1358
    snf-manage user-modify -f --all --base-quota astakos.pending_app 2
1359
    snf-manage user-modify -f --all --base-quota cyclades.vm 4
1360
    snf-manage user-modify -f --all --base-quota cyclades.disk 40G
1361
    snf-manage user-modify -f --all --base-quota cyclades.total_ram 16G
1362
    snf-manage user-modify -f --all --base-quota cyclades.ram 8G
1363
    snf-manage user-modify -f --all --base-quota cyclades.total_cpu 32
1364
    snf-manage user-modify -f --all --base-quota cyclades.cpu 16
1365
    snf-manage user-modify -f --all --base-quota cyclades.network.private 4
1366
    snf-manage user-modify -f --all --base-quota cyclades.floating_ip 4
1367
    """
1368
    try_run(cmd)
1369

    
1370

    
1371
@roles("cyclades")
1372
def add_network():
1373
    debug(env.host, " * Adding public network in cyclades...")
1374
    cmd = """
1375
    snf-manage network-create --subnet={0} --gateway={1} --public \
1376
        --dhcp=True --flavor={2} --mode=bridged --link={3} --name=Internet \
1377
        --floating-ip-pool=True
1378
    """.format(env.env.synnefo_public_network_subnet,
1379
               env.env.synnefo_public_network_gateway,
1380
               env.env.synnefo_public_network_type,
1381
               env.env.common_bridge)
1382
    try_run(cmd)
1383
    if env.env.testing_vm:
1384
        cmd = ("snf-manage network-create --subnet6=babe::/64"
1385
               " --gateway6=babe::1 --public --flavor={0} --mode=bridged"
1386
               " --link={1} --name=IPv6PublicNetwork"
1387
               .format(env.env.synnefo_public_network_type,
1388
                       env.env.common_bridge))
1389
        try_run(cmd)
1390

    
1391

    
1392
@roles("cyclades")
1393
def setup_vncauthproxy():
1394
    debug(env.host, " * Setting up vncauthproxy...")
1395
    user = "synnefo"
1396
    salt = "$6$7FUdSvFcWAs3hfVj$"
1397
    passhash = "ZwvnvpQclTrDYWEwBvZDMRJZNgb6ZUKT1vNsh9NzUIxMpzBuGgMqYxCDTYF"\
1398
               "6OZcbunDZb88pjL2EIBnzrGMQW1"
1399
    cmd = """
1400
    mkdir /var/lib/vncauthproxy
1401
    echo '%s:%s%s' > /var/lib/vncauthproxy/users
1402
    """ % (user, salt, passhash)
1403
    try_run(cmd)
1404
    install_package("snf-vncauthproxy")
1405

    
1406

    
1407
@roles("client")
1408
def setup_kamaki():
1409
    debug(env.host, "Setting up kamaki client...")
1410
    with settings(hide("everything")):
1411
        try_run("ping -c1 accounts." + env.env.domain)
1412
        try_run("ping -c1 cyclades." + env.env.domain)
1413
        try_run("ping -c1 pithos." + env.env.domain)
1414

    
1415
    with settings(host_string=env.env.db.ip):
1416
        uid, user_auth_token, user_uuid = \
1417
            get_auth_token_from_db(env.env.user_email)
1418

    
1419
    install_package("python-progress")
1420
    install_package("kamaki")
1421
    cmd = """
1422
    kamaki config set cloud.default.url "https://{0}/astakos/identity/v2.0"
1423
    kamaki config set cloud.default.token {1}
1424
    """.format(env.env.accounts.fqdn, user_auth_token)
1425
    try_run(cmd)
1426
    try_run("kamaki container create images")
1427

    
1428

    
1429
@roles("client")
1430
def upload_image(image="debian_base.diskdump"):
1431
    debug(env.host, " * Uploading initial image to pithos...")
1432
    image = "debian_base.diskdump"
1433
    try_run("wget {0} -O /tmp/{1}".format(env.env.debian_base_url, image))
1434
    try_run("kamaki file upload --container images /tmp/{0} {0}".format(image))
1435

    
1436

    
1437
@roles("client")
1438
def register_image(image="debian_base.diskdump"):
1439
    debug(env.host, " * Register image to plankton...")
1440
    # with settings(host_string=env.env.db.ip):
1441
    #     uid, user_auth_token, user_uuid = \
1442
    #        get_auth_token_from_db(env.env.user_email)
1443

    
1444
    image_location = "images:{0}".format(image)
1445
    cmd = """
1446
    sleep 5
1447
    kamaki image register "Debian Base" {0} --public --disk-format=diskdump \
1448
            --property OSFAMILY=linux --property ROOT_PARTITION=1 \
1449
            --property description="Debian Squeeze Base System" \
1450
            --property size=450M --property kernel=2.6.32 \
1451
            --property GUI="No GUI" --property sortorder=1 \
1452
            --property USERS=root --property OS=debian
1453
    """.format(image_location)
1454
    try_run(cmd)
1455

    
1456

    
1457
@roles("client")
1458
def setup_burnin():
1459
    debug(env.host, "Setting up burnin testing tool...")
1460
    install_package("kamaki")
1461
    install_package("snf-tools")
1462

    
1463

    
1464
@roles("pithos")
1465
def add_image_locally():
1466
    debug(env.host,
1467
          " * Getting image locally in order snf-image to use it directly..")
1468
    image = "debian_base.diskdump"
1469
    try_run("wget {0} -O {1}/{2}".format(
1470
            env.env.debian_base_url, env.env.image_dir, image))
1471

    
1472

    
1473
@roles("master")
1474
def gnt_instance_add(name="test"):
1475
    debug(env.host, " * Adding test instance to Ganeti...")
1476
    osp = """img_passwd=gamwtosecurity,\
1477
img_format=diskdump,img_id=debian_base,\
1478
img_properties='{"OSFAMILY":"linux"\,"ROOT_PARTITION":"1"}'"""
1479
    cmd = """
1480
    gnt-instance add  -o snf-image+default --os-parameters {0} \
1481
            -t plain --disk 0:size=1G --no-name-check --no-ip-check \
1482
            --net 0:ip=pool,network=test --no-install \
1483
            --hypervisor-parameters kvm:machine_version=pc-1.0 {1}
1484
    """.format(osp, name)
1485
    try_run(cmd)
1486

    
1487

    
1488
@roles("master")
1489
def gnt_network_add(name="test", subnet="10.0.0.0/26", gw="10.0.0.1",
1490
                    mode="bridged", link="br0"):
1491
    debug(env.host, " * Adding test network to Ganeti...")
1492
    cmd = """
1493
    gnt-network add --network={1} --gateway={2} {0}
1494
    gnt-network connect {0} {3} {4}
1495
    """.format(name, subnet, gw, mode, link)
1496
    try_run(cmd)
1497

    
1498

    
1499
@roles("ips")
1500
def test():
1501
    debug(env.host, "Testing...")
1502
    try_run("hostname && date")