Revision 3c3bccab snf-deploy/snfdeploy/__init__.py

b/snf-deploy/snfdeploy/__init__.py
1
# Copyright (C) 2010, 2011, 2012, 2013 GRNET S.A. All rights reserved.
2
#
3
# Redistribution and use in source and binary forms, with or
4
# without modification, are permitted provided that the following
5
# conditions are met:
6
#
7
#   1. Redistributions of source code must retain the above
8
#      copyright notice, this list of conditions and the following
9
#      disclaimer.
10
#
11
#   2. Redistributions in binary form must reproduce the above
12
#      copyright notice, this list of conditions and the following
13
#      disclaimer in the documentation and/or other materials
14
#      provided with the distribution.
15
#
16
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
17
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
19
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A. OR
20
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
23
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
24
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
26
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
27
# POSSIBILITY OF SUCH DAMAGE.
28
#
29
# The views and conclusions contained in the software and
30
# documentation are those of the authors and should not be
31
# interpreted as representing official policies, either expressed
32
# or implied, of GRNET S.A.
33

  
1 34
import time
2 35
import os
3 36
import argparse
......
5 38
import re
6 39
import random
7 40
import ast
41
import glob
8 42
from snfdeploy.lib import check_pidfile, create_dir, get_default_route, \
9
    random_mac, Conf, Env
10
from snfdeploy import fabfile
43
    random_mac, Conf, Env, Status
44
# from snfdeploy import fabfile
45
from snfdeploy import fabfile2 as fabfile
11 46
from fabric.api import hide, settings, execute, show
12 47

  
13 48

  
......
35 70

  
36 71
  """
37 72

  
38
    if command == "prepare":
39
        print """
40
Usage: snf-deploy prepare
41

  
42
  Run the following actions concerning deployment preparation:
43

  
44
    - Setup an internal Domain Name Server
45
    - Tweak hosts and add ssh keys
46
    - Check network setup
47
    - Setup apt repository and apt-get update
48
    - Setup the nfs server and clients among all nodes
49

  
50
  """
51

  
52 73
    if command == "backend":
53 74
        print """
54
Usage: snf-deploy backend [update]
75
Usage: snf-deploy backend
55 76

  
56 77
  Run the following actions concerning a ganeti backend:
57 78

  
58 79
    - Create and add a backend to cyclades
59
    - Does all the net-infra specific actions in backend nodes
60
      (create/connect bridges, iptables..)
61
    - Does all the storage-infra specific actions in backend nodes
62
      depending on the --extra-disk option \
63
(create VG, enable lvm/drbd storage..)
64

  
65
    or
66

  
67
    - Update packages in an already registered backend in cyclades.
68 80

  
69 81
  """
70 82

  
......
74 86

  
75 87
  Run any of the following fabric commands:
76 88

  
89
    Role setup:
90

  
91
      setup_ns_role
92
      setup_nfs_role
93
      setup_db_role
94
      setup_mq_role
95
      setup_astakos_role
96
      setup_pithos_role
97
      setup_cyclades_role
98
      setup_cms_role
99
      setup_ganeti_role
100
      setup_master_role
101
      setup_stats_role
102
      setup_client_role
103

  
104
    Helper commands:
105

  
106
      update_env_with_user_info
107
      update_env_with_service_info
108
      update_env_with_backend_info
77 109

  
78
    Setup commands:        Init commands:                Admin commands:
79
      setup_apache           add_pools                     activate_user
80
      setup_apt              add_rapi_user                 add_backend
81
      setup_astakos          add_nodes                     add_image_locally
82
      setup_cms              astakos_loaddata              add_network
83
      setup_collectd
84
      setup_common           astakos_register_components   add_ns
85
      setup_cyclades         cms_loaddata                  add_user
86
      setup_db               cyclades_loaddata             connect_bridges
87
      setup_ganeti           enable_drbd                   create_bridges
88
      setup_ganeti_collectd
89
      setup_gtools           init_cluster                  create_vlans
90
      setup_gunicorn         setup_nfs_clients             destroy_db
91
      setup_hosts            setup_nfs_server              \
92
get_auth_token_from_db
93
      setup_image_helper     update_ns_for_ganeti          get_service_details
94
      setup_image_host       astakos_register_pithos_view  gnt_instance_add
95
      setup_iptables                                       gnt_network_add
96
      setup_kamaki         Test commands:                  register_image
97
      setup_lvm              test                          restart_services
98
      setup_mq                                             setup_drbd_dparams
99
      setup_net_infra
100
      setup_network
101
      setup_ns
102
      setup_pithos
103
      setup_pithos_dir
104
      setup_router
105
      setup_stats
106
      setup_stats_collectd
107
      setup_vncauthproxy
108
      setup_webproject
110
    Admin commands:
111

  
112
      update_ns_for_node
113
      update_exports_for_node
114
      allow_db_access
115
      add_ganeti_backend
116
      add_synnefo_user
117
      activate_user
118
      set_default_quota
119
      add_public_networks
120
      add_image
121

  
122

  
123
    Custom command:
124

  
125
      setup --node NODE [--role ROLE | --method METHOD --component COMPONENT]
109 126

  
110 127
  """
111 128

  
......
216 233

  
217 234

  
218 235
def image(args, env):
236
    #FIXME: Create a clean wheezy image and use it for vcluster
219 237
    if env.os == "ubuntu":
220 238
        url = env.ubuntu_image_url
221 239
    else:
......
270 288
    if nodes:
271 289
        ips = [env.nodes_info[n].ip for n in nodes]
272 290

  
273
    fabfile.setup_env(args)
291
    fabfile.setup_env(args, env)
274 292
    with settings(hide(*lhide), show(*lshow)):
275 293
        print " ".join(actions)
276 294
        for a in actions:
277 295
            fn = getattr(fabfile, a)
278
            if not args.dry_run:
279
                if nodes:
280
                    execute(fn, hosts=ips)
281
                else:
282
                    execute(fn)
296
            if nodes:
297
                execute(fn, hosts=ips)
298
            else:
299
                execute(fn)
283 300

  
284 301

  
285 302
def cluster(args, env):
......
379 396
                             "console or not")
380 397
    parser.add_argument("--force", dest="force",
381 398
                        default=False, action="store_true",
382
                        help="Force the creation of new ssh key pairs")
399
                        help="Force things (creation of key pairs"
400
                             " do not abort execution if something fails")
383 401

  
384 402
    parser.add_argument("-i", "--ssh-key", dest="ssh_key",
385 403
                        default=None,
......
399 417
                        default=None,
400 418
                        help="The node to add to the existing cluster")
401 419

  
420
    # options related to custom setup
421
    parser.add_argument("--component", dest="component",
422
                        default=None,
423
                        help="The component class")
424

  
425
    parser.add_argument("--method", dest="method",
426
                        default=None,
427
                        help="The component method")
428

  
429
    parser.add_argument("--role", dest="role",
430
                        default=None,
431
                        help="The target node's role")
432

  
433
    parser.add_argument("--node", dest="node",
434
                        default="node1",
435
                        help="The target node")
436

  
402 437
    # available commands
403 438
    parser.add_argument("command", type=str,
404
                        choices=["packages", "vcluster", "prepare",
405
                                 "synnefo", "backend", "ganeti",
406
                                 "run", "cleanup", "test",
407
                                 "all", "add", "keygen"],
439
                        choices=["packages", "vcluster", "cleanup",
440
                                 "run", "test", "all", "keygen"],
408 441
                        help="Run on of the supported deployment commands")
409 442

  
410 443
    # available actions for the run command
......
421 454

  
422 455
def get_actions(*args):
423 456
    actions = {
424
        # prepare actions
425
        "ns":  ["setup_ns", "setup_resolv_conf"],
426
        "hosts": ["setup_hosts", "add_keys"],
427
        "check": ["check_dhcp", "check_dns",
428
                  "check_connectivity", "check_ssh"],
429
        "apt": ["apt_get_update", "setup_apt"],
430
        "nfs": ["setup_nfs_server", "setup_nfs_clients"],
431
        "prepare":  [
432
            "setup_hosts", "add_keys",
433
            "setup_ns", "setup_resolv_conf",
434
            "check_dhcp", "check_dns", "check_connectivity", "check_ssh",
435
            "apt_get_update", "setup_apt",
436
            "setup_nfs_server", "setup_nfs_clients"
437
        ],
438
        # synnefo actions
439
        "synnefo": [
440
            "setup_mq", "setup_db",
441
            "setup_astakos",
442
            #TODO: astakos-quota fails if no user is added.
443
            #      add_user fails if no groups found
444
            "astakos_loaddata", "add_user", "activate_user",
445
            "astakos_register_components",
446
            "astakos_register_pithos_view",
447
            "setup_cms", "cms_loaddata",
448
            "setup_pithos",
449
            "setup_vncauthproxy",
450
            "setup_cyclades", "cyclades_loaddata", "add_pools",
451
            "export_services", "import_services", "set_user_quota",
452
            "setup_kamaki", "upload_image", "register_image",
453
            "setup_burnin",
454
            "setup_stats"
455
        ],
456
        "supdate": [
457
            "apt_get_update", "setup_astakos",
458
            "setup_cms", "setup_pithos", "setup_cyclades"
459
        ],
460
        # backend actions
461 457
        "backend": [
462
            "setup_hosts",
463
            "update_ns_for_ganeti",
464
            "setup_ganeti", "init_cluster",
465
            "add_rapi_user", "add_nodes",
466
            "setup_image_host", "setup_image_helper",
467
            "setup_network",
468
            "setup_gtools", "add_backend", "add_network",
469
            "setup_lvm", "enable_lvm",
470
            "enable_drbd", "setup_drbd_dparams",
471
            "setup_net_infra", "setup_iptables", "setup_router",
472
        ],
473
        "bstorage": [
474
            "setup_lvm", "enable_lvm",
475
            "enable_drbd", "setup_drbd_dparams"
458
            "setup_master_role",
459
            "setup_ganeti_role",
460
            "add_ganeti_backend",
476 461
        ],
477
        "bnetwork": ["setup_net_infra", "setup_iptables", "setup_router"],
478
        "bupdate": [
479
            "apt_get_update", "setup_ganeti", "setup_image_host",
480
            "setup_image_helper", "setup_network", "setup_gtools"
481
        ],
482
        # ganeti actions
483 462
        "ganeti": [
484
            "update_ns_for_ganeti",
485
            "setup_ganeti", "init_cluster", "add_nodes",
486
            "setup_image_host", "setup_image_helper", "add_image_locally",
487
            "debootstrap", "setup_net_infra",
488
            "setup_lvm", "enable_lvm", "enable_drbd", "setup_drbd_dparams",
489
            "setup_ganeti_collectd"
463
            "setup_ns_role",
464
            "setup_nfs_role",
465
            "setup_master_role",
466
            "setup_ganeti_role",
467
        ],
468
        "all": [
469
            "setup_ns_role",
470
            "setup_nfs_role",
471
            "setup_db_role",
472
            "setup_mq_role",
473
            "setup_astakos_role",
474
            "setup_pithos_role",
475
            "setup_cyclades_role",
476
            "setup_cms_role",
477
            "setup_master_role",
478
            "setup_ganeti_role",
479
            "setup_stats_role",
480
            "set_default_quota",
481
            "add_ganeti_backend",
482
            "add_public_networks",
483
            "add_synnefo_user",
484
            "activate_user",
485
            "setup_client_role",
486
            "add_image",
490 487
        ],
491
        "gupdate": ["setup_apt", "setup_ganeti"],
492
        "gdestroy": ["destroy_cluster"],
488

  
493 489
    }
494 490

  
495 491
    ret = []
......
499 495
    return ret
500 496

  
501 497

  
502
def must_create_keys(force, env):
503
    """Check if we need to create ssh keys
504

  
505
    If force is true we are going to overide the old keys.
506
    Else if there are already generated keys to use, don't create new ones.
498
def must_create_keys(env):
499
    """Check if we ssh keys already exist
507 500

  
508 501
    """
509
    if force:
510
        return True
511 502
    d = os.path.join(env.templates, "root/.ssh")
512 503
    auth_keys_exists = os.path.exists(os.path.join(d, "authorized_keys"))
513 504
    dsa_exists = os.path.exists(os.path.join(d, "id_dsa"))
......
535 526
        os.system(cmd)
536 527

  
537 528

  
538
def add_node(args, env):
539
    actions = [
540
        "update_ns_for_node:" + args.cluster_node,
541
    ]
542
    fabcommand(args, env, actions)
543
    actions = [
544
        "setup_resolv_conf",
545
        "apt_get_update",
546
        "setup_apt",
547
        "setup_hosts",
548
        "add_keys",
549
    ]
550
    fabcommand(args, env, actions, [args.cluster_node])
551

  
552
    actions = get_actions("check")
553
    fabcommand(args, env, actions)
554

  
555
    actions = [
556
        "setup_nfs_clients",
557
        "setup_ganeti",
558
        "setup_image_host", "setup_image_helper",
559
        "setup_network", "setup_gtools",
560
    ]
561
    fabcommand(args, env, actions, [args.cluster_node])
562

  
563
    actions = [
564
        "add_node:" + args.cluster_node,
565
    ]
566
    fabcommand(args, env, actions)
567

  
568
    actions = [
569
        "setup_lvm", "enable_drbd",
570
        "setup_net_infra", "setup_iptables",
571
    ]
572
    fabcommand(args, env, actions, [args.cluster_node])
529
def must_create_ddns_keys(env):
530
    d = os.path.join(env.templates, "root/ddns")
531
    key_exists = glob.glob(os.path.join(d, "Kddns*key"))
532
    private_exists = glob.glob(os.path.join(d, "Kddns*private"))
533
    bind_key_exists = os.path.exists(os.path.join(d, "ddns.key"))
534
    return not (key_exists and private_exists and bind_key_exists)
535

  
536

  
537
def find_ddns_key_files(env):
538
    d = os.path.join(env.templates, "root/ddns")
539
    keys = glob.glob(os.path.join(d, "Kddns*"))
540
    # Here we must have a key!
541
    return map(os.path.basename, keys)
542

  
543

  
544
def do_create_ddns_keys(args, env):
545
    d = os.path.join(env.templates, "root/ddns")
546
    if not os.path.exists(d):
547
        os.mkdir(d)
548
    for filename in os.listdir(d):
549
        os.remove(os.path.join(d, filename))
550
    cmd = """
551
dnssec-keygen -a HMAC-MD5 -b 128 -K {0} -r /dev/urandom -n USER DDNS_UPDATE
552
key=$(cat {0}/Kddns_update*.key | awk '{{ print $7 }}')
553
cat > {0}/ddns.key <<EOF
554
key DDNS_UPDATE {{
555
        algorithm HMAC-MD5.SIG-ALG.REG.INT;
556
        secret "$key";
557
}};
558
EOF
559
""".format(d)
560
    os.system(cmd)
561

  
573 562

  
574 563

  
575 564
def main():
......
577 566

  
578 567
    conf = Conf(args)
579 568
    env = Env(conf)
569
    env.status = Status(args)
580 570

  
581 571
    create_dir(env.run, False)
582 572
    create_dir(env.dns, False)
583 573

  
584 574
    # Check if there are keys to use
585 575
    if args.command == "keygen":
586
        if must_create_keys(args.force, env):
587
            do_create_keys(args, env)
588
            return 0
589
        else:
590
            print "Keys already existed.. aborting"
591
            return 1
576
        if not args.force:
577
            if not must_create_keys(env) or not must_create_ddns_keys(env):
578
                print "Keys already exist.."
579
                print "To override existing ones use --force."
580
                return 1
581
        do_create_keys(args, env)
582
        do_create_ddns_keys(args, env)
583
        return 0
592 584
    else:
593
        if (args.key_inject and (args.ssh_key is None)
594
                and must_create_keys(False, env)):
595
            print "No ssh keys to use. Run `snf-deploy keygen' first."
585
        if ((args.key_inject and not args.ssh_key and must_create_keys(env)) or
586
            must_create_ddns_keys(env)):
587
            print "No ssh/ddns keys to use. Run `snf-deploy keygen' first."
596 588
            return 1
589
        env.ddns_keys = find_ddns_key_files(env)
590
        env.ddns_private_key = "/root/ddns/" + env.ddns_keys[0]
597 591

  
598 592
    if args.command == "test":
599 593
        conf.print_config()
......
612 606
        dnsmasq(args, env)
613 607
        cluster(args, env)
614 608

  
615
    if args.command == "prepare":
616
        actions = get_actions("prepare")
617
        fabcommand(args, env, actions)
618

  
619
    if args.command == "synnefo":
620
        actions = get_actions("synnefo")
621
        fabcommand(args, env, actions)
622

  
623 609
    if args.command == "backend":
624 610
        actions = get_actions("backend")
625 611
        fabcommand(args, env, actions)
......
629 615
        fabcommand(args, env, actions)
630 616

  
631 617
    if args.command == "all":
632
        actions = get_actions("prepare", "synnefo", "backend")
618
        actions = get_actions("all")
633 619
        fabcommand(args, env, actions)
634 620

  
635
    if args.command == "add":
636
        if args.cluster_node:
637
            add_node(args, env)
638
        else:
639
            actions = get_actions("backend")
640
            fabcommand(args, env, actions)
641

  
642 621
    if args.command == "run":
643 622
        if not args.actions:
644 623
            print_available_actions(args.command)

Also available in: Unified diff