Revision 3c3bccab snf-deploy/snfdeploy/__init__.py
b/snf-deploy/snfdeploy/__init__.py | ||
---|---|---|
1 |
# Copyright (C) 2010, 2011, 2012, 2013 GRNET S.A. All rights reserved. |
|
2 |
# |
|
3 |
# Redistribution and use in source and binary forms, with or |
|
4 |
# without modification, are permitted provided that the following |
|
5 |
# conditions are met: |
|
6 |
# |
|
7 |
# 1. Redistributions of source code must retain the above |
|
8 |
# copyright notice, this list of conditions and the following |
|
9 |
# disclaimer. |
|
10 |
# |
|
11 |
# 2. Redistributions in binary form must reproduce the above |
|
12 |
# copyright notice, this list of conditions and the following |
|
13 |
# disclaimer in the documentation and/or other materials |
|
14 |
# provided with the distribution. |
|
15 |
# |
|
16 |
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS |
|
17 |
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED |
|
18 |
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR |
|
19 |
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A. OR |
|
20 |
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, |
|
21 |
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT |
|
22 |
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF |
|
23 |
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED |
|
24 |
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT |
|
25 |
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN |
|
26 |
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE |
|
27 |
# POSSIBILITY OF SUCH DAMAGE. |
|
28 |
# |
|
29 |
# The views and conclusions contained in the software and |
|
30 |
# documentation are those of the authors and should not be |
|
31 |
# interpreted as representing official policies, either expressed |
|
32 |
# or implied, of GRNET S.A. |
|
33 |
|
|
1 | 34 |
import time |
2 | 35 |
import os |
3 | 36 |
import argparse |
... | ... | |
5 | 38 |
import re |
6 | 39 |
import random |
7 | 40 |
import ast |
41 |
import glob |
|
8 | 42 |
from snfdeploy.lib import check_pidfile, create_dir, get_default_route, \ |
9 |
random_mac, Conf, Env |
|
10 |
from snfdeploy import fabfile |
|
43 |
random_mac, Conf, Env, Status |
|
44 |
# from snfdeploy import fabfile |
|
45 |
from snfdeploy import fabfile2 as fabfile |
|
11 | 46 |
from fabric.api import hide, settings, execute, show |
12 | 47 |
|
13 | 48 |
|
... | ... | |
35 | 70 |
|
36 | 71 |
""" |
37 | 72 |
|
38 |
if command == "prepare": |
|
39 |
print """ |
|
40 |
Usage: snf-deploy prepare |
|
41 |
|
|
42 |
Run the following actions concerning deployment preparation: |
|
43 |
|
|
44 |
- Setup an internal Domain Name Server |
|
45 |
- Tweak hosts and add ssh keys |
|
46 |
- Check network setup |
|
47 |
- Setup apt repository and apt-get update |
|
48 |
- Setup the nfs server and clients among all nodes |
|
49 |
|
|
50 |
""" |
|
51 |
|
|
52 | 73 |
if command == "backend": |
53 | 74 |
print """ |
54 |
Usage: snf-deploy backend [update]
|
|
75 |
Usage: snf-deploy backend |
|
55 | 76 |
|
56 | 77 |
Run the following actions concerning a ganeti backend: |
57 | 78 |
|
58 | 79 |
- Create and add a backend to cyclades |
59 |
- Does all the net-infra specific actions in backend nodes |
|
60 |
(create/connect bridges, iptables..) |
|
61 |
- Does all the storage-infra specific actions in backend nodes |
|
62 |
depending on the --extra-disk option \ |
|
63 |
(create VG, enable lvm/drbd storage..) |
|
64 |
|
|
65 |
or |
|
66 |
|
|
67 |
- Update packages in an already registered backend in cyclades. |
|
68 | 80 |
|
69 | 81 |
""" |
70 | 82 |
|
... | ... | |
74 | 86 |
|
75 | 87 |
Run any of the following fabric commands: |
76 | 88 |
|
89 |
Role setup: |
|
90 |
|
|
91 |
setup_ns_role |
|
92 |
setup_nfs_role |
|
93 |
setup_db_role |
|
94 |
setup_mq_role |
|
95 |
setup_astakos_role |
|
96 |
setup_pithos_role |
|
97 |
setup_cyclades_role |
|
98 |
setup_cms_role |
|
99 |
setup_ganeti_role |
|
100 |
setup_master_role |
|
101 |
setup_stats_role |
|
102 |
setup_client_role |
|
103 |
|
|
104 |
Helper commands: |
|
105 |
|
|
106 |
update_env_with_user_info |
|
107 |
update_env_with_service_info |
|
108 |
update_env_with_backend_info |
|
77 | 109 |
|
78 |
Setup commands: Init commands: Admin commands: |
|
79 |
setup_apache add_pools activate_user |
|
80 |
setup_apt add_rapi_user add_backend |
|
81 |
setup_astakos add_nodes add_image_locally |
|
82 |
setup_cms astakos_loaddata add_network |
|
83 |
setup_collectd |
|
84 |
setup_common astakos_register_components add_ns |
|
85 |
setup_cyclades cms_loaddata add_user |
|
86 |
setup_db cyclades_loaddata connect_bridges |
|
87 |
setup_ganeti enable_drbd create_bridges |
|
88 |
setup_ganeti_collectd |
|
89 |
setup_gtools init_cluster create_vlans |
|
90 |
setup_gunicorn setup_nfs_clients destroy_db |
|
91 |
setup_hosts setup_nfs_server \ |
|
92 |
get_auth_token_from_db |
|
93 |
setup_image_helper update_ns_for_ganeti get_service_details |
|
94 |
setup_image_host astakos_register_pithos_view gnt_instance_add |
|
95 |
setup_iptables gnt_network_add |
|
96 |
setup_kamaki Test commands: register_image |
|
97 |
setup_lvm test restart_services |
|
98 |
setup_mq setup_drbd_dparams |
|
99 |
setup_net_infra |
|
100 |
setup_network |
|
101 |
setup_ns |
|
102 |
setup_pithos |
|
103 |
setup_pithos_dir |
|
104 |
setup_router |
|
105 |
setup_stats |
|
106 |
setup_stats_collectd |
|
107 |
setup_vncauthproxy |
|
108 |
setup_webproject |
|
110 |
Admin commands: |
|
111 |
|
|
112 |
update_ns_for_node |
|
113 |
update_exports_for_node |
|
114 |
allow_db_access |
|
115 |
add_ganeti_backend |
|
116 |
add_synnefo_user |
|
117 |
activate_user |
|
118 |
set_default_quota |
|
119 |
add_public_networks |
|
120 |
add_image |
|
121 |
|
|
122 |
|
|
123 |
Custom command: |
|
124 |
|
|
125 |
setup --node NODE [--role ROLE | --method METHOD --component COMPONENT] |
|
109 | 126 |
|
110 | 127 |
""" |
111 | 128 |
|
... | ... | |
216 | 233 |
|
217 | 234 |
|
218 | 235 |
def image(args, env): |
236 |
#FIXME: Create a clean wheezy image and use it for vcluster |
|
219 | 237 |
if env.os == "ubuntu": |
220 | 238 |
url = env.ubuntu_image_url |
221 | 239 |
else: |
... | ... | |
270 | 288 |
if nodes: |
271 | 289 |
ips = [env.nodes_info[n].ip for n in nodes] |
272 | 290 |
|
273 |
fabfile.setup_env(args) |
|
291 |
fabfile.setup_env(args, env)
|
|
274 | 292 |
with settings(hide(*lhide), show(*lshow)): |
275 | 293 |
print " ".join(actions) |
276 | 294 |
for a in actions: |
277 | 295 |
fn = getattr(fabfile, a) |
278 |
if not args.dry_run: |
|
279 |
if nodes: |
|
280 |
execute(fn, hosts=ips) |
|
281 |
else: |
|
282 |
execute(fn) |
|
296 |
if nodes: |
|
297 |
execute(fn, hosts=ips) |
|
298 |
else: |
|
299 |
execute(fn) |
|
283 | 300 |
|
284 | 301 |
|
285 | 302 |
def cluster(args, env): |
... | ... | |
379 | 396 |
"console or not") |
380 | 397 |
parser.add_argument("--force", dest="force", |
381 | 398 |
default=False, action="store_true", |
382 |
help="Force the creation of new ssh key pairs") |
|
399 |
help="Force things (creation of key pairs" |
|
400 |
" do not abort execution if something fails") |
|
383 | 401 |
|
384 | 402 |
parser.add_argument("-i", "--ssh-key", dest="ssh_key", |
385 | 403 |
default=None, |
... | ... | |
399 | 417 |
default=None, |
400 | 418 |
help="The node to add to the existing cluster") |
401 | 419 |
|
420 |
# options related to custom setup |
|
421 |
parser.add_argument("--component", dest="component", |
|
422 |
default=None, |
|
423 |
help="The component class") |
|
424 |
|
|
425 |
parser.add_argument("--method", dest="method", |
|
426 |
default=None, |
|
427 |
help="The component method") |
|
428 |
|
|
429 |
parser.add_argument("--role", dest="role", |
|
430 |
default=None, |
|
431 |
help="The target node's role") |
|
432 |
|
|
433 |
parser.add_argument("--node", dest="node", |
|
434 |
default="node1", |
|
435 |
help="The target node") |
|
436 |
|
|
402 | 437 |
# available commands |
403 | 438 |
parser.add_argument("command", type=str, |
404 |
choices=["packages", "vcluster", "prepare", |
|
405 |
"synnefo", "backend", "ganeti", |
|
406 |
"run", "cleanup", "test", |
|
407 |
"all", "add", "keygen"], |
|
439 |
choices=["packages", "vcluster", "cleanup", |
|
440 |
"run", "test", "all", "keygen"], |
|
408 | 441 |
help="Run on of the supported deployment commands") |
409 | 442 |
|
410 | 443 |
# available actions for the run command |
... | ... | |
421 | 454 |
|
422 | 455 |
def get_actions(*args): |
423 | 456 |
actions = { |
424 |
# prepare actions |
|
425 |
"ns": ["setup_ns", "setup_resolv_conf"], |
|
426 |
"hosts": ["setup_hosts", "add_keys"], |
|
427 |
"check": ["check_dhcp", "check_dns", |
|
428 |
"check_connectivity", "check_ssh"], |
|
429 |
"apt": ["apt_get_update", "setup_apt"], |
|
430 |
"nfs": ["setup_nfs_server", "setup_nfs_clients"], |
|
431 |
"prepare": [ |
|
432 |
"setup_hosts", "add_keys", |
|
433 |
"setup_ns", "setup_resolv_conf", |
|
434 |
"check_dhcp", "check_dns", "check_connectivity", "check_ssh", |
|
435 |
"apt_get_update", "setup_apt", |
|
436 |
"setup_nfs_server", "setup_nfs_clients" |
|
437 |
], |
|
438 |
# synnefo actions |
|
439 |
"synnefo": [ |
|
440 |
"setup_mq", "setup_db", |
|
441 |
"setup_astakos", |
|
442 |
#TODO: astakos-quota fails if no user is added. |
|
443 |
# add_user fails if no groups found |
|
444 |
"astakos_loaddata", "add_user", "activate_user", |
|
445 |
"astakos_register_components", |
|
446 |
"astakos_register_pithos_view", |
|
447 |
"setup_cms", "cms_loaddata", |
|
448 |
"setup_pithos", |
|
449 |
"setup_vncauthproxy", |
|
450 |
"setup_cyclades", "cyclades_loaddata", "add_pools", |
|
451 |
"export_services", "import_services", "set_user_quota", |
|
452 |
"setup_kamaki", "upload_image", "register_image", |
|
453 |
"setup_burnin", |
|
454 |
"setup_stats" |
|
455 |
], |
|
456 |
"supdate": [ |
|
457 |
"apt_get_update", "setup_astakos", |
|
458 |
"setup_cms", "setup_pithos", "setup_cyclades" |
|
459 |
], |
|
460 |
# backend actions |
|
461 | 457 |
"backend": [ |
462 |
"setup_hosts", |
|
463 |
"update_ns_for_ganeti", |
|
464 |
"setup_ganeti", "init_cluster", |
|
465 |
"add_rapi_user", "add_nodes", |
|
466 |
"setup_image_host", "setup_image_helper", |
|
467 |
"setup_network", |
|
468 |
"setup_gtools", "add_backend", "add_network", |
|
469 |
"setup_lvm", "enable_lvm", |
|
470 |
"enable_drbd", "setup_drbd_dparams", |
|
471 |
"setup_net_infra", "setup_iptables", "setup_router", |
|
472 |
], |
|
473 |
"bstorage": [ |
|
474 |
"setup_lvm", "enable_lvm", |
|
475 |
"enable_drbd", "setup_drbd_dparams" |
|
458 |
"setup_master_role", |
|
459 |
"setup_ganeti_role", |
|
460 |
"add_ganeti_backend", |
|
476 | 461 |
], |
477 |
"bnetwork": ["setup_net_infra", "setup_iptables", "setup_router"], |
|
478 |
"bupdate": [ |
|
479 |
"apt_get_update", "setup_ganeti", "setup_image_host", |
|
480 |
"setup_image_helper", "setup_network", "setup_gtools" |
|
481 |
], |
|
482 |
# ganeti actions |
|
483 | 462 |
"ganeti": [ |
484 |
"update_ns_for_ganeti", |
|
485 |
"setup_ganeti", "init_cluster", "add_nodes", |
|
486 |
"setup_image_host", "setup_image_helper", "add_image_locally", |
|
487 |
"debootstrap", "setup_net_infra", |
|
488 |
"setup_lvm", "enable_lvm", "enable_drbd", "setup_drbd_dparams", |
|
489 |
"setup_ganeti_collectd" |
|
463 |
"setup_ns_role", |
|
464 |
"setup_nfs_role", |
|
465 |
"setup_master_role", |
|
466 |
"setup_ganeti_role", |
|
467 |
], |
|
468 |
"all": [ |
|
469 |
"setup_ns_role", |
|
470 |
"setup_nfs_role", |
|
471 |
"setup_db_role", |
|
472 |
"setup_mq_role", |
|
473 |
"setup_astakos_role", |
|
474 |
"setup_pithos_role", |
|
475 |
"setup_cyclades_role", |
|
476 |
"setup_cms_role", |
|
477 |
"setup_master_role", |
|
478 |
"setup_ganeti_role", |
|
479 |
"setup_stats_role", |
|
480 |
"set_default_quota", |
|
481 |
"add_ganeti_backend", |
|
482 |
"add_public_networks", |
|
483 |
"add_synnefo_user", |
|
484 |
"activate_user", |
|
485 |
"setup_client_role", |
|
486 |
"add_image", |
|
490 | 487 |
], |
491 |
"gupdate": ["setup_apt", "setup_ganeti"], |
|
492 |
"gdestroy": ["destroy_cluster"], |
|
488 |
|
|
493 | 489 |
} |
494 | 490 |
|
495 | 491 |
ret = [] |
... | ... | |
499 | 495 |
return ret |
500 | 496 |
|
501 | 497 |
|
502 |
def must_create_keys(force, env): |
|
503 |
"""Check if we need to create ssh keys |
|
504 |
|
|
505 |
If force is true we are going to overide the old keys. |
|
506 |
Else if there are already generated keys to use, don't create new ones. |
|
498 |
def must_create_keys(env): |
|
499 |
"""Check if we ssh keys already exist |
|
507 | 500 |
|
508 | 501 |
""" |
509 |
if force: |
|
510 |
return True |
|
511 | 502 |
d = os.path.join(env.templates, "root/.ssh") |
512 | 503 |
auth_keys_exists = os.path.exists(os.path.join(d, "authorized_keys")) |
513 | 504 |
dsa_exists = os.path.exists(os.path.join(d, "id_dsa")) |
... | ... | |
535 | 526 |
os.system(cmd) |
536 | 527 |
|
537 | 528 |
|
538 |
def add_node(args, env): |
|
539 |
actions = [ |
|
540 |
"update_ns_for_node:" + args.cluster_node, |
|
541 |
] |
|
542 |
fabcommand(args, env, actions) |
|
543 |
actions = [ |
|
544 |
"setup_resolv_conf", |
|
545 |
"apt_get_update", |
|
546 |
"setup_apt", |
|
547 |
"setup_hosts", |
|
548 |
"add_keys", |
|
549 |
] |
|
550 |
fabcommand(args, env, actions, [args.cluster_node]) |
|
551 |
|
|
552 |
actions = get_actions("check") |
|
553 |
fabcommand(args, env, actions) |
|
554 |
|
|
555 |
actions = [ |
|
556 |
"setup_nfs_clients", |
|
557 |
"setup_ganeti", |
|
558 |
"setup_image_host", "setup_image_helper", |
|
559 |
"setup_network", "setup_gtools", |
|
560 |
] |
|
561 |
fabcommand(args, env, actions, [args.cluster_node]) |
|
562 |
|
|
563 |
actions = [ |
|
564 |
"add_node:" + args.cluster_node, |
|
565 |
] |
|
566 |
fabcommand(args, env, actions) |
|
567 |
|
|
568 |
actions = [ |
|
569 |
"setup_lvm", "enable_drbd", |
|
570 |
"setup_net_infra", "setup_iptables", |
|
571 |
] |
|
572 |
fabcommand(args, env, actions, [args.cluster_node]) |
|
529 |
def must_create_ddns_keys(env): |
|
530 |
d = os.path.join(env.templates, "root/ddns") |
|
531 |
key_exists = glob.glob(os.path.join(d, "Kddns*key")) |
|
532 |
private_exists = glob.glob(os.path.join(d, "Kddns*private")) |
|
533 |
bind_key_exists = os.path.exists(os.path.join(d, "ddns.key")) |
|
534 |
return not (key_exists and private_exists and bind_key_exists) |
|
535 |
|
|
536 |
|
|
537 |
def find_ddns_key_files(env): |
|
538 |
d = os.path.join(env.templates, "root/ddns") |
|
539 |
keys = glob.glob(os.path.join(d, "Kddns*")) |
|
540 |
# Here we must have a key! |
|
541 |
return map(os.path.basename, keys) |
|
542 |
|
|
543 |
|
|
544 |
def do_create_ddns_keys(args, env): |
|
545 |
d = os.path.join(env.templates, "root/ddns") |
|
546 |
if not os.path.exists(d): |
|
547 |
os.mkdir(d) |
|
548 |
for filename in os.listdir(d): |
|
549 |
os.remove(os.path.join(d, filename)) |
|
550 |
cmd = """ |
|
551 |
dnssec-keygen -a HMAC-MD5 -b 128 -K {0} -r /dev/urandom -n USER DDNS_UPDATE |
|
552 |
key=$(cat {0}/Kddns_update*.key | awk '{{ print $7 }}') |
|
553 |
cat > {0}/ddns.key <<EOF |
|
554 |
key DDNS_UPDATE {{ |
|
555 |
algorithm HMAC-MD5.SIG-ALG.REG.INT; |
|
556 |
secret "$key"; |
|
557 |
}}; |
|
558 |
EOF |
|
559 |
""".format(d) |
|
560 |
os.system(cmd) |
|
561 |
|
|
573 | 562 |
|
574 | 563 |
|
575 | 564 |
def main(): |
... | ... | |
577 | 566 |
|
578 | 567 |
conf = Conf(args) |
579 | 568 |
env = Env(conf) |
569 |
env.status = Status(args) |
|
580 | 570 |
|
581 | 571 |
create_dir(env.run, False) |
582 | 572 |
create_dir(env.dns, False) |
583 | 573 |
|
584 | 574 |
# Check if there are keys to use |
585 | 575 |
if args.command == "keygen": |
586 |
if must_create_keys(args.force, env): |
|
587 |
do_create_keys(args, env) |
|
588 |
return 0 |
|
589 |
else: |
|
590 |
print "Keys already existed.. aborting" |
|
591 |
return 1 |
|
576 |
if not args.force: |
|
577 |
if not must_create_keys(env) or not must_create_ddns_keys(env): |
|
578 |
print "Keys already exist.." |
|
579 |
print "To override existing ones use --force." |
|
580 |
return 1 |
|
581 |
do_create_keys(args, env) |
|
582 |
do_create_ddns_keys(args, env) |
|
583 |
return 0 |
|
592 | 584 |
else: |
593 |
if (args.key_inject and (args.ssh_key is None)
|
|
594 |
and must_create_keys(False, env)):
|
|
595 |
print "No ssh keys to use. Run `snf-deploy keygen' first." |
|
585 |
if ((args.key_inject and not args.ssh_key and must_create_keys(env)) or
|
|
586 |
must_create_ddns_keys(env)):
|
|
587 |
print "No ssh/ddns keys to use. Run `snf-deploy keygen' first."
|
|
596 | 588 |
return 1 |
589 |
env.ddns_keys = find_ddns_key_files(env) |
|
590 |
env.ddns_private_key = "/root/ddns/" + env.ddns_keys[0] |
|
597 | 591 |
|
598 | 592 |
if args.command == "test": |
599 | 593 |
conf.print_config() |
... | ... | |
612 | 606 |
dnsmasq(args, env) |
613 | 607 |
cluster(args, env) |
614 | 608 |
|
615 |
if args.command == "prepare": |
|
616 |
actions = get_actions("prepare") |
|
617 |
fabcommand(args, env, actions) |
|
618 |
|
|
619 |
if args.command == "synnefo": |
|
620 |
actions = get_actions("synnefo") |
|
621 |
fabcommand(args, env, actions) |
|
622 |
|
|
623 | 609 |
if args.command == "backend": |
624 | 610 |
actions = get_actions("backend") |
625 | 611 |
fabcommand(args, env, actions) |
... | ... | |
629 | 615 |
fabcommand(args, env, actions) |
630 | 616 |
|
631 | 617 |
if args.command == "all": |
632 |
actions = get_actions("prepare", "synnefo", "backend")
|
|
618 |
actions = get_actions("all")
|
|
633 | 619 |
fabcommand(args, env, actions) |
634 | 620 |
|
635 |
if args.command == "add": |
|
636 |
if args.cluster_node: |
|
637 |
add_node(args, env) |
|
638 |
else: |
|
639 |
actions = get_actions("backend") |
|
640 |
fabcommand(args, env, actions) |
|
641 |
|
|
642 | 621 |
if args.command == "run": |
643 | 622 |
if not args.actions: |
644 | 623 |
print_available_actions(args.command) |
Also available in: Unified diff