Synnefo

cluster_nodes =

[passwords]

node1 = 12345

# node2 = 67890

# comma separated node names e.g. node1,node2

nfs = node1

dnsutils =

ganeti2 = wheezy

snf-vncauthproxy = wheezy

        sed -i 's/12345/{0}/' /etc/snf-deploy/nodes.conf

Copyright (C) 2010, 2011, 2012, 2013 GRNET S.A. All rights reserved.

Redistribution and use in source and binary forms, with or

without modification, are permitted provided that the following

conditions are met:

  1. Redistributions of source code must retain the above

     copyright notice, this list of conditions and the following

     disclaimer.

  2. Redistributions in binary form must reproduce the above

     copyright notice, this list of conditions and the following

     disclaimer in the documentation and/or other materials

     provided with the distribution.

THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS

OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED

WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR

PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A. OR

CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,

SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT

LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF

USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED

AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT

LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN

ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE

POSSIBILITY OF SUCH DAMAGE.

The views and conclusions contained in the software and

documentation are those of the authors and should not be

interpreted as representing official policies, either expressed

or implied, of GRNET S.A.

cluster_nodes =

synnefo_public_network_subnet = 10.2.1.0/24

synnefo_public_network_gateway = 10.2.1.1

node1 = wheezy

[passwords]

node1 = 12345

# node2 = 67890

# comma separated node names e.g. node1,node2

nfs = node1

dnsutils =

ganeti2 = wheezy

include "/etc/bind/ddns.key";

// all synnefo components share the same domain/zone

        notify no;

        allow-update { key DDNS_UPDATE; };

};

# domain/zone for the VMs

zone "vm.%DOMAIN%" in {

        type master;

        notify no;

        file "/etc/bind/zones/vm.%DOMAIN%";

        allow-update { key DDNS_UPDATE; };

// reverse dns zone for all IPs

        notify no;

        allow-update { key DDNS_UPDATE; };

// v6 reverse dns zone for all IPs

zone "ip6.arpa" in {

        type master;

        notify no;

        file "/etc/bind/rev/synnefo.ip6.arpa.zone";

        allow-update { key DDNS_UPDATE; };

};

$TTL 86400

$ORIGIN ip6.arpa.

@       IN      SOA     ns.%DOMAIN%. admin.%DOMAIN%. (

                2012070900; the Serial Number

                172800; the Refresh Rate

                7200;  the Retry Time

                604800; the Expiration Time

                3600) ; the Minimum Time

@                       IN      NS      ns.%DOMAIN%.

$ORIGIN .

$TTL 86400      ; 1 day

ip6.arpa                IN SOA  ns.vm.qa.live. admin.vm.qa.live. (

                                2012071070 ; serial

                                172800     ; refresh (2 days)

                                7200       ; retry (2 hours)

                                604800     ; expire (1 week)

                                3600       ; minimum (1 hour)

                                )

                        NS      ns.vm.qa.live.

$TTL 14400

$origin vm.%DOMAIN%.

@              IN      SOA     ns.vm.%DOMAIN%. admin.vm.%DOMAIN%. (

2012111903; the Serial Number

172800; the Refresh Rate

7200;  the Retry Time

604800; the Expiration Time

3600; the Minimum Time

)

@               IN      NS      ns.vm.%DOMAIN%.

@               IN      A       %NS_NODE_IP%

ns              IN      A       %NS_NODE_IP%

IMAGE_DIR=%IMAGE_DIR%

HELPER_SOFT_TIMEOUT=100

PITHOS_DB=postgresql://%SYNNEFO_USER%:%SYNNEFO_DB_PASSWD%@%DB_NODE%:5432/snf_pithos

PITHOS_DATA=%PITHOS_DIR%/data

PROGRESS_MONITOR=snf-progress-monitor

MAC_MASK=ff:ff:f0:00:00:00

TAP_CONSTANT_MAC=cc:47:52:4e:45:54 # GRNET in hex :-)

MAC2EUI64=/usr/bin/mac2eui64

NFDHCPD_STATE_DIR=/var/lib/nfdhcpd

GANETI_NIC_DIR=/var/run/ganeti/xen-hypervisor/nic

MAC_FILTERED_TAG=private-filtered

NFDHCPD_TAG=nfdhcpd

IP_LESS_ROUTED_TAG=ip-less-routed

MASQ_TAG=masq

PUBLIC_TAG=public

DNS_TAG=public

# Default options for runlocked helper script (uncomment to modify)

#RUNLOCKED_OPTS="--id 10001 --retry-sec 0.5"

# NS options needed by nsupdate

# A proper bind configuration is a prerequisite

# Please see: https://wiki.debian.org/DDNS

# If one of the following vars are not set dnshook wont do a thing

# Name server IP/FQDN

SERVER=%SERVER%

# zone for the vms

FZONE=vm.%DOMAIN%

# keyfile path to pass to nsupdate with -k option

# see man page for more info

KEYFILE=%KEYFILE%

/srv/ganeti/file-storage

/srv/ganeti/shared-file-storage

   '--workers=6',

#!/bin/bash

brctl addbr %COMMON_BRIDGE%

ip link set %COMMON_BRIDGE% up

iptables -t mangle -A PREROUTING -i %COMMON_BRIDGE% -p udp -m udp --dport 67 -j NFQUEUE --queue-num 42

if [ %ROUTER_IP% == %NODE_IP% ]; then

  iptables -t nat -A POSTROUTING -o %PUBLIC_IFACE% -s %SUBNET% -j MASQUERADE

  echo 1 > /proc/sys/net/ipv4/ip_forward

  ip addr add %GATEWAY% dev %COMMON_BRIDGE%

  ip route add %SUBNET% dev %COMMON_BRIDGE% src %GATEWAY%

fi

exit 0

# This has been generated automatically by snf-deploy, at %DATE%

# The immutable bit (+i attribute) has been used to avoid it being

# overwritten by software such as NetworkManager or resolvconf.

# Use lsattr/chattr to view or modify its file attributes.

DEFAULT_BRIDGE = '%COMMON_BRIDGE%'

            "_cached_url": "/home/", 

            "override_url": "/home/", 

            "slug": "synnefo", 

            "title": "Synnefo", 

            "text": "Welcome to Synnefo!!\r\n\r\n"

            "domain": "%DOMAIN%",

            "name": "%DOMAIN%"

# Copyright (C) 2010, 2011, 2012, 2013 GRNET S.A. All rights reserved.

#

# Redistribution and use in source and binary forms, with or

# without modification, are permitted provided that the following

# conditions are met:

#

#   1. Redistributions of source code must retain the above

#      copyright notice, this list of conditions and the following

#      disclaimer.

#

#   2. Redistributions in binary form must reproduce the above

#      copyright notice, this list of conditions and the following

#      disclaimer in the documentation and/or other materials

#      provided with the distribution.

#

# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS

# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED

# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR

# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A. OR

# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,

# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT

# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF

# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED

# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT

# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN

# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE

# POSSIBILITY OF SUCH DAMAGE.

#

# The views and conclusions contained in the software and

# documentation are those of the authors and should not be

# interpreted as representing official policies, either expressed

# or implied, of GRNET S.A.

import glob

    random_mac, Conf, Env, Status

# from snfdeploy import fabfile

from snfdeploy import fabfile2 as fabfile

Usage: snf-deploy backend

    Role setup:

      setup_ns_role

      setup_nfs_role

      setup_db_role

      setup_mq_role

      setup_astakos_role

      setup_pithos_role

      setup_cyclades_role

      setup_cms_role

      setup_ganeti_role

      setup_master_role

      setup_stats_role

      setup_client_role

    Helper commands:

      update_env_with_user_info

      update_env_with_service_info

      update_env_with_backend_info

    Admin commands:

      update_ns_for_node

      update_exports_for_node

      allow_db_access

      add_ganeti_backend

      add_synnefo_user

      activate_user

      set_default_quota

      add_public_networks

      add_image

    Custom command:

      setup --node NODE [--role ROLE | --method METHOD --component COMPONENT]

    #FIXME: Create a clean wheezy image and use it for vcluster

    fabfile.setup_env(args, env)

            if nodes:

                execute(fn, hosts=ips)

            else:

                execute(fn)

                        help="Force things (creation of key pairs"

                             " do not abort execution if something fails")

    # options related to custom setup

    parser.add_argument("--component", dest="component",

                        default=None,

                        help="The component class")

    parser.add_argument("--method", dest="method",

                        default=None,

                        help="The component method")

    parser.add_argument("--role", dest="role",

                        default=None,

                        help="The target node's role")

    parser.add_argument("--node", dest="node",

                        default="node1",

                        help="The target node")

                        choices=["packages", "vcluster", "cleanup",

                                 "run", "test", "all", "keygen"],

            "setup_master_role",

            "setup_ganeti_role",

            "add_ganeti_backend",

            "setup_ns_role",

            "setup_nfs_role",

            "setup_master_role",

            "setup_ganeti_role",

        ],

        "all": [

            "setup_ns_role",

            "setup_nfs_role",

            "setup_db_role",

            "setup_mq_role",

            "setup_astakos_role",

            "setup_pithos_role",

            "setup_cyclades_role",

            "setup_cms_role",

            "setup_master_role",

            "setup_ganeti_role",

            "setup_stats_role",

            "set_default_quota",

            "add_ganeti_backend",

            "add_public_networks",

            "add_synnefo_user",

            "activate_user",

            "setup_client_role",

            "add_image",

def must_create_keys(env):

    """Check if we ssh keys already exist

def must_create_ddns_keys(env):

    d = os.path.join(env.templates, "root/ddns")

    key_exists = glob.glob(os.path.join(d, "Kddns*key"))

    private_exists = glob.glob(os.path.join(d, "Kddns*private"))

    bind_key_exists = os.path.exists(os.path.join(d, "ddns.key"))

    return not (key_exists and private_exists and bind_key_exists)

def find_ddns_key_files(env):

    d = os.path.join(env.templates, "root/ddns")

    keys = glob.glob(os.path.join(d, "Kddns*"))

    # Here we must have a key!

    return map(os.path.basename, keys)

def do_create_ddns_keys(args, env):

    d = os.path.join(env.templates, "root/ddns")

    if not os.path.exists(d):

        os.mkdir(d)

    for filename in os.listdir(d):

        os.remove(os.path.join(d, filename))

    cmd = """

dnssec-keygen -a HMAC-MD5 -b 128 -K {0} -r /dev/urandom -n USER DDNS_UPDATE

key=$(cat {0}/Kddns_update*.key | awk '{{ print $7 }}')

cat > {0}/ddns.key <<EOF

key DDNS_UPDATE {{

        algorithm HMAC-MD5.SIG-ALG.REG.INT;

        secret "$key";

}};

EOF

""".format(d)

    os.system(cmd)

    env.status = Status(args)

        if not args.force:

            if not must_create_keys(env) or not must_create_ddns_keys(env):

                print "Keys already exist.."

                print "To override existing ones use --force."

                return 1

        do_create_keys(args, env)

        do_create_ddns_keys(args, env)

        return 0

        if ((args.key_inject and not args.ssh_key and must_create_keys(env)) or

            must_create_ddns_keys(env)):

            print "No ssh/ddns keys to use. Run `snf-deploy keygen' first."

        env.ddns_keys = find_ddns_key_files(env)

        env.ddns_private_key = "/root/ddns/" + env.ddns_keys[0]

        actions = get_actions("all")

# Copyright (C) 2010, 2011, 2012, 2013 GRNET S.A. All rights reserved.

#

# Redistribution and use in source and binary forms, with or

# without modification, are permitted provided that the following

# conditions are met:

#

#   1. Redistributions of source code must retain the above

#      copyright notice, this list of conditions and the following

#      disclaimer.

#

#   2. Redistributions in binary form must reproduce the above

#      copyright notice, this list of conditions and the following

#      disclaimer in the documentation and/or other materials

#      provided with the distribution.

#

# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS

# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED

# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR

# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A. OR

# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,

# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT

# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF

# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED

# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT

# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN

# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE

# POSSIBILITY OF SUCH DAMAGE.

#

# The views and conclusions contained in the software and

# documentation are those of the authors and should not be

# interpreted as representing official policies, either expressed

# or implied, of GRNET S.A.

import datetime

from snfdeploy.utils import debug

class SynnefoComponent(object):

    REQUIRED_PACKAGES = []

    def debug(self, msg, info=""):

        debug(self.__class__.__name__, msg, info)

    def __init__(self, node_info, env, *args, **kwargs):

        """ Take a node_info and env as argument and initialize local vars """

        self.node_info = node_info

        self.env = env

    def check(self):

        """ Returns a list of bash commands that check prerequisites """

        return []

    def install(self):

        """ Returns a list of debian packages to install """

        return self.REQUIRED_PACKAGES

    def prepare(self):

        """ Returs a list of bash commands that prepares the component """

        return []

    def configure(self):

        """ Must return a list of tuples (tmpl_path, replace_dict, mode) """

        return []

    def initialize(self):

        """ Returs a list of bash commands that initialize the component """

        return []

    def test(self):

        """ Returs a list of bash commands that test existing installation """

        return []

    def restart(self):

        return []

    #TODO: add cleanup method for each component

    def clean(self):

        return []

class HW(SynnefoComponent):

    def test(self):

        return [

            "ping -c 1 %s" % self.node_info.ip,

            "ping -c 1 www.google.com",

            "apt-get update",

            ]

class SSH(SynnefoComponent):

    def prepare(self):

        return [

            "mkdir -p /root/.ssh",

            "for f in $(ls /root/.ssh/*); do cp $f $f.bak ; done",

            "echo StrictHostKeyChecking no >> /etc/ssh/ssh_config",

            ]

    def configure(self):

        files = [

            "authorized_keys", "id_dsa", "id_dsa.pub", "id_rsa", "id_rsa.pub"

            ]

        ssh = [("/root/.ssh/%s" % f, {}, {"mode":0600}) for f in files]

        return ssh

    def initialize(self):

        f = "/root/.ssh/authorized_keys"

        return [

            "test -e {0}.bak && cat {0}.bak >> {0}".format(f)

            ]

    def test(self):

        return ["ssh %s date" % self.node_info.ip]

class DNS(SynnefoComponent):

    def prepare(self):

        return [

            "chattr -i /etc/resolv.conf",

            "sed -i 's/^127.*$/127.0.0.1 localhost/g' /etc/hosts",

            ]

    def configure(self):

        r1 = {

            "date": str(datetime.datetime.today()),

            "domain": self.env.env.domain,

            "ns_node_ip": self.env.env.ns.ip,

            }

        resolv = [

            ("/etc/resolv.conf", r1, {})

            ]

        return resolv

    def initialize(self):

        return ["chattr +i /etc/resolv.conf"]

class DDNS(SynnefoComponent):

    REQUIRED_PACKAGES = [

        "dnsutils",

        ]

    def prepare(self):

        return [

            "mkdir -p /root/ddns/"

            ]

    def configure(self):

        return [

            ("/root/ddns/" + k, {}, {}) for k in self.env.env.ddns_keys

            ]

class NS(SynnefoComponent):

    REQUIRED_PACKAGES = [

        "bind9",

        ]

    def nsupdate(self, cmd):

        ret = """

nsupdate -k {0} > /dev/null <<EOF || true

server {1}

{2}

send

EOF

""".format(self.env.env.ddns_private_key, self.node_info.ip, cmd)

        return ret

    def prepare(self):

        return [

            "mkdir -p /etc/bind/zones",

            "chmod g+w /etc/bind/zones",

            "mkdir -p /etc/bind/rev",

            "chmod g+w /etc/bind/rev",

            ]

    def configure(self):

        d = self.env.env.domain

        ip = self.node_info.ip

        return [

            ("/etc/bind/named.conf.local", {"domain": d}, {}),

            ("/etc/bind/zones/example.com",

             {"domain": d, "ns_node_ip": ip},

             {"remote": "/etc/bind/zones/%s" % d}),

            ("/etc/bind/zones/vm.example.com",

             {"domain": d, "ns_node_ip": ip},

             {"remote": "/etc/bind/zones/vm.%s" % d}),

            ("/etc/bind/rev/synnefo.in-addr.arpa.zone", {"domain": d}, {}),

            ("/etc/bind/rev/synnefo.ip6.arpa.zone", {"domain": d}, {}),

            ("/etc/bind/named.conf.options",

             {"node_ips": ";".join(self.env.env.ips)}, {}),

            ("/root/ddns/ddns.key", {}, {"remote": "/etc/bind/ddns.key"}),

            ]

    def update_cnamerecord(self, node_info):

        return self.nsupdate("update add %s" % node_info.cnamerecord)

    def update_arecord(self, node_info):

        return self.nsupdate("update add %s" % node_info.arecord)

    def update_ptrrecord(self, node_info):

        return self.nsupdate("update add %s" % node_info.ptrrecord)

    def update_ns_for_node(self, node_info):

        return [

            self.update_arecord(node_info),

            self.update_cnamerecord(node_info),

            self.update_ptrrecord(node_info)

            ]

    def initialize(self):

        a = [self.update_arecord(n)

             for n in self.env.env.nodes_info.values()]

        ptr = [self.update_ptrrecord(n)

               for n in self.env.env.nodes_info.values()]

        cnames = [self.update_cnamerecord(n)

                  for n in self.env.env.roles_info.values()]

        return a + ptr + cnames

    def restart(self):

        return ["/etc/init.d/bind9 restart"]

    def test(self):

        n = ["host %s localhost" % i.fqdn

             for i in self.env.env.nodes_info.values()]

        a = ["host %s localhost" % i.fqdn

             for i in self.env.env.roles_info.values()]

        return n + a

class APT(SynnefoComponent):

    """ Setup apt repos and check fqdns """

    REQUIRED_PACKAGES = ["curl"]

    def prepare(self):

        return [

            "echo 'APT::Install-Suggests \"false\";' >> /etc/apt/apt.conf",

            "curl -k https://dev.grnet.gr/files/apt-grnetdev.pub | apt-key add -",

            ]

    def configure(self):

        return [

            ("/etc/apt/sources.list.d/synnefo.wheezy.list", {}, {})

            ]

    def initialize(self):

        return [

            "apt-get update",

            ]

class MQ(SynnefoComponent):

    REQUIRED_PACKAGES = ["rabbitmq-server"]

    def check(self):

        return ["ping -c 1 mq.%s" % self.env.env.domain]

    def initialize(self):

        u = self.env.env.synnefo_user

        p = self.env.env.synnefo_rabbitmq_passwd

        return [

            "rabbitmqctl add_user %s %s" % (u, p),

            "rabbitmqctl set_permissions %s \".*\" \".*\" \".*\"" % u,

            "rabbitmqctl delete_user guest",

            "rabbitmqctl set_user_tags %s administrator" % u,

            ]

class DB(SynnefoComponent):

    REQUIRED_PACKAGES = ["postgresql"]

    def check(self):

        return ["ping -c 1 db.%s" % self.env.env.domain]

    def get_user_info_from_db(self):

        cmd = """

cat > /tmp/psqlcmd <<EOF

select id, auth_token, uuid, email from auth_user, im_astakosuser \

where auth_user.id = im_astakosuser.user_ptr_id and auth_user.email = '{0}';

EOF

su - postgres -c  "psql -w -d snf_apps -f /tmp/psqlcmd"

""".format(self.env.env.user_email)

        return [cmd]

    def allow_access_in_db(self, node_info, user="all", method="md5"):

        f = "/etc/postgresql/*/main/pg_hba.conf"

        cmd1 = "echo host all %s %s/32 %s >> %s" % \

                (user, node_info.ip, method, f)

        cmd2 = "sed -i 's/\(host.*127.0.0.1.*\)md5/\\1trust/' %s" % f

        return [cmd1, cmd2] + self.restart()

    def configure(self):

        u = self.env.env.synnefo_user

        p = self.env.env.synnefo_db_passwd

        replace = {"synnefo_user": u, "synnefo_db_passwd": p}

        return [