Synnefo

*.pyc

lala.tmp

kamaki-fab

images/*

*.pid

*.monitor

packages

*.swp

include README Changelog

include distribute_setup.py

#!/bin/bash

usage(){

  echo "

Usage: $0: [options]

  -h, --help          Prints this help message

  --debian [branch]   Local debian branch to use (default debian)

  --upstream [branch] Local upstream branch to use (default master)

  --remote [repo]     Remote repo to use (default origin)

  --packages [dir]    Where to store the created packages (default ~/packages)

  --validate          Fetch remote repo branches and

                      check if local are up-to-date (default false)

  --push              Whether to push upstream (default false)

"

  exit 1

}

parse_git_branch()

{

    git branch 2> /dev/null | grep '^*' | sed 's/^*\ //g'

}

die()

{

    echo -e $* 1>&2

    echo Aborting.

    exit 1

}

cleanup()

{

    trap - EXIT

    echo -n Cleaning up...

    if [ ${#CLEANUP[*]} -gt 0 ]; then

        LAST_ELEMENT=$((${#CLEANUP[*]}-1))

        REVERSE_INDEXES=$(seq ${LAST_ELEMENT} -1 0)

        for i in $REVERSE_INDEXES; do

            local cmd=${CLEANUP[$i]}

            $cmd

        done

    fi

    echo "done"

}

add_cleanup() {

    local cmd=""

    for arg; do cmd+=$(printf "%q " "$arg"); done

    CLEANUP+=("$cmd")

}

add_checkpoint()

{

    commit=$(git reflog | head -n1 | cut -f 1 -d " ")

    add_cleanup git reset --hard $commit

    LASTCHECKPOINT=$commit

}

CLEANUP=( )

TEMP=$(getopt -o h --long help,validate,push,packages:,upstream:,debian:,remote: -n 'autopkg.sh' -- "$@")

if [ $? != 0 ] ; then echo "Terminating..." >&2 ; exit 1 ; fi

eval set -- "$TEMP"

while true ; do

  case "$1" in

    -h|--help) usage ;;

    --upstream) LOCALUPSTREAM=$2 ; shift 2 ;;

    --debian) LOCALDEBIAN=$2 ; shift 2 ;;

    --remote) REMOTE=$2 ; shift 2 ;;

    --packages) PKGAREA=$2 ; shift 2 ;;

    --validate) VALIDATE=true ; shift ;;

    --push) PUSH=true ; shift ;;

    --) shift ; break ;;

    *) echo "Internal error!" ; usage ;;

  esac

done

# The root of the git repository, no matter where we're called from

TOPLEVEL="$(git rev-parse --show-toplevel)"

: ${LOCALUPSTREAM:=$(parse_git_branch)}

: ${LOCALDEBIAN:=debian}

: ${REMOTE:=origin}

: ${VALIDATE:=false}

: ${PUSH:=false}

: ${PKGAREA:=~/packages}

: ${BACKUPAREA:=~/backup}

cd "$TOPLEVEL"

# Prerequisites: Test all important directories exist

test -d "$PKGAREA" || die "Package area directory $PKGAREA missing"

test -d "$BACKUPAREA" || die "Backup area directory $BACKUPAREA missing"

# Prerequisite: Test the dialog utility is available

dialog --help &>/dev/null || die "Could not run the 'dialog' utility"

BUILDAREA=$(mktemp -d --tmpdir=/tmp build-area.XXX)

add_cleanup rm -r $BUILDAREA

echo "############################################################################"

echo "Will build packages under $BUILDAREA"

echo "Local upstream branch: $LOCALUPSTREAM"

echo "Local debian branch: $LOCALDEBIAN"

$VALIDATE && echo "Will fetch $REMOTE and check if $LOCALUPSTREAM and $LOCALDEBIAN are up-to-date"

echo "############################################################################"

echo "Press Enter to continue..."

read

MODIFIED=$(git status --short | grep -v "??")

test -z "$MODIFIED" || die "error: Repository is dirty. Commit your local changes:\n $MODIFIED"

set -e

trap cleanup EXIT

add_checkpoint

# Create a temporary debian branch to do everything

TMPDEBIAN=$(mktemp -u debian.XXX)

git branch --track $TMPDEBIAN  $LOCALDEBIAN

add_cleanup git branch -D $TMPDEBIAN

git checkout $TMPDEBIAN

add_cleanup git checkout $LOCALUPSTREAM

# Whether we are in snapshot or release mode

snap=false

mrgextra=-m

dchextra=-R

mrgmsg="Merge branch '$LOCALUPSTREAM' into $LOCALDEBIAN"

dialog --yesno "Create Snapshot?" 5 20 && snap=true  && dchextra=-S && mrgextra= && mrgmsg=

# merge local branch into tmp branch with a nice commit message,

# so it can be pushed as is to upstream debian

export GIT_MERGE_AUTOEDIT=no

git merge $mrgextra ${mrgextra:+"$mrgmsg"} $LOCALUPSTREAM

# auto edit Debian changelog depending on Snapshot or Release mode

export EDITOR=/usr/bin/vim

git-dch --debian-branch=$TMPDEBIAN --git-author --ignore-regex=".*" --multimaint-merge --since=HEAD $dchextra

git add debian/changelog

# get version from the changelog

# we add a git tag here, so setup.py sdist works as expected

# FIXME: This is a workaround for the way Synnefo packages determine

#        the versions for their Python packages

version=$(IFS="()" ; read  x v x < debian/changelog  ; echo $v)

if ! $snap; then

  git commit -s -a -m "Bump new upstream version"

  TAGFILE=$(mktemp -t tag.XXX)

  add_cleanup rm $TAGFILE

  dialog --inputbox "New Debian Tag: " 5 30 "debian/$version" 2>$TAGFILE

  git tag $(<$TAGFILE)

  add_cleanup git tag -d $(<$TAGFILE)

fi

add_cleanup git reset --hard HEAD

# Build all packages

git-buildpackage --git-export-dir="$BUILDAREA" \

                 --git-upstream-branch=$LOCALUPSTREAM \

                 --git-debian-branch=$TMPDEBIAN \

                 --git-export=INDEX \

                 --git-ignore-new -sa

# do some dirty backup

# pkgarea might be needed by auto-deploy tool

rm -f "$PKGAREA"/* || true

cp -v "$BUILDAREA"/* "$PKGAREA"/ || true

cp -v "$BUILDAREA"/* "$BACKUPAREA"/ || true

function check_remote(){

  git fetch $1 2>/dev/null || die "Could not fetch $1"

  git fetch $1 $2 2>/dev/null|| die "Could not fetch $1/$2"

  commits_behind=$(git rev-list $2..$1/$2 | wc -l)

  if [ $commits_behind -ne 0 ]; then

    die "Your local branch is outdated. Please run:\ngit pull --rebase $1/$2"

  fi

}

if $VALIDATE; then

  check_remote $REMOTE $LOCALUPSTREAM

  check_remote $REMOTE $LOCALDEBIAN

fi

  # trap - EXIT

  # here we can push the commits to the remote debian branch as they are

echo

echo "#################################################"

echo "##                  SUCCESS                    ##"

echo "#################################################"

if $PUSH; then

  git push --tags $REMOTE $TMPDEBIAN:$LOCALDEBIAN

  git push $REMOTE $LOCALUPSTREAM:$LOCALUPSTREAM

fi

exit 0

[packages]

# whether to use apt-get or local generated package found in packages dir

use_local_packages = True

# url to obtain latest synnefo packages.

# To use them change USE_LOCAL_PACKAGES setting to yes

# To get them run: snf-deploy packages

package_url = http://builder.dev.grnet.gr/synnefo/packages/Squeeze/40/

[dirs]

# dir to find all template files used to customize setup

# in case you want to add another setting please modify the corresponding file

templates = /var/lib/snf-deploy/files

# dir to store local images (disk0, disk1 of the virtual cluster)

images = /var/lib/snf-deploy/images

# dir to store/find local packages

# dir to locally save packages that will be downloaded from package_url

# put here any locally created packages (useful for development)

packages = /var/lib/snf-deploy/packages

# dir to store pidfiles (dnsmasq, kvm)

run = /var/run/snf-deploy

# dir to store dnsmasq related files

dns = /var/lib/snf-deploy/dnsmasq

# dir to lookup fabfile and ifup script

lib = /usr/lib/snf-deploy

# dir to store executed commands (to enforce sequential execution)

cmd = /var/run/snf-deploy/cmd

[ganeti1]

cluster_nodes = node1

master_node = node1

cluster_netdev = eth0

cluster_name = ganeti1

cluster_ip = 192.168.0.13

vg = autovg

synnefo_public_network_subnet = 10.0.1.0/24

synnefo_public_network_gateway = 10.0.1.1

synnefo_public_network_type = CUSTOM

image_dir = /srv/okeanos

# To add another cluster repeat the above section

# with different header and nodes

# please note that currently is only supported deployment

# with nodes (both ganeti and synnefo) residing in the same subnet/domain

[network]

domain = synnefo.live

subnet = 192.168.0.0/28

gateway = 192.168.0.14

[hostnames]

node1 = auto1

# node2 = auto2

[ips]

node1 = 192.168.0.1

# node2 = 192.168.0.2

# This is used only in case of vcluster

# needed to pass the correct dhcp responces to the virtual nodes

[macs]

node1 = 52:54:00:00:00:01

# node2 = 52:54:00:00:00:02

[info]

# Here we define which nodes from the predefined ones to use

nodes = node1

# login credentials for the nodes

# please note that in case of vcluster these are preconfigured

# and not editable.

# in case of physical nodes all nodes should have the same login account

user = root

password = 12345

public_iface = eth0

vm_public_iface = eth1

vm_private_iface = eth2

# extra disk name inside the nodes

# if defined, snf-deploy will create a VG for ganeti in order to support lvm storage

# if not then only file disk template will be supported

extra_disk = /dev/vdb

[debian]

rabbitmq-server = testing

gunicorn = squeeze-backports

qemu-kvm = squeeze-backports

qemu = squeeze-backports

python-gevent = squeeze-backports

apache2 =

postgresql =

python-psycopg2 =

python-argparse =

nfs-kernel-server = squeeze-backports

nfs-common = squeeze-backports

bind9 =

vlan =

vlan =

lvm2 =

curl =

memcached =

python-memcache =

bridge-utils =

python-progress =

ganeti-instance-debootstrap =

[synnefo]

snf-astakos-app = stable

snf-common = stable

snf-cyclades-app = stable

snf-cyclades-gtools = stable

snf-django-lib = stable

python-astakosclient = stable

python-objpool = stable

snf-branding = stable

snf-webproject = stable

snf-pithos-app = stable

snf-pithos-backend = stable

snf-tools = stable

python-django-south = stable

[ganeti]

snf-ganeti = 2.6.2+ippool11+hotplug5+extstorage3+rbdfix1+kvmfix2+nolvm+netxen-1~squeeze

ganeti-htools = 2.6.2+ippool11+hotplug5+extstorage3+rbdfix1+kvmfix2+nolvm+netxen-1~squeeze

[other]

snf-cloudcms = stable

snf-vncauthproxy = stable

snf-pithos-webclient = stable

snf-image = stable

snf-network = stable

nfdhcpd = stable

kamaki = stable

python-bitarray = stable

nfqueue-bindings-python = stable

[cred]

synnefo_user = synnefo

synnefo_db_passwd = example_passw0rd

synnefo_rapi_passwd = example_rapi_passw0rd

synnefo_rabbitmq_passwd = example_rabbitmq_passw0rd

user_email = user@synnefo.org

user_name = John

user_lastname = Doe

user_passwd = 12345

[roles]

accounts = node1

compute = node1

object-store = node1

cyclades = node1

pithos = node1

cms = node1

db = node1

mq = node1

ns = node1

client = node1

router = node1

[synnefo]

pithos_dir = /srv/pithos

vm_public_bridge = br0

vm_private_bridge = prv0

common_bridge = br0

debian_base_url = https://pithos.okeanos.grnet.gr/public/RDISy7sNVIJ9KIm4JkmbX4

[image]

# url to get the base image. This is a debian base image with preconfigured

# root password and installed rsa/dsa keys. Plus a NetworkManager hook that

# changes the VM's name based on info provided by dhcp response.

# To create it run: snf-deploy image

squeeze_image_url = https://pithos.okeanos.grnet.gr/public/832xv

ubuntu_image_url =

# in order ganeti nodes to support lvm storage (plain disk template) it will

# be needed an extra disk to eventually be able to create a VG. Ganeti requires

# this VG to be at least of 30GB. To this end in order the virtual nodes to have

# this extra disk an image should be created locally. There are three options:

# 1. not create an extra disk (only file storage template will be supported)

# 2. create an image of 30G in image dir (default /var/lib/snf-deploy/images)

#    using dd if=/dev/zero of=squeeze.disk1

# 3. create this image in a local VG using lvgreate -L30G squeeze.disk1 lvg

#    and create a symbolic link in /var/lib/snf-deploy/images

# Whether to create an extra disk or not

create_extra_disk = False

# lvg is the name of the local VG if any

lvg =

# OS istalled in the virtual cluster

os = squeeze

[cluster]

# the bridge to use for the virtual cluster

# on this bridge we will launch a dnsnmasq and provide

# fqdns needed to the cluster.

# In ordrer cluster nodes to have internet access, host must do NAT.

# iptables -t nat -A POSTROUTING -s 192.0.0.0/28 -j MASQUERADE

# ip addr add 192.0.0.14/28 dev auto_nodes_br

# To create run: snf-deploy cluster

bridge = auto_nodes_br

#!python

"""Bootstrap distribute installation

If you want to use setuptools in your package's setup.py, just include this

file in the same directory with it, and add this to the top of your setup.py::

    from distribute_setup import use_setuptools

    use_setuptools()

If you want to require a specific version of setuptools, set a download

mirror, or use an alternate download directory, you can do so by supplying

the appropriate options to ``use_setuptools()``.

This file can also be run as a script to install or upgrade setuptools.

"""

import os

import sys

import time

import fnmatch

import tempfile

import tarfile

from distutils import log

try:

    from site import USER_SITE

except ImportError:

    USER_SITE = None

try:

    import subprocess

    def _python_cmd(*args):

        args = (sys.executable,) + args

        return subprocess.call(args) == 0

except ImportError:

    # will be used for python 2.3

    def _python_cmd(*args):

        args = (sys.executable,) + args

        # quoting arguments if windows

        if sys.platform == 'win32':

            def quote(arg):

                if ' ' in arg:

                    return '"%s"' % arg

                return arg

            args = [quote(arg) for arg in args]

        return os.spawnl(os.P_WAIT, sys.executable, *args) == 0

DEFAULT_VERSION = "0.6.10"

DEFAULT_URL = "http://pypi.python.org/packages/source/d/distribute/"

SETUPTOOLS_FAKED_VERSION = "0.6c11"

SETUPTOOLS_PKG_INFO = """\

Metadata-Version: 1.0

Name: setuptools

Version: %s

Summary: xxxx

Home-page: xxx

Author: xxx

Author-email: xxx

License: xxx

Description: xxx

""" % SETUPTOOLS_FAKED_VERSION

def _install(tarball):

    # extracting the tarball

    tmpdir = tempfile.mkdtemp()

    log.warn('Extracting in %s', tmpdir)

    old_wd = os.getcwd()

    try:

        os.chdir(tmpdir)

        tar = tarfile.open(tarball)

        _extractall(tar)

        tar.close()

        # going in the directory

        subdir = os.path.join(tmpdir, os.listdir(tmpdir)[0])

        os.chdir(subdir)

        log.warn('Now working in %s', subdir)

        # installing

        log.warn('Installing Distribute')

        if not _python_cmd('setup.py', 'install'):

            log.warn('Something went wrong during the installation.')

            log.warn('See the error message above.')

    finally:

        os.chdir(old_wd)

def _build_egg(egg, tarball, to_dir):

    # extracting the tarball

    tmpdir = tempfile.mkdtemp()

    log.warn('Extracting in %s', tmpdir)

    old_wd = os.getcwd()

    try:

        os.chdir(tmpdir)

        tar = tarfile.open(tarball)

        _extractall(tar)

        tar.close()

        # going in the directory

        subdir = os.path.join(tmpdir, os.listdir(tmpdir)[0])

        os.chdir(subdir)

        log.warn('Now working in %s', subdir)

        # building an egg

        log.warn('Building a Distribute egg in %s', to_dir)

        _python_cmd('setup.py', '-q', 'bdist_egg', '--dist-dir', to_dir)

    finally:

        os.chdir(old_wd)

    # returning the result

    log.warn(egg)

    if not os.path.exists(egg):

        raise IOError('Could not build the egg.')

def _do_download(version, download_base, to_dir, download_delay):

    egg = os.path.join(to_dir, 'distribute-%s-py%d.%d.egg'

                       % (version, sys.version_info[0], sys.version_info[1]))

    if not os.path.exists(egg):

        tarball = download_setuptools(version, download_base,

                                      to_dir, download_delay)

        _build_egg(egg, tarball, to_dir)

    sys.path.insert(0, egg)

    import setuptools

    setuptools.bootstrap_install_from = egg

def use_setuptools(version=DEFAULT_VERSION, download_base=DEFAULT_URL,

                   to_dir=os.curdir, download_delay=15, no_fake=True):

    # making sure we use the absolute path

    to_dir = os.path.abspath(to_dir)

    was_imported = 'pkg_resources' in sys.modules or \

        'setuptools' in sys.modules

    try:

        try:

            import pkg_resources

            if not hasattr(pkg_resources, '_distribute'):

                if not no_fake:

                    _fake_setuptools()

                raise ImportError

        except ImportError:

            return _do_download(version, download_base, to_dir, download_delay)

        try:

            pkg_resources.require("distribute>="+version)

            return

        except pkg_resources.VersionConflict:

            e = sys.exc_info()[1]

            if was_imported:

                sys.stderr.write(

                "The required version of distribute (>=%s) is not available,\n"

                "and can't be installed while this script is running. Please\n"

                "install a more recent version first, using\n"

                "'easy_install -U distribute'."

                "\n\n(Currently using %r)\n" % (version, e.args[0]))

                sys.exit(2)

            else:

                del pkg_resources, sys.modules['pkg_resources']    # reload ok

                return _do_download(version, download_base, to_dir,

                                    download_delay)

        except pkg_resources.DistributionNotFound:

            return _do_download(version, download_base, to_dir,

                                download_delay)

    finally:

        if not no_fake:

            _create_fake_setuptools_pkg_info(to_dir)

def download_setuptools(version=DEFAULT_VERSION, download_base=DEFAULT_URL,

                        to_dir=os.curdir, delay=15):

    """Download distribute from a specified location and return its filename

    `version` should be a valid distribute version number that is available

    as an egg for download under the `download_base` URL (which should end

    with a '/'). `to_dir` is the directory where the egg will be downloaded.

    `delay` is the number of seconds to pause before an actual download

    attempt.

    """

    # making sure we use the absolute path

    to_dir = os.path.abspath(to_dir)

    try:

        from urllib.request import urlopen

    except ImportError:

        from urllib2 import urlopen

    tgz_name = "distribute-%s.tar.gz" % version

    url = download_base + tgz_name

    saveto = os.path.join(to_dir, tgz_name)

    src = dst = None

    if not os.path.exists(saveto):  # Avoid repeated downloads

        try:

            log.warn("Downloading %s", url)

            src = urlopen(url)

            # Read/write all in one block, so we don't create a corrupt file

            # if the download is interrupted.

            data = src.read()

            dst = open(saveto, "wb")

            dst.write(data)

        finally:

            if src:

                src.close()

            if dst:

                dst.close()

    return os.path.realpath(saveto)

def _no_sandbox(function):

    def __no_sandbox(*args, **kw):

        try:

            from setuptools.sandbox import DirectorySandbox

            if not hasattr(DirectorySandbox, '_old'):

                def violation(*args):

                    pass

                DirectorySandbox._old = DirectorySandbox._violation

                DirectorySandbox._violation = violation

                patched = True

            else:

                patched = False

        except ImportError:

            patched = False

        try:

            return function(*args, **kw)

        finally:

            if patched:

                DirectorySandbox._violation = DirectorySandbox._old

                del DirectorySandbox._old

    return __no_sandbox

def _patch_file(path, content):

    """Will backup the file then patch it"""

    existing_content = open(path).read()

    if existing_content == content:

        # already patched

        log.warn('Already patched.')

        return False

    log.warn('Patching...')

    _rename_path(path)

    f = open(path, 'w')

    try:

        f.write(content)

    finally:

        f.close()

    return True

_patch_file = _no_sandbox(_patch_file)

def _same_content(path, content):

    return open(path).read() == content

def _rename_path(path):

    new_name = path + '.OLD.%s' % time.time()

    log.warn('Renaming %s into %s', path, new_name)

    os.rename(path, new_name)

    return new_name

def _remove_flat_installation(placeholder):

    if not os.path.isdir(placeholder):

        log.warn('Unkown installation at %s', placeholder)

        return False

    found = False

    for file in os.listdir(placeholder):

        if fnmatch.fnmatch(file, 'setuptools*.egg-info'):

            found = True

            break

    if not found:

        log.warn('Could not locate setuptools*.egg-info')

        return

    log.warn('Removing elements out of the way...')

    pkg_info = os.path.join(placeholder, file)

    if os.path.isdir(pkg_info):

        patched = _patch_egg_dir(pkg_info)

    else:

        patched = _patch_file(pkg_info, SETUPTOOLS_PKG_INFO)

    if not patched:

        log.warn('%s already patched.', pkg_info)

        return False

    # now let's move the files out of the way

    for element in ('setuptools', 'pkg_resources.py', 'site.py'):

        element = os.path.join(placeholder, element)

        if os.path.exists(element):

            _rename_path(element)

        else:

            log.warn('Could not find the %s element of the '

                     'Setuptools distribution', element)

    return True

_remove_flat_installation = _no_sandbox(_remove_flat_installation)

def _after_install(dist):

    log.warn('After install bootstrap.')

    placeholder = dist.get_command_obj('install').install_purelib

    _create_fake_setuptools_pkg_info(placeholder)

def _create_fake_setuptools_pkg_info(placeholder):

    if not placeholder or not os.path.exists(placeholder):

        log.warn('Could not find the install location')

        return

    pyver = '%s.%s' % (sys.version_info[0], sys.version_info[1])

    setuptools_file = 'setuptools-%s-py%s.egg-info' % \

            (SETUPTOOLS_FAKED_VERSION, pyver)

    pkg_info = os.path.join(placeholder, setuptools_file)

    if os.path.exists(pkg_info):

        log.warn('%s already exists', pkg_info)

        return

    log.warn('Creating %s', pkg_info)

    f = open(pkg_info, 'w')

    try:

        f.write(SETUPTOOLS_PKG_INFO)

    finally:

        f.close()

    pth_file = os.path.join(placeholder, 'setuptools.pth')

    log.warn('Creating %s', pth_file)

    f = open(pth_file, 'w')

    try:

        f.write(os.path.join(os.curdir, setuptools_file))

    finally:

        f.close()

_create_fake_setuptools_pkg_info = _no_sandbox(_create_fake_setuptools_pkg_info)

def _patch_egg_dir(path):

    # let's check if it's already patched

    pkg_info = os.path.join(path, 'EGG-INFO', 'PKG-INFO')

    if os.path.exists(pkg_info):

        if _same_content(pkg_info, SETUPTOOLS_PKG_INFO):

            log.warn('%s already patched.', pkg_info)

            return False

    _rename_path(path)

    os.mkdir(path)

    os.mkdir(os.path.join(path, 'EGG-INFO'))

    pkg_info = os.path.join(path, 'EGG-INFO', 'PKG-INFO')

    f = open(pkg_info, 'w')

    try:

        f.write(SETUPTOOLS_PKG_INFO)

    finally:

        f.close()

    return True

_patch_egg_dir = _no_sandbox(_patch_egg_dir)

def _before_install():

    log.warn('Before install bootstrap.')

    _fake_setuptools()

def _under_prefix(location):

    if 'install' not in sys.argv:

        return True

    args = sys.argv[sys.argv.index('install')+1:]

    for index, arg in enumerate(args):

        for option in ('--root', '--prefix'):

            if arg.startswith('%s=' % option):

                top_dir = arg.split('root=')[-1]

                return location.startswith(top_dir)

            elif arg == option:

                if len(args) > index:

                    top_dir = args[index+1]

                    return location.startswith(top_dir)

        if arg == '--user' and USER_SITE is not None:

            return location.startswith(USER_SITE)

    return True

def _fake_setuptools():

    log.warn('Scanning installed packages')

    try:

        import pkg_resources

    except ImportError:

        # we're cool

        log.warn('Setuptools or Distribute does not seem to be installed.')

        return

    ws = pkg_resources.working_set

    try:

        setuptools_dist = ws.find(pkg_resources.Requirement.parse('setuptools',

                                  replacement=False))

    except TypeError:

        # old distribute API

        setuptools_dist = ws.find(pkg_resources.Requirement.parse('setuptools'))

    if setuptools_dist is None:

        log.warn('No setuptools distribution found')

        return

    # detecting if it was already faked

    setuptools_location = setuptools_dist.location

    log.warn('Setuptools installation detected at %s', setuptools_location)

    # if --root or --preix was provided, and if

    # setuptools is not located in them, we don't patch it

    if not _under_prefix(setuptools_location):

        log.warn('Not patching, --root or --prefix is installing Distribute'

                 ' in another location')

        return

    # let's see if its an egg

    if not setuptools_location.endswith('.egg'):

        log.warn('Non-egg installation')

        res = _remove_flat_installation(setuptools_location)

        if not res:

            return

    else:

        log.warn('Egg installation')

        pkg_info = os.path.join(setuptools_location, 'EGG-INFO', 'PKG-INFO')

        if (os.path.exists(pkg_info) and

            _same_content(pkg_info, SETUPTOOLS_PKG_INFO)):

            log.warn('Already patched.')

            return

        log.warn('Patching...')

        # let's create a fake egg replacing setuptools one

        res = _patch_egg_dir(setuptools_location)

        if not res:

            return

    log.warn('Patched done.')

    _relaunch()

def _relaunch():

    log.warn('Relaunching...')

    # we have to relaunch the process

    # pip marker to avoid a relaunch bug

    if sys.argv[:3] == ['-c', 'install', '--single-version-externally-managed']:

        sys.argv[0] = 'setup.py'

    args = [sys.executable] + sys.argv

    sys.exit(subprocess.call(args))

def _extractall(self, path=".", members=None):

    """Extract all members from the archive to the current working

       directory and set owner, modification time and permissions on

       directories afterwards. `path' specifies a different directory

       to extract to. `members' is optional and must be a subset of the

       list returned by getmembers().

    """

    import copy

    import operator

    from tarfile import ExtractError

    directories = []

    if members is None:

        members = self

    for tarinfo in members:

        if tarinfo.isdir():

            # Extract directories with a safe mode.

            directories.append(tarinfo)

            tarinfo = copy.copy(tarinfo)

            tarinfo.mode = 448 # decimal for oct 0700

        self.extract(tarinfo, path)

    # Reverse sort directories.

    if sys.version_info < (2, 4):

        def sorter(dir1, dir2):

            return cmp(dir1.name, dir2.name)

        directories.sort(sorter)

        directories.reverse()

    else:

        directories.sort(key=operator.attrgetter('name'), reverse=True)

    # Set correct owner, mtime and filemode on directories.

    for tarinfo in directories:

        dirpath = os.path.join(path, tarinfo.name)

        try:

            self.chown(tarinfo, dirpath)

            self.utime(tarinfo, dirpath)

            self.chmod(tarinfo, dirpath)

        except ExtractError:

            e = sys.exc_info()[1]

            if self.errorlevel > 1:

                raise

            else:

                self._dbg(1, "tarfile: %s" % e)

def main(argv, version=DEFAULT_VERSION):

    """Install or upgrade setuptools and EasyInstall"""

    tarball = download_setuptools()

    _install(tarball)

if __name__ == '__main__':

    main(sys.argv[1:])

from __future__ import with_statement

from fabric.api import *

from fabric.contrib.console import confirm

from random import choice

from fabric.operations import run, put

import re

import shutil, os

from functools import wraps

import imp

import ConfigParser

import sys

import tempfile

import ast

from snfdeploy.lib import *

from snfdeploy import massedit

def setup_env(confdir="conf", packages="packages",

              templates="files", cluster_name="ganeti1", autoconf=False):

    print("Loading configuration for synnefo...")

    print(" * Using config files under %s..." % confdir)

    print(" * Using %s and %s for packages and templates accordingly..." % (packages, templates))

    autoconf = ast.literal_eval(autoconf)

    conf = Conf.configure(confdir=confdir, cluster_name=cluster_name, autoconf=autoconf)

    env.env = Env(conf)

    env.local = autoconf

    env.password = env.env.password

    env.user = env.env.user

    env.shell = "/bin/bash -c"

    if env.env.cms.hostname in [env.env.accounts.hostname, env.env.cyclades.hostname, env.env.pithos.hostname]:

      env.cms_pass = True

    else:

      env.cms_pass = False

    if env.env.accounts.hostname in [env.env.cyclades.hostname, env.env.pithos.hostname]:

      env.csrf_disable = True

    else:

      env.csrf_disable = False

    env.roledefs = {

        "nodes": env.env.ips,

        "ips": env.env.ips,

        "accounts": [env.env.accounts.ip],

        "cyclades": [env.env.cyclades.ip],

        "pithos": [env.env.pithos.ip],

        "cms": [env.env.cms.ip],

        "mq": [env.env.mq.ip],

        "db": [env.env.db.ip],

        "ns": [env.env.ns.ip],

        "client": [env.env.client.ip],

        "router": [env.env.router.ip],

    }

    env.enable_lvm = False

    env.enable_drbd = False

    if ast.literal_eval(env.env.create_extra_disk) and env.env.extra_disk:

        env.enable_lvm = True

        env.enable_drbd = True

    env.roledefs.update({

        "ganeti": env.env.cluster_ips,

        "master": [env.env.master.ip],

    })

def install_package(package):

    debug(env.host, " * Installing package %s..." % package)

    APT_GET = "export DEBIAN_FRONTEND=noninteractive ;apt-get install -y --force-yes "

    if ast.literal_eval(env.env.use_local_packages):

        with settings(warn_only=True):

            deb = local("ls %s/%s*deb" % (env.env.packages, package))

            if deb:

                debug(env.host, " * Package %s found in %s..." % (package, env.env.packages))

                put(deb, "/tmp/")

                try_run("dpkg -i /tmp/%s*deb || " % package + APT_GET + "-f")

                try_run("rm /tmp/%s*deb" % package)

                return

    info = getattr(env.env, package)

    if info in ["stable", "squeeze-backports", "testing", "unstable"]:

        APT_GET += " -t %s %s " % (info, package)

    elif info:

        APT_GET += " %s=%s " % (package, info)

    else:

        APT_GET += package

    try_run(APT_GET)

    return

@roles("ns")

def update_ns_for_ganeti():

    debug(env.host, "Updating name server entries for backend %s..." % env.env.cluster.fqdn)

    update_arecord(env.env.cluster)

    update_ptrrecord(env.env.cluster)

    try_run("/etc/init.d/bind9 restart")

@roles("ns")

def update_ns_for_node(node):

    info = env.env.nodes_info.get(node)

    update_arecord(info)

    update_ptrrecord(info)

    try_run("/etc/init.d/bind9 restart")

@roles("ns")

def update_arecord(host):

    filename = "/etc/bind/zones/" + env.env.domain

    cmd = """

    echo '{0}' >> {1}

    """.format(host.arecord, filename)

    try_run(cmd)

@roles("ns")

def update_cnamerecord(host):

    filename = "/etc/bind/zones/" + env.env.domain

    cmd = """

    echo '{0}' >> {1}

    """.format(host.cnamerecord, filename)

    try_run(cmd)

@roles("ns")

def update_ptrrecord(host):

    filename = "/etc/bind/rev/synnefo.in-addr.arpa.zone"

    cmd = """

    echo '{0}' >> {1}

    """.format(host.ptrrecord, filename)

    try_run(cmd)

@roles("nodes")

def apt_get_update():

    debug(env.host, "apt-get update....")

    try_run("apt-get update")

@roles("ns")

def setup_ns():

    debug(env.host, "Setting up name server..")

    #WARNING: this should be remove after we are done

    # because gevent does pick randomly nameservers and google does

    # not know our setup!!!!!

    apt_get_update()

    install_package("bind9")

    tmpl = "/etc/bind/named.conf.local"

    replace = {

      "domain": env.env.domain,

      }

    custom = customize_settings_from_tmpl(tmpl, replace)

    put(custom, tmpl)

    try_run("mkdir -p /etc/bind/zones")

    tmpl = "/etc/bind/zones/example.com"

    replace = {

      "domain": env.env.domain,

      "ns_node_ip": env.env.ns.ip,

      }

    custom = customize_settings_from_tmpl(tmpl, replace)

    remote = "/etc/bind/zones/" + env.env.domain

    put(custom, remote)

    try_run("mkdir -p /etc/bind/rev")

    tmpl = "/etc/bind/rev/synnefo.in-addr.arpa.zone"

    replace = {

      "domain": env.env.domain,

      }

    custom = customize_settings_from_tmpl(tmpl, replace)

    put(custom, tmpl)

    tmpl = "/etc/bind/named.conf.options"

    replace = {

      "NODE_IPS": ";".join(env.env.ips),

      }

    custom = customize_settings_from_tmpl(tmpl, replace)

    put(custom, tmpl, mode=0644)

    for role, info in env.env.roles.iteritems():

        if role == "ns":

            continue

        update_cnamerecord(info)

    for node, info in env.env.nodes_info.iteritems():

        update_arecord(info)

        update_ptrrecord(info)

    try_run("/etc/init.d/bind9 restart")

@roles("nodes")

def check_dhcp():

    debug(env.host, "Checking IPs for synnefo..")

    for n, info in env.env.nodes_info.iteritems():

        try_run("ping -c 1 " + info.ip)

@roles("nodes")

def check_dns():

    debug(env.host, "Checking fqdns for synnefo..")

    for n, info in env.env.nodes_info.iteritems():

        try_run("ping -c 1 " + info.fqdn)

    for n, info in env.env.roles.iteritems():

        try_run("ping -c 1 " + info.fqdn)

@roles("nodes")

def check_connectivity():

    debug(env.host, "Checking internet connectivity..")

    try_run("ping -c 1 www.google.com")

@roles("nodes")

def check_ssh():

    debug(env.host, "Checking password-less ssh..")

    for n, info in env.env.nodes_info.iteritems():

        try_run("ssh " + info.fqdn + "  date")

@roles("ips")

def add_keys():

    debug(env.host, "Adding rsa/dsa keys..")

    try_run("mkdir -p /root/.ssh")

    cmd = """

for f in $(ls /root/.ssh/*); do

  cp $f $f.bak

done

    """

    try_run(cmd)

    files = ["authorized_keys", "id_dsa", "id_dsa.pub",

             "id_rsa", "id_rsa.pub"]

    for f in files:

      tmpl = "/root/.ssh/" + f

      replace = {}

      custom = customize_settings_from_tmpl(tmpl, replace)

      put(custom, tmpl)

    cmd = """

if [ -e /root/.ssh/authorized_keys.bak ]; then

  cat /root/.ssh/authorized_keys.bak >> /root/.ssh/authorized_keys

fi

    """

    debug(env.host, "Updating exising authorized keys..")

    try_run(cmd)

@roles("ips")

def setup_resolv_conf():

    debug(env.host, "Tweak /etc/resolv.conf...")

    try_run("/etc/init.d/network-manager stop")

    tmpl = "/etc/dhcp/dhclient-enter-hooks.d/nodnsupdate"

    replace = {}

    custom = customize_settings_from_tmpl(tmpl, replace)

    put(custom, tmpl, mode=0644)

    try_run("cp /etc/resolv.conf /etc/resolv.conf.bak")

    tmpl = "/etc/resolv.conf"

    replace = {

      "domain": env.env.domain,

      "ns_node_ip": env.env.ns.ip,

      }

    custom = customize_settings_from_tmpl(tmpl, replace)

    put(custom, tmpl)

    try_run("chattr +i /etc/resolv.conf")

@roles("ips")

def setup_hosts():

    debug(env.host, "Tweaking /etc/hosts and ssh_config files...")

    try_run("echo StrictHostKeyChecking no >> /etc/ssh/ssh_config")

    cmd = " sed -i 's/^127.*/127.0.0.1 localhost/g' /etc/hosts "

    try_run(cmd)

def try_run(cmd):

    try:

      if env.local:

        return local(cmd, capture=True)

      else:

        return run(cmd)

    except:

      debug(env.host, "WARNING: command failed. Continuing anyway...")

def create_bridges():

    debug(env.host, " * Creating bridges...")

    install_package("bridge-utils")

    cmd = """

    brctl addbr {0} ; ip link set {0} up

    """.format(env.env.common_bridge)

    try_run(cmd)

def connect_bridges():

    debug(env.host, " * Connecting bridges...")

    cmd = """

    brctl addif {0} {1}

    """.format(env.env.common_bridge, env.env.public_iface)

    #try_run(cmd)

@roles("ganeti")

def setup_net_infra():

    debug(env.host, "Setup networking infrastracture..")

    create_bridges()

    connect_bridges()

@roles("ganeti")

def setup_lvm():

    debug(env.host, "create volume group %s for ganeti.." % env.env.vg)

    if env.enable_lvm:

        install_package("lvm2")

        cmd = """

        pvcreate {0}

        vgcreate {1} {0}

        """.format(env.env.extra_disk, env.env.vg)

        try_run(cmd)

def customize_settings_from_tmpl(tmpl, replace):

    debug(env.host, " * Customizing template %s..." % tmpl)

    local = env.env.templates + tmpl

    _, custom = tempfile.mkstemp()

    shutil.copyfile(local, custom)

    for k, v in replace.iteritems():

        regex = "re.sub('%{0}%', '{1}', line)".format(k.upper(), v)

        massedit.edit_files([custom], [regex], dry_run=False)

    return custom

@roles("nodes")

def setup_apt():

    debug(env.host, "Setting up apt sources...")

    install_package("curl")

    cmd = """

    echo 'APT::Install-Suggests "false";' >> /etc/apt/apt.conf

    curl -k https://dev.grnet.gr/files/apt-grnetdev.pub | apt-key add -

    """

    try_run(cmd)

    tmpl = "/etc/apt/sources.list.d/okeanos.list"

    replace = {}

    custom = customize_settings_from_tmpl(tmpl, replace)

    put(custom, tmpl)

    apt_get_update()

@roles("cyclades", "cms", "pithos", "accounts")

def restart_services():

    debug(env.host, " * Restarting apache2 and gunicorn...")

    try_run("/etc/init.d/gunicorn restart")

    try_run("/etc/init.d/apache2 restart")

def setup_gunicorn():

    debug(env.host, " * Setting up gunicorn...")

    install_package("gunicorn")

    tmpl = "/etc/gunicorn.d/synnefo"

    replace = {}

    custom = customize_settings_from_tmpl(tmpl, replace)

    put(custom, tmpl, mode=0644)

    try_run("/etc/init.d/gunicorn restart")

def setup_apache():

    debug(env.host, " * Setting up apache2...")

    host_info = env.env.ips_info[env.host]

    install_package("apache2")

    tmpl = "/etc/apache2/sites-available/synnefo"

    replace = {

        "HOST": host_info.fqdn,

    }

    custom = customize_settings_from_tmpl(tmpl, replace)

    put(custom, tmpl)

    tmpl = "/etc/apache2/sites-available/synnefo-ssl"

    custom = customize_settings_from_tmpl(tmpl, replace)

    put(custom, tmpl)

    cmd = """

    a2enmod ssl

    a2enmod rewrite

    a2dissite default

    a2ensite synnefo

    a2ensite synnefo-ssl

    a2enmod headers

    a2enmod proxy_http

    a2dismod autoindex

    """

    try_run(cmd)

    try_run("/etc/init.d/apache2 restart")

@roles("mq")

def setup_mq():

    debug(env.host, "Setting up RabbitMQ...")

    install_package("rabbitmq-server")

    cmd = """

    rabbitmqctl add_user {0} {1}

    rabbitmqctl set_permissions {0} ".*" ".*" ".*"

    rabbitmqctl delete_user guest

    rabbitmqctl set_user_tags {0} administrator

    """.format(env.env.synnefo_user, env.env.synnefo_rabbitmq_passwd)

    try_run(cmd)

    try_run("/etc/init.d/rabbitmq-server restart")

@roles("db")

def allow_access_in_db(ip):

    cmd = """

    echo host all all {0}/32 md5 >> /etc/postgresql/8.4/main/pg_hba.conf

    """.format(ip)

    try_run(cmd)

    try_run("/etc/init.d/postgresql restart")

@roles("db")

def setup_db():

    debug(env.host, "Setting up DataBase server...")

    install_package("postgresql")

    tmpl = "/tmp/db-init.psql"

    replace = {

        "synnefo_user": env.env.synnefo_user,

        "synnefo_db_passwd": env.env.synnefo_db_passwd,

        }

    custom = customize_settings_from_tmpl(tmpl, replace)

    put(custom, tmpl)

    cmd = 'su - postgres -c "psql -w -f %s" ' % tmpl

    try_run(cmd)

    cmd = """

    echo "listen_addresses = '*'" >> /etc/postgresql/8.4/main/postgresql.conf

    """

    try_run(cmd)

    try_run("/etc/init.d/postgresql restart")

@roles("db")

def destroy_db():

    try_run("""su - postgres -c ' psql -w -c "drop database snf_apps" '""")

    try_run("""su - postgres -c ' psql -w -c "drop database snf_pithos" '""")

def setup_webproject():

    debug(env.host, " * Setting up snf-webproject...")

    with settings(hide("everything")):

        try_run("ping -c1 " + env.env.db.ip)

    setup_common()

    install_package("snf-webproject")

    install_package("python-psycopg2")

    install_package("python-gevent")

    tmpl = "/etc/synnefo/webproject.conf"

    replace = {

        "synnefo_user": env.env.synnefo_user,

        "synnefo_db_passwd": env.env.synnefo_db_passwd,

        "db_node": env.env.db.ip,

        "domain": env.env.domain,

    }

    custom = customize_settings_from_tmpl(tmpl, replace)

    put(custom, tmpl, mode=0644)

    with settings(host_string=env.env.db.hostname):

        host_info = env.env.ips_info[env.host]

        allow_access_in_db(host_info.ip)

    try_run("/etc/init.d/gunicorn restart")

def setup_common():

    debug(env.host, " * Setting up snf-common...")

    host_info = env.env.ips_info[env.host]

    install_package("python-objpool")

    install_package("snf-common")

    install_package("python-astakosclient")

    install_package("snf-django-lib")

    install_package("snf-branding")

    tmpl = "/etc/synnefo/common.conf"

    replace = {

        #FIXME:

        "EMAIL_SUBJECT_PREFIX": env.host,

        "domain": env.env.domain,

        "HOST": host_info.fqdn,

    }

    custom = customize_settings_from_tmpl(tmpl, replace)

    put(custom, tmpl, mode=0644)

    try_run("/etc/init.d/gunicorn restart")

@roles("accounts")

def astakos_loaddata():

    debug(env.host, " * Loading initial data to astakos...")

    cmd = """

    snf-manage loaddata groups

    """

    try_run(cmd)

@roles("accounts")

def astakos_register_services():

    debug(env.host, " * Register services in astakos...")

    cmd = """

    snf-manage component-add "home" https://{0} home-icon.png

    snf-manage component-add "cyclades" https://{1}/cyclades/ui/

    snf-manage component-add "pithos" https://{2}/pithos/ui/

    snf-manage component-add "astakos" https://{3}/astakos/ui/

    """.format(env.env.cms.fqdn, env.env.cyclades.fqdn, env.env.pithos.fqdn, env.env.accounts.fqdn)

    try_run(cmd)

    import_service("astakos")

    import_service("pithos")

    import_service("cyclades")