Revision c29ac11d
b/ci/autopkg_debian.py | ||
---|---|---|
1 |
#!/usr/bin/env python |
|
2 |
|
|
3 |
""" |
|
4 |
Build Synnefo packages for debian |
|
5 |
""" |
|
6 |
|
|
7 |
from utils import SynnefoCI |
|
8 |
|
|
9 |
|
|
10 |
def autopkg_debian(): |
|
11 |
"""Build synnefo packages for debian""" |
|
12 |
synnefo_ci = SynnefoCI() |
|
13 |
synnefo_ci.build_synnefo() |
|
14 |
|
|
15 |
|
|
16 |
if __name__ == "__main__": |
|
17 |
autopkg_debian() |
b/ci/deploy.py | ||
---|---|---|
1 |
#!/usr/bin/env python |
|
2 |
|
|
3 |
""" |
|
4 |
Deploy Synnefo using snf-deploy |
|
5 |
""" |
|
6 |
|
|
7 |
from utils import SynnefoCI |
|
8 |
|
|
9 |
|
|
10 |
def deploy_synnefo(): |
|
11 |
"""Deploy Synnefo using snf-deploy""" |
|
12 |
synnefo_ci = SynnefoCI() |
|
13 |
synnefo_ci.deploy_synnefo() |
|
14 |
|
|
15 |
|
|
16 |
if __name__ == "__main__": |
|
17 |
deploy_synnefo() |
b/ci/fetch_packages.py | ||
---|---|---|
1 |
#!/usr/bin/env python |
|
2 |
|
|
3 |
""" |
|
4 |
Download Synnefo packages |
|
5 |
""" |
|
6 |
|
|
7 |
from utils import SynnefoCI |
|
8 |
|
|
9 |
|
|
10 |
def fetch_packages(): |
|
11 |
"""Download Synnefo packages""" |
|
12 |
synnefo_ci = SynnefoCI() |
|
13 |
synnefo_ci.fetch_packages() |
|
14 |
|
|
15 |
|
|
16 |
if __name__ == "__main__": |
|
17 |
fetch_packages() |
b/ci/new_config | ||
---|---|---|
1 |
[Global] |
|
2 |
# Timeouts in seconds |
|
3 |
build_timeout = 240 |
|
4 |
# Apt repository to use |
|
5 |
apt_repo = http://apt.dev.grnet.gr squeeze/ |
|
6 |
# Synnefo git repo |
|
7 |
synnefo_repo = https://code.grnet.gr/git/synnefo |
|
8 |
# Git branch to test (specify sha1 or branch name) |
|
9 |
synnefo_branch = HEAD |
|
10 |
# snf-deploy git repo |
|
11 |
deploy_repo = https://code.grnet.gr/git/snf-deploy |
|
12 |
# Defines the schema that snf-deploy will use |
|
13 |
schema = one_node |
|
14 |
# Local dir to save builded packages |
|
15 |
pkgs_dir = /tmp/synnefo_pkgs |
|
16 |
# If True patch the pydist.py module (see Debian bug #657665) |
|
17 |
patch_pydist = True |
|
18 |
# Configuration of git (on remote server) |
|
19 |
git_config_name = Buildbot |
|
20 |
git_config_mail = synnefo@builder.dev.grnet.gr |
|
21 |
# Url to fetch ssh public keys |
|
22 |
public_ssh_keys_url = |
|
23 |
# Network address from which we allow access to server |
|
24 |
filter_access_network = 195.251.29.0/24,62.217.123.39 |
|
25 |
# Config file to save temporary options (eg IPs, passwords etc) |
|
26 |
temporary_config = /tmp/ci_temp_conf |
|
27 |
|
|
28 |
|
|
29 |
[Deployment] |
|
30 |
# Credentials |
|
31 |
auth_url = https://accounts.okeanos.io/identity/v2.0/ |
|
32 |
token = |
|
33 |
# If we deploy on okeanos.io we have to set this to True |
|
34 |
# The server will reside besides a NAT and we have to compute ssh port |
|
35 |
deploy_on_io = True |
|
36 |
# Server name to use for our machine |
|
37 |
server_name = Synnefo Deployment |
|
38 |
# Flavor ID to use |
|
39 |
# (149 for production, 639 for okeanos.io) |
|
40 |
flavor_id = 639 |
|
41 |
# Image to use (name must contain this) |
|
42 |
image_name = OldStable |
|
43 |
# UUID of owner of system images |
|
44 |
# (25ecced9-bf53-4145-91ee-cf47377e9fb2 for production, |
|
45 |
# 04cbe33f-29b7-4ef1-94fb-015929e5fc06 for okeanos.io) |
|
46 |
system_uuid = 04cbe33f-29b7-4ef1-94fb-015929e5fc06 |
|
47 |
|
|
48 |
|
|
49 |
[Burnin] |
|
50 |
# Maybe add some burnin options |
|
51 |
# (e.g. tests to run/ignore, timeouts etc) |
|
52 |
cmd_options = --nofailfast --no-ipv6 --action-timeout=240 |
|
53 |
|
|
54 |
|
|
55 |
[Unit Tests] |
|
56 |
component = astakos cyclades pithos |
|
57 |
|
|
58 |
|
|
59 |
[Repository] |
|
60 |
# Projects reside on this repo |
|
61 |
projects = |
|
62 |
snf-common |
|
63 |
astakosclient |
|
64 |
snf-django-lib |
|
65 |
snf-webproject |
|
66 |
snf-branding |
|
67 |
snf-astakos-app |
|
68 |
snf-pithos-backend |
|
69 |
snf-cyclades-gtools |
|
70 |
snf-cyclades-app |
|
71 |
snf-pithos-app |
|
72 |
snf-tools |
b/ci/run_burnin.py | ||
---|---|---|
1 |
#!/usr/bin/env python |
|
2 |
|
|
3 |
""" |
|
4 |
Run burnin functional test suite |
|
5 |
""" |
|
6 |
|
|
7 |
from utils import SynnefoCI |
|
8 |
|
|
9 |
|
|
10 |
def run_burnin(): |
|
11 |
"""Run burnin functional test suite""" |
|
12 |
synnefo_ci = SynnefoCI() |
|
13 |
synnefo_ci.run_burnin() |
|
14 |
|
|
15 |
|
|
16 |
if __name__ == "__main__": |
|
17 |
run_burnin() |
b/ci/schemas/one_node/deploy.conf | ||
---|---|---|
1 |
[packages] |
|
2 |
# whether to use apt-get or local generated package found in packages dir |
|
3 |
use_local_packages = True |
|
4 |
|
|
5 |
# url to obtain latest synnefo packages. |
|
6 |
# To use them change USE_LOCAL_PACKAGES setting to yes |
|
7 |
# To get them run: snf-deploy packages |
|
8 |
package_url = http://builder.dev.grnet.gr/synnefo/packages/Squeeze/40/ |
|
9 |
|
|
10 |
[dirs] |
|
11 |
# dir to find all template files used to customize setup |
|
12 |
# in case you want to add another setting please modify the corresponding file |
|
13 |
templates = /var/lib/snf-deploy/files |
|
14 |
# dir to store local images (disk0, disk1 of the virtual cluster) |
|
15 |
images = /var/lib/snf-deploy/images |
|
16 |
# dir to store/find local packages |
|
17 |
# dir to locally save packages that will be downloaded from package_url |
|
18 |
# put here any locally created packages (useful for development) |
|
19 |
packages = /var/lib/snf-deploy/packages |
|
20 |
# dir to store pidfiles (dnsmasq, kvm) |
|
21 |
run = /var/run/snf-deploy |
|
22 |
# dir to store dnsmasq related files |
|
23 |
dns = /var/lib/snf-deploy/dnsmasq |
|
24 |
# dir to lookup fabfile and ifup script |
|
25 |
lib = /usr/lib/snf-deploy |
|
26 |
# dir to store executed commands (to enforce sequential execution) |
|
27 |
cmd = /var/run/snf-deploy/cmd |
b/ci/schemas/one_node/ganeti.conf | ||
---|---|---|
1 |
[ganeti1] |
|
2 |
cluster_nodes = node1 |
|
3 |
master_node = node1 |
|
4 |
|
|
5 |
cluster_netdev = eth0 |
|
6 |
cluster_name = ganeti1 |
|
7 |
cluster_ip = 192.168.0.13 |
|
8 |
|
|
9 |
vg = autovg |
|
10 |
|
|
11 |
synnefo_public_network_subnet = 10.2.1.0/24 |
|
12 |
synnefo_public_network_gateway = 10.2.1.1 |
|
13 |
synnefo_public_network_type = CUSTOM |
|
14 |
|
|
15 |
image_dir = /srv/okeanos |
|
16 |
|
|
17 |
# To add another cluster repeat the above section |
|
18 |
# with different header and nodes |
b/ci/schemas/one_node/nodes.conf | ||
---|---|---|
1 |
# please note that currently is only supported deployment |
|
2 |
# with nodes (both ganeti and synnefo) residing in the same subnet/domain |
|
3 |
[network] |
|
4 |
domain = synnefo.live |
|
5 |
subnet = 192.168.0.0/28 |
|
6 |
gateway = 192.168.0.14 |
|
7 |
|
|
8 |
|
|
9 |
[hostnames] |
|
10 |
node1 = auto1 |
|
11 |
# node2 = auto2 |
|
12 |
|
|
13 |
[ips] |
|
14 |
node1 = 192.168.0.1 |
|
15 |
# node2 = 192.168.0.2 |
|
16 |
|
|
17 |
# This is used only in case of vcluster |
|
18 |
# needed to pass the correct dhcp responces to the virtual nodes |
|
19 |
[macs] |
|
20 |
node1 = 52:54:00:00:00:01 |
|
21 |
# node2 = 52:54:00:00:00:02 |
|
22 |
|
|
23 |
[info] |
|
24 |
# Here we define which nodes from the predefined ones to use |
|
25 |
nodes = node1 |
|
26 |
|
|
27 |
# login credentials for the nodes |
|
28 |
# please note that in case of vcluster these are preconfigured |
|
29 |
# and not editable. |
|
30 |
# in case of physical nodes all nodes should have the same login account |
|
31 |
user = root |
|
32 |
password = 12345 |
|
33 |
|
|
34 |
public_iface = eth0 |
|
35 |
vm_public_iface = eth1 |
|
36 |
vm_private_iface = eth2 |
|
37 |
|
|
38 |
# extra disk name inside the nodes |
|
39 |
# if defined, snf-deploy will create a VG for ganeti in order to support lvm storage |
|
40 |
# if not then only file disk template will be supported |
|
41 |
extra_disk = /dev/vdb |
b/ci/schemas/one_node/packages.conf | ||
---|---|---|
1 |
[debian] |
|
2 |
rabbitmq-server = testing |
|
3 |
gunicorn = squeeze-backports |
|
4 |
qemu-kvm = squeeze-backports |
|
5 |
qemu = squeeze-backports |
|
6 |
python-gevent = squeeze-backports |
|
7 |
apache2 = |
|
8 |
postgresql = |
|
9 |
python-psycopg2 = |
|
10 |
python-argparse = |
|
11 |
nfs-kernel-server = squeeze-backports |
|
12 |
nfs-common = squeeze-backports |
|
13 |
bind9 = |
|
14 |
vlan = |
|
15 |
vlan = |
|
16 |
lvm2 = |
|
17 |
curl = |
|
18 |
memcached = |
|
19 |
python-memcache = |
|
20 |
bridge-utils = |
|
21 |
python-progress = |
|
22 |
ganeti-instance-debootstrap = |
|
23 |
|
|
24 |
|
|
25 |
[synnefo] |
|
26 |
snf-astakos-app = stable |
|
27 |
snf-common = stable |
|
28 |
snf-cyclades-app = stable |
|
29 |
snf-cyclades-gtools = stable |
|
30 |
snf-django-lib = stable |
|
31 |
python-astakosclient = stable |
|
32 |
python-objpool = stable |
|
33 |
snf-branding = stable |
|
34 |
snf-webproject = stable |
|
35 |
snf-pithos-app = stable |
|
36 |
snf-pithos-backend = stable |
|
37 |
snf-tools = stable |
|
38 |
|
|
39 |
|
|
40 |
[ganeti] |
|
41 |
snf-ganeti = 2.6.2+ippool11+hotplug5+extstorage3+rbdfix1+kvmfix2+nolvm+netxen-1~squeeze |
|
42 |
ganeti-htools = 2.6.2+ippool11+hotplug5+extstorage3+rbdfix1+kvmfix2+nolvm+netxen-1~squeeze |
|
43 |
|
|
44 |
[other] |
|
45 |
snf-cloudcms = stable |
|
46 |
snf-vncauthproxy = stable |
|
47 |
snf-pithos-webclient = stable |
|
48 |
snf-image = stable |
|
49 |
snf-network = stable |
|
50 |
nfdhcpd = stable |
|
51 |
kamaki = stable |
|
52 |
python-bitarray = stable |
|
53 |
nfqueue-bindings-python = stable |
|
54 |
|
b/ci/schemas/one_node/synnefo.conf | ||
---|---|---|
1 |
[cred] |
|
2 |
synnefo_user = synnefo |
|
3 |
synnefo_db_passwd = example_passw0rd |
|
4 |
synnefo_rapi_passwd = example_rapi_passw0rd |
|
5 |
synnefo_rabbitmq_passwd = example_rabbitmq_passw0rd |
|
6 |
user_email = user@synnefo.org |
|
7 |
user_name = John |
|
8 |
user_lastname = Doe |
|
9 |
user_passwd = 12345 |
|
10 |
|
|
11 |
|
|
12 |
[roles] |
|
13 |
accounts = node1 |
|
14 |
compute = node1 |
|
15 |
object-store = node1 |
|
16 |
cyclades = node1 |
|
17 |
pithos = node1 |
|
18 |
cms = node1 |
|
19 |
db = node1 |
|
20 |
mq = node1 |
|
21 |
ns = node1 |
|
22 |
client = node1 |
|
23 |
router = node1 |
|
24 |
|
|
25 |
|
|
26 |
[synnefo] |
|
27 |
pithos_dir = /srv/pithos |
|
28 |
|
|
29 |
vm_public_bridge = br0 |
|
30 |
vm_private_bridge = prv0 |
|
31 |
common_bridge = br0 |
|
32 |
|
|
33 |
debian_base_url = https://pithos.okeanos.grnet.gr/public/RDISy7sNVIJ9KIm4JkmbX4 |
b/ci/schemas/one_node/vcluster.conf | ||
---|---|---|
1 |
[image] |
|
2 |
# url to get the base image. This is a debian base image with preconfigured |
|
3 |
# root password and installed rsa/dsa keys. Plus a NetworkManager hook that |
|
4 |
# changes the VM's name based on info provided by dhcp response. |
|
5 |
# To create it run: snf-deploy image |
|
6 |
squeeze_image_url = https://pithos.okeanos.grnet.gr/public/832xv |
|
7 |
ubuntu_image_url = |
|
8 |
|
|
9 |
# in order ganeti nodes to support lvm storage (plain disk template) it will |
|
10 |
# be needed an extra disk to eventually be able to create a VG. Ganeti requires |
|
11 |
# this VG to be at least of 30GB. To this end in order the virtual nodes to have |
|
12 |
# this extra disk an image should be created locally. There are three options: |
|
13 |
# 1. not create an extra disk (only file storage template will be supported) |
|
14 |
# 2. create an image of 30G in image dir (default /var/lib/snf-deploy/images) |
|
15 |
# using dd if=/dev/zero of=squeeze.disk1 |
|
16 |
# 3. create this image in a local VG using lvgreate -L30G squeeze.disk1 lvg |
|
17 |
# and create a symbolic link in /var/lib/snf-deploy/images |
|
18 |
|
|
19 |
# Whether to create an extra disk or not |
|
20 |
create_extra_disk = False |
|
21 |
# lvg is the name of the local VG if any |
|
22 |
lvg = |
|
23 |
|
|
24 |
# OS istalled in the virtual cluster |
|
25 |
os = squeeze |
|
26 |
|
|
27 |
|
|
28 |
[cluster] |
|
29 |
# the bridge to use for the virtual cluster |
|
30 |
# on this bridge we will launch a dnsnmasq and provide |
|
31 |
# fqdns needed to the cluster. |
|
32 |
# In ordrer cluster nodes to have internet access, host must do NAT. |
|
33 |
# iptables -t nat -A POSTROUTING -s 192.0.0.0/28 -j MASQUERADE |
|
34 |
# ip addr add 192.0.0.14/28 dev auto_nodes_br |
|
35 |
# To create run: snf-deploy cluster |
|
36 |
bridge = auto_nodes_br |
b/ci/setup_slave.py | ||
---|---|---|
1 |
#!/usr/bin/env python |
|
2 |
|
|
3 |
""" |
|
4 |
Setup slave server |
|
5 |
""" |
|
6 |
|
|
7 |
from utils import SynnefoCI |
|
8 |
|
|
9 |
|
|
10 |
def setup_slave(): |
|
11 |
"""Setup slave server""" |
|
12 |
synnefo_ci = SynnefoCI(cleanup_config=True) |
|
13 |
# Get token from /nfs/token |
|
14 |
try: |
|
15 |
token = open("/nfs/synnefo_token").read().strip() |
|
16 |
synnefo_ci.write_config('token', token, 'Deployment') |
|
17 |
except: |
|
18 |
pass |
|
19 |
# Build slave server |
|
20 |
synnefo_ci.create_server() |
|
21 |
# Copy synnefo repo to server |
|
22 |
synnefo_ci.clone_repo() |
|
23 |
|
|
24 |
|
|
25 |
if __name__ == "__main__": |
|
26 |
setup_slave() |
b/ci/unit_tests.py | ||
---|---|---|
1 |
#!/usr/bin/env python |
|
2 |
|
|
3 |
""" |
|
4 |
Run Synnefo unit test suite |
|
5 |
""" |
|
6 |
|
|
7 |
from utils import SynnefoCI |
|
8 |
|
|
9 |
|
|
10 |
def unit_test(): |
|
11 |
"""Run Synnefo unit test suite""" |
|
12 |
synnefo_ci = SynnefoCI() |
|
13 |
synnefo_ci.unit_test() |
|
14 |
|
|
15 |
|
|
16 |
if __name__ == "__main__": |
|
17 |
unit_test() |
b/ci/utils.py | ||
---|---|---|
1 |
#!/usr/bin/env python |
|
2 |
|
|
3 |
""" |
|
4 |
Synnefo ci utils module |
|
5 |
""" |
|
6 |
|
|
7 |
import os |
|
8 |
import sys |
|
9 |
import time |
|
10 |
import logging |
|
11 |
import fabric.api as fabric |
|
12 |
from ConfigParser import ConfigParser, DuplicateSectionError |
|
13 |
|
|
14 |
from kamaki.clients.astakos import AstakosClient |
|
15 |
from kamaki.clients.cyclades import CycladesClient |
|
16 |
from kamaki.clients.image import ImageClient |
|
17 |
|
|
18 |
|
|
19 |
def _run(cmd, verbose): |
|
20 |
"""Run fabric with verbose level""" |
|
21 |
if verbose: |
|
22 |
args = ('running',) |
|
23 |
else: |
|
24 |
args = ('running', 'stdout',) |
|
25 |
with fabric.hide(*args): |
|
26 |
return fabric.run(cmd) |
|
27 |
|
|
28 |
|
|
29 |
def _red(msg): |
|
30 |
"""Red color""" |
|
31 |
#return "\x1b[31m" + str(msg) + "\x1b[0m" |
|
32 |
return str(msg) |
|
33 |
|
|
34 |
|
|
35 |
def _yellow(msg): |
|
36 |
"""Yellow color""" |
|
37 |
#return "\x1b[33m" + str(msg) + "\x1b[0m" |
|
38 |
return str(msg) |
|
39 |
|
|
40 |
|
|
41 |
def _green(msg): |
|
42 |
"""Green color""" |
|
43 |
#return "\x1b[32m" + str(msg) + "\x1b[0m" |
|
44 |
return str(msg) |
|
45 |
|
|
46 |
|
|
47 |
def _check_fabric(fun): |
|
48 |
"""Check if fabric env has been set""" |
|
49 |
def wrapper(self, *args): |
|
50 |
"""wrapper function""" |
|
51 |
if not self.fabric_installed: |
|
52 |
self.setup_fabric() |
|
53 |
return fun(self, *args) |
|
54 |
return wrapper |
|
55 |
|
|
56 |
|
|
57 |
def _check_kamaki(fun): |
|
58 |
"""Check if kamaki has been initialized""" |
|
59 |
def wrapper(self, *args): |
|
60 |
"""wrapper function""" |
|
61 |
if not self.kamaki_installed: |
|
62 |
self.setup_kamaki() |
|
63 |
return fun(self, *args) |
|
64 |
return wrapper |
|
65 |
|
|
66 |
|
|
67 |
class _MyFormatter(logging.Formatter): |
|
68 |
"""Logging Formatter""" |
|
69 |
def format(self, record): |
|
70 |
format_orig = self._fmt |
|
71 |
if record.levelno == logging.DEBUG: |
|
72 |
self._fmt = " %(msg)s" |
|
73 |
elif record.levelno == logging.INFO: |
|
74 |
self._fmt = "%(msg)s" |
|
75 |
elif record.levelno == logging.WARNING: |
|
76 |
self._fmt = _yellow("[W] %(msg)s") |
|
77 |
elif record.levelno == logging.ERROR: |
|
78 |
self._fmt = _red("[E] %(msg)s") |
|
79 |
result = logging.Formatter.format(self, record) |
|
80 |
self._fmt = format_orig |
|
81 |
return result |
|
82 |
|
|
83 |
|
|
84 |
class SynnefoCI(object): |
|
85 |
"""SynnefoCI python class""" |
|
86 |
|
|
87 |
def __init__(self, cleanup_config=False): |
|
88 |
""" Initialize SynnefoCI python class |
|
89 |
|
|
90 |
Setup logger, local_dir, config and kamaki |
|
91 |
""" |
|
92 |
# Setup logger |
|
93 |
self.logger = logging.getLogger('synnefo-ci') |
|
94 |
self.logger.setLevel(logging.DEBUG) |
|
95 |
handler = logging.StreamHandler() |
|
96 |
handler.setFormatter(_MyFormatter()) |
|
97 |
self.logger.addHandler(handler) |
|
98 |
|
|
99 |
# Get our local dir |
|
100 |
self.ci_dir = os.path.dirname(os.path.abspath(__file__)) |
|
101 |
self.repo_dir = os.path.dirname(self.ci_dir) |
|
102 |
|
|
103 |
# Read config file |
|
104 |
self.conffile = os.path.join(self.ci_dir, "new_config") |
|
105 |
self.config = ConfigParser() |
|
106 |
self.config.optionxform = str |
|
107 |
self.config.read(self.conffile) |
|
108 |
temp_config = self.config.get('Global', 'temporary_config') |
|
109 |
if cleanup_config: |
|
110 |
try: |
|
111 |
os.remove(temp_config) |
|
112 |
except: |
|
113 |
pass |
|
114 |
else: |
|
115 |
self.config.read(self.config.get('Global', 'temporary_config')) |
|
116 |
|
|
117 |
# Initialize variables |
|
118 |
self.fabric_installed = False |
|
119 |
self.kamaki_installed = False |
|
120 |
self.cyclades_client = None |
|
121 |
self.image_client = None |
|
122 |
|
|
123 |
def setup_kamaki(self): |
|
124 |
"""Initialize kamaki |
|
125 |
|
|
126 |
Setup cyclades_client and image_client |
|
127 |
""" |
|
128 |
self.logger.info("Setup kamaki client..") |
|
129 |
auth_url = self.config.get('Deployment', 'auth_url') |
|
130 |
self.logger.debug("Authentication URL is %s" % _green(auth_url)) |
|
131 |
token = self.config.get('Deployment', 'token') |
|
132 |
#self.logger.debug("Token is %s" % _green(token)) |
|
133 |
|
|
134 |
astakos_client = AstakosClient(auth_url, token) |
|
135 |
|
|
136 |
cyclades_url = \ |
|
137 |
astakos_client.get_service_endpoints('compute')['publicURL'] |
|
138 |
self.logger.debug("Cyclades API url is %s" % _green(cyclades_url)) |
|
139 |
self.cyclades_client = CycladesClient(cyclades_url, token) |
|
140 |
self.cyclades_client.CONNECTION_RETRY_LIMIT = 2 |
|
141 |
|
|
142 |
image_url = \ |
|
143 |
astakos_client.get_service_endpoints('image')['publicURL'] |
|
144 |
self.logger.debug("Images API url is %s" % _green(image_url)) |
|
145 |
self.image_client = ImageClient(cyclades_url, token) |
|
146 |
self.image_client.CONNECTION_RETRY_LIMIT = 2 |
|
147 |
|
|
148 |
def _wait_transition(self, server_id, current_status, new_status): |
|
149 |
"""Wait for server to go from current_status to new_status""" |
|
150 |
self.logger.debug("Waiting for server to become %s" % new_status) |
|
151 |
timeout = self.config.getint('Global', 'build_timeout') |
|
152 |
sleep_time = 5 |
|
153 |
while True: |
|
154 |
server = self.cyclades_client.get_server_details(server_id) |
|
155 |
if server['status'] == new_status: |
|
156 |
return server |
|
157 |
elif timeout < 0: |
|
158 |
self.logger.error( |
|
159 |
"Waiting for server to become %s timed out" % new_status) |
|
160 |
self.destroy_server(False) |
|
161 |
sys.exit(-1) |
|
162 |
elif server['status'] == current_status: |
|
163 |
# Sleep for #n secs and continue |
|
164 |
timeout = timeout - sleep_time |
|
165 |
time.sleep(sleep_time) |
|
166 |
else: |
|
167 |
self.logger.error( |
|
168 |
"Server failed with status %s" % server['status']) |
|
169 |
self.destroy_server(False) |
|
170 |
sys.exit(-1) |
|
171 |
|
|
172 |
@_check_kamaki |
|
173 |
def destroy_server(self, wait=True): |
|
174 |
"""Destroy slave server""" |
|
175 |
server_id = self.config.getint('Temporary Options', 'server_id') |
|
176 |
self.logger.info("Destoying server with id %s " % server_id) |
|
177 |
self.cyclades_client.delete_server(server_id) |
|
178 |
if wait: |
|
179 |
self._wait_transition(server_id, "ACTIVE", "DELETED") |
|
180 |
|
|
181 |
@_check_kamaki |
|
182 |
def create_server(self): |
|
183 |
"""Create slave server""" |
|
184 |
self.logger.info("Create a new server..") |
|
185 |
image = self._find_image() |
|
186 |
self.logger.debug("Will use image \"%s\"" % _green(image['name'])) |
|
187 |
self.logger.debug("Image has id %s" % _green(image['id'])) |
|
188 |
server = self.cyclades_client.create_server( |
|
189 |
self.config.get('Deployment', 'server_name'), |
|
190 |
self.config.getint('Deployment', 'flavor_id'), |
|
191 |
image['id']) |
|
192 |
server_id = server['id'] |
|
193 |
self.write_config('server_id', server_id) |
|
194 |
self.logger.debug("Server got id %s" % _green(server_id)) |
|
195 |
server_user = server['metadata']['users'] |
|
196 |
self.write_config('server_user', server_user) |
|
197 |
self.logger.debug("Server's admin user is %s" % _green(server_user)) |
|
198 |
server_passwd = server['adminPass'] |
|
199 |
self.write_config('server_passwd', server_passwd) |
|
200 |
self.logger.debug( |
|
201 |
"Server's admin password is %s" % _green(server_passwd)) |
|
202 |
|
|
203 |
server = self._wait_transition(server_id, "BUILD", "ACTIVE") |
|
204 |
self._get_server_ip_and_port(server) |
|
205 |
|
|
206 |
self.setup_fabric() |
|
207 |
self.logger.info("Setup firewall") |
|
208 |
accept_ssh_from = self.config.get('Global', 'filter_access_network') |
|
209 |
self.logger.debug("Block ssh except from %s" % accept_ssh_from) |
|
210 |
cmd = """ |
|
211 |
iptables -A INPUT -s localhost -j ACCEPT |
|
212 |
iptables -A INPUT -s {0} -p tcp --dport 22 -j ACCEPT |
|
213 |
iptables -A INPUT -p tcp --dport 22 -j DROP |
|
214 |
""".format(accept_ssh_from) |
|
215 |
_run(cmd, False) |
|
216 |
|
|
217 |
def _find_image(self): |
|
218 |
"""Find a suitable image to use |
|
219 |
|
|
220 |
It has to belong to the `system_uuid' user and |
|
221 |
contain the word `image_name' |
|
222 |
""" |
|
223 |
system_uuid = self.config.get('Deployment', 'system_uuid') |
|
224 |
image_name = self.config.get('Deployment', 'image_name').lower() |
|
225 |
images = self.image_client.list_public(detail=True)['images'] |
|
226 |
# Select images by `system_uuid' user |
|
227 |
images = [x for x in images if x['user_id'] == system_uuid] |
|
228 |
# Select images with `image_name' in their names |
|
229 |
images = \ |
|
230 |
[x for x in images if x['name'].lower().find(image_name) != -1] |
|
231 |
# Let's select the first one |
|
232 |
return images[0] |
|
233 |
|
|
234 |
def _get_server_ip_and_port(self, server): |
|
235 |
"""Compute server's IPv4 and ssh port number""" |
|
236 |
self.logger.info("Get server connection details..") |
|
237 |
# XXX: check if this IP is from public network |
|
238 |
server_ip = server['attachments'][0]['ipv4'] |
|
239 |
if eval(self.config.get('Deployment', 'deploy_on_io')): |
|
240 |
tmp1 = int(server_ip.split(".")[2]) |
|
241 |
tmp2 = int(server_ip.split(".")[3]) |
|
242 |
server_ip = "gate.okeanos.io" |
|
243 |
server_port = 10000 + tmp1 * 256 + tmp2 |
|
244 |
else: |
|
245 |
server_port = 22 |
|
246 |
self.write_config('server_ip', server_ip) |
|
247 |
self.logger.debug("Server's IPv4 is %s" % _green(server_ip)) |
|
248 |
self.write_config('server_port', server_port) |
|
249 |
self.logger.debug("Server's ssh port is %s" % _green(server_port)) |
|
250 |
|
|
251 |
def write_config(self, option, value, section="Temporary Options"): |
|
252 |
"""Write changes back to config file""" |
|
253 |
try: |
|
254 |
self.config.add_section(section) |
|
255 |
except DuplicateSectionError: |
|
256 |
pass |
|
257 |
self.config.set(section, option, str(value)) |
|
258 |
temp_conf_file = self.config.get('Global', 'temporary_config') |
|
259 |
with open(temp_conf_file, 'wb') as tcf: |
|
260 |
self.config.write(tcf) |
|
261 |
|
|
262 |
def setup_fabric(self): |
|
263 |
"""Setup fabric environment""" |
|
264 |
self.logger.info("Setup fabric parameters..") |
|
265 |
fabric.env.user = self.config.get('Temporary Options', 'server_user') |
|
266 |
fabric.env.host_string = \ |
|
267 |
self.config.get('Temporary Options', 'server_ip') |
|
268 |
fabric.env.port = self.config.getint('Temporary Options', 'server_port') |
|
269 |
fabric.env.password = \ |
|
270 |
self.config.get('Temporary Options', 'server_passwd') |
|
271 |
fabric.env.connection_attempts = 10 |
|
272 |
fabric.env.shell = "/bin/bash -c" |
|
273 |
fabric.env.disable_known_hosts = True |
|
274 |
fabric.env.output_prefix = None |
|
275 |
|
|
276 |
def _check_hash_sum(self, localfile, remotefile): |
|
277 |
"""Check hash sums of two files""" |
|
278 |
self.logger.debug("Check hash sum for local file %s" % localfile) |
|
279 |
hash1 = os.popen("sha256sum %s" % localfile).read().split(' ')[0] |
|
280 |
self.logger.debug("Local file has sha256 hash %s" % hash1) |
|
281 |
self.logger.debug("Check hash sum for remote file %s" % remotefile) |
|
282 |
hash2 = _run("sha256sum %s" % remotefile, False) |
|
283 |
hash2 = hash2.split(' ')[0] |
|
284 |
self.logger.debug("Remote file has sha256 hash %s" % hash2) |
|
285 |
if hash1 != hash2: |
|
286 |
self.logger.error("Hashes differ.. aborting") |
|
287 |
sys.exit(-1) |
|
288 |
|
|
289 |
@_check_fabric |
|
290 |
def clone_repo(self): |
|
291 |
"""Clone Synnefo repo from slave server""" |
|
292 |
self.logger.info("Configure repositories on remote server..") |
|
293 |
self.logger.debug("Setup apt, install curl and git") |
|
294 |
cmd = """ |
|
295 |
echo 'APT::Install-Suggests "false";' >> /etc/apt/apt.conf |
|
296 |
apt-get update |
|
297 |
apt-get install curl git --yes |
|
298 |
echo -e "\n\ndeb {0}" >> /etc/apt/sources.list |
|
299 |
curl https://dev.grnet.gr/files/apt-grnetdev.pub | apt-key add - |
|
300 |
apt-get update |
|
301 |
git config --global user.name {1} |
|
302 |
git config --global user.mail {2} |
|
303 |
""".format(self.config.get('Global', 'apt_repo'), |
|
304 |
self.config.get('Global', 'git_config_name'), |
|
305 |
self.config.get('Global', 'git_config_mail')) |
|
306 |
_run(cmd, False) |
|
307 |
|
|
308 |
synnefo_repo = self.config.get('Global', 'synnefo_repo') |
|
309 |
# Currently clonning synnefo can fail unexpectedly |
|
310 |
for i in range(3): |
|
311 |
self.logger.debug("Clone synnefo from %s" % synnefo_repo) |
|
312 |
try: |
|
313 |
_run("git clone %s" % synnefo_repo, False) |
|
314 |
break |
|
315 |
except: |
|
316 |
self.logger.warning("Clonning synnefo failed.. retrying %s" % i) |
|
317 |
|
|
318 |
synnefo_branch = self.config.get('Global', 'synnefo_branch') |
|
319 |
if synnefo_branch == "HEAD": |
|
320 |
# Get current branch |
|
321 |
synnefo_branch = os.popen("git rev-parse HEAD").read().strip() |
|
322 |
self.logger.debug( |
|
323 |
"Checkout %s in feature-ci branch" % synnefo_branch) |
|
324 |
with fabric.cd("synnefo"): |
|
325 |
_run("git checkout -b feature-ci %s" % synnefo_branch, False) |
|
326 |
elif synnefo_branch == "origin/master": |
|
327 |
pass |
|
328 |
elif "origin" in synnefo_branch: |
|
329 |
self.logger.debug("Checkout %s branch" % synnefo_branch) |
|
330 |
with fabric.cd("synnefo"): |
|
331 |
_run("git checkout -t %s" % synnefo_branch, False) |
|
332 |
else: |
|
333 |
self.logger.debug( |
|
334 |
"Checkout %s in feature-ci branch" % synnefo_branch) |
|
335 |
with fabric.cd("synnefo"): |
|
336 |
_run("git checkout -b feature-ci %s" % synnefo_branch, False) |
|
337 |
|
|
338 |
deploy_repo = self.config.get('Global', 'deploy_repo') |
|
339 |
self.logger.debug("Clone snf-deploy from %s" % deploy_repo) |
|
340 |
_run("git clone %s" % deploy_repo, False) |
|
341 |
|
|
342 |
@_check_fabric |
|
343 |
def build_synnefo(self): |
|
344 |
"""Build Synnefo packages""" |
|
345 |
self.logger.info("Build Synnefo packages..") |
|
346 |
self.logger.debug("Install development packages") |
|
347 |
cmd = """ |
|
348 |
apt-get update |
|
349 |
apt-get install zlib1g-dev dpkg-dev debhelper git-buildpackage \ |
|
350 |
python-dev python-all python-pip --yes |
|
351 |
pip install devflow |
|
352 |
""" |
|
353 |
_run(cmd, False) |
|
354 |
|
|
355 |
if eval(self.config.get('Global', 'patch_pydist')): |
|
356 |
self.logger.debug("Patch pydist.py module") |
|
357 |
cmd = r""" |
|
358 |
sed -r -i 's/(\(\?P<name>\[A-Za-z\]\[A-Za-z0-9_\.)/\1\\\-/' \ |
|
359 |
/usr/share/python/debpython/pydist.py |
|
360 |
""" |
|
361 |
_run(cmd, False) |
|
362 |
|
|
363 |
self.logger.debug("Build snf-deploy package") |
|
364 |
cmd = """ |
|
365 |
git checkout -t origin/debian |
|
366 |
git-buildpackage --git-upstream-branch=master \ |
|
367 |
--git-debian-branch=debian \ |
|
368 |
--git-export-dir=../snf-deploy_build-area \ |
|
369 |
-uc -us |
|
370 |
""" |
|
371 |
with fabric.cd("snf-deploy"): |
|
372 |
_run(cmd, True) |
|
373 |
|
|
374 |
self.logger.debug("Install snf-deploy package") |
|
375 |
cmd = """ |
|
376 |
dpkg -i snf-deploy*.deb |
|
377 |
apt-get -f install --yes |
|
378 |
""" |
|
379 |
with fabric.cd("snf-deploy_build-area"): |
|
380 |
with fabric.settings(warn_only=True): |
|
381 |
_run(cmd, True) |
|
382 |
|
|
383 |
self.logger.debug("Build synnefo packages") |
|
384 |
cmd = """ |
|
385 |
devflow-autopkg snapshot -b ~/synnefo_build-area --no-sign |
|
386 |
""" |
|
387 |
with fabric.cd("synnefo"): |
|
388 |
_run(cmd, True) |
|
389 |
|
|
390 |
self.logger.debug("Copy synnefo debs to snf-deploy packages dir") |
|
391 |
cmd = """ |
|
392 |
cp ~/synnefo_build-area/*.deb /var/lib/snf-deploy/packages/ |
|
393 |
""" |
|
394 |
_run(cmd, False) |
|
395 |
|
|
396 |
@_check_fabric |
|
397 |
def deploy_synnefo(self): |
|
398 |
"""Deploy Synnefo using snf-deploy""" |
|
399 |
self.logger.info("Deploy Synnefo..") |
|
400 |
schema = self.config.get('Global', 'schema') |
|
401 |
schema_files = os.path.join(self.ci_dir, "schemas/%s/*" % schema) |
|
402 |
self.logger.debug("Will use %s schema" % schema) |
|
403 |
|
|
404 |
self.logger.debug("Upload schema files to server") |
|
405 |
with fabric.quiet(): |
|
406 |
fabric.put(schema_files, "/etc/snf-deploy/") |
|
407 |
|
|
408 |
self.logger.debug("Change password in nodes.conf file") |
|
409 |
cmd = """ |
|
410 |
sed -i 's/^password =.*/password = {0}/' /etc/snf-deploy/nodes.conf |
|
411 |
""".format(fabric.env.password) |
|
412 |
_run(cmd, False) |
|
413 |
|
|
414 |
self.logger.debug("Run snf-deploy") |
|
415 |
cmd = """ |
|
416 |
snf-deploy all --autoconf |
|
417 |
""" |
|
418 |
_run(cmd, True) |
|
419 |
|
|
420 |
@_check_fabric |
|
421 |
def unit_test(self): |
|
422 |
"""Run Synnefo unit test suite""" |
|
423 |
self.logger.info("Run Synnefo unit test suite") |
|
424 |
component = self.config.get('Unit Tests', 'component') |
|
425 |
|
|
426 |
self.logger.debug("Install needed packages") |
|
427 |
cmd = """ |
|
428 |
pip install mock |
|
429 |
pip install factory_boy |
|
430 |
""" |
|
431 |
_run(cmd, False) |
|
432 |
|
|
433 |
self.logger.debug("Upload local_unit_tests.sh file") |
|
434 |
unit_tests_file = os.path.join(self.ci_dir, "local_unit_tests.sh") |
|
435 |
with fabric.quiet(): |
|
436 |
fabric.put(unit_tests_file, ".") |
|
437 |
|
|
438 |
self.logger.debug("Run unit tests") |
|
439 |
cmd = """ |
|
440 |
bash local_unit_tests.sh {0} |
|
441 |
""".format(component) |
|
442 |
_run(cmd, True) |
|
443 |
|
|
444 |
@_check_fabric |
|
445 |
def run_burnin(self): |
|
446 |
"""Run burnin functional test suite""" |
|
447 |
self.logger.info("Run Burnin functional test suite") |
|
448 |
cmd = """ |
|
449 |
auth_url=$(grep -e '^url =' .kamakirc | cut -d' ' -f3) |
|
450 |
token=$(grep -e '^token =' .kamakirc | cut -d' ' -f3) |
|
451 |
images_user=$(kamaki image list -l | grep owner | \ |
|
452 |
cut -d':' -f2 | tr -d ' ') |
|
453 |
snf-burnin --auth-url=$auth_url --token=$token \ |
|
454 |
--force-flavor=2 --image-id=all \ |
|
455 |
--system-images-user=$images_user \ |
|
456 |
{0} |
|
457 |
log_folder=$(ls -1d /var/log/burnin/* | tail -n1) |
|
458 |
for i in $(ls $log_folder/*/details*); do |
|
459 |
echo -e "\\n\\n" |
|
460 |
echo -e "***** $i\\n" |
|
461 |
cat $i |
|
462 |
done |
|
463 |
""".format(self.config.get('Burnin', 'cmd_options')) |
|
464 |
_run(cmd, True) |
|
465 |
|
|
466 |
@_check_fabric |
|
467 |
def fetch_packages(self): |
|
468 |
"""Download Synnefo packages""" |
|
469 |
self.logger.info("Download Synnefo packages") |
|
470 |
self.logger.debug("Create tarball with packages") |
|
471 |
cmd = """ |
|
472 |
tar czf synnefo_build-area.tgz synnefo_build-area |
|
473 |
""" |
|
474 |
_run(cmd, False) |
|
475 |
|
|
476 |
pkgs_dir = self.config.get('Global', 'pkgs_dir') |
|
477 |
self.logger.debug("Fetch packages to local dir %s" % pkgs_dir) |
|
478 |
os.makedirs(pkgs_dir) |
|
479 |
with fabric.quiet(): |
|
480 |
fabric.get("synnefo_build-area.tgz", pkgs_dir) |
|
481 |
|
|
482 |
pkgs_file = os.path.join(pkgs_dir, "synnefo_build-area.tgz") |
|
483 |
self._check_hash_sum(pkgs_file, "synnefo_build-area.tgz") |
|
484 |
|
|
485 |
self.logger.debug("Untar packages file %s" % pkgs_file) |
|
486 |
os.system("cd %s; tar xzf synnefo_build-area.tgz" % pkgs_dir) |
Also available in: Unified diff