Revision 525f2979

b/ci/autopkg_debian.py
1
#!/usr/bin/env python
2

  
3
"""
4
Build Synnefo packages for debian
5
"""
6

  
7
from utils import SynnefoCI
8

  
9

  
10
def autopkg_debian():
11
    """Build synnefo packages for debian"""
12
    synnefo_ci = SynnefoCI()
13
    synnefo_ci.build_synnefo()
14

  
15

  
16
if __name__ == "__main__":
17
    autopkg_debian()
b/ci/deploy.py
1
#!/usr/bin/env python
2

  
3
"""
4
Deploy Synnefo using snf-deploy
5
"""
6

  
7
from utils import SynnefoCI
8

  
9

  
10
def deploy_synnefo():
11
    """Deploy Synnefo using snf-deploy"""
12
    synnefo_ci = SynnefoCI()
13
    synnefo_ci.deploy_synnefo()
14

  
15

  
16
if __name__ == "__main__":
17
    deploy_synnefo()
b/ci/fetch_packages.py
1
#!/usr/bin/env python
2

  
3
"""
4
Download Synnefo packages
5
"""
6

  
7
from utils import SynnefoCI
8

  
9

  
10
def fetch_packages():
11
    """Download Synnefo packages"""
12
    synnefo_ci = SynnefoCI()
13
    synnefo_ci.fetch_packages()
14

  
15

  
16
if __name__ == "__main__":
17
    fetch_packages()
b/ci/new_config
1
[Global]
2
# Timeouts in seconds
3
build_timeout = 240
4
# Apt repository to use
5
apt_repo = http://apt.dev.grnet.gr squeeze/
6
# Synnefo git repo
7
synnefo_repo = https://code.grnet.gr/git/synnefo
8
# Git branch to test (specify sha1 or branch name)
9
synnefo_branch = HEAD
10
# snf-deploy git repo
11
deploy_repo = https://code.grnet.gr/git/snf-deploy
12
# Defines the schema that snf-deploy will use
13
schema = one_node
14
# Local dir to save builded packages
15
pkgs_dir = /tmp/synnefo_pkgs
16
# If True patch the pydist.py module (see Debian bug #657665)
17
patch_pydist = True
18
# Configuration of git (on remote server)
19
git_config_name = Buildbot
20
git_config_mail = synnefo@builder.dev.grnet.gr
21
# Url to fetch ssh public keys
22
public_ssh_keys_url =
23
# Network address from which we allow access to server
24
filter_access_network = 195.251.29.0/24,62.217.123.39
25
# Config file to save temporary options (eg IPs, passwords etc)
26
temporary_config = /tmp/ci_temp_conf
27

  
28

  
29
[Deployment]
30
# Credentials
31
auth_url = https://accounts.okeanos.io/identity/v2.0/
32
token =
33
# If we deploy on okeanos.io we have to set this to True
34
# The server will reside besides a NAT and we have to compute ssh port
35
deploy_on_io = True
36
# Server name to use for our machine
37
server_name = Synnefo Deployment
38
# Flavor ID to use
39
# (149 for production, 639 for okeanos.io)
40
flavor_id = 639
41
# Image to use (name must contain this)
42
image_name = OldStable
43
# UUID of owner of system images
44
# (25ecced9-bf53-4145-91ee-cf47377e9fb2 for production,
45
#  04cbe33f-29b7-4ef1-94fb-015929e5fc06 for okeanos.io)
46
system_uuid = 04cbe33f-29b7-4ef1-94fb-015929e5fc06
47

  
48

  
49
[Burnin]
50
# Maybe add some burnin options
51
# (e.g. tests to run/ignore, timeouts etc)
52
cmd_options = --nofailfast --no-ipv6 --action-timeout=240
53

  
54

  
55
[Unit Tests]
56
component = astakos cyclades pithos
57

  
58

  
59
[Repository]
60
# Projects reside on this repo
61
projects =
62
    snf-common
63
    astakosclient
64
    snf-django-lib
65
    snf-webproject
66
    snf-branding
67
    snf-astakos-app
68
    snf-pithos-backend
69
    snf-cyclades-gtools
70
    snf-cyclades-app
71
    snf-pithos-app
72
    snf-tools
b/ci/run_burnin.py
1
#!/usr/bin/env python
2

  
3
"""
4
Run burnin functional test suite
5
"""
6

  
7
from utils import SynnefoCI
8

  
9

  
10
def run_burnin():
11
    """Run burnin functional test suite"""
12
    synnefo_ci = SynnefoCI()
13
    synnefo_ci.run_burnin()
14

  
15

  
16
if __name__ == "__main__":
17
    run_burnin()
b/ci/schemas/one_node/deploy.conf
1
[packages]
2
# whether to use apt-get or local generated package found in packages dir
3
use_local_packages = True
4

  
5
# url to obtain latest synnefo packages.
6
# To use them change USE_LOCAL_PACKAGES setting to yes
7
# To get them run: snf-deploy packages
8
package_url = http://builder.dev.grnet.gr/synnefo/packages/Squeeze/40/
9

  
10
[dirs]
11
# dir to find all template files used to customize setup
12
# in case you want to add another setting please modify the corresponding file
13
templates = /var/lib/snf-deploy/files
14
# dir to store local images (disk0, disk1 of the virtual cluster)
15
images = /var/lib/snf-deploy/images
16
# dir to store/find local packages
17
# dir to locally save packages that will be downloaded from package_url
18
# put here any locally created packages (useful for development)
19
packages = /var/lib/snf-deploy/packages
20
# dir to store pidfiles (dnsmasq, kvm)
21
run = /var/run/snf-deploy
22
# dir to store dnsmasq related files
23
dns = /var/lib/snf-deploy/dnsmasq
24
# dir to lookup fabfile and ifup script
25
lib = /usr/lib/snf-deploy
26
# dir to store executed commands (to enforce sequential execution)
27
cmd = /var/run/snf-deploy/cmd
b/ci/schemas/one_node/ganeti.conf
1
[ganeti1]
2
cluster_nodes = node1
3
master_node = node1
4

  
5
cluster_netdev = eth0
6
cluster_name = ganeti1
7
cluster_ip = 192.168.0.13
8

  
9
vg = autovg
10

  
11
synnefo_public_network_subnet = 10.2.1.0/24
12
synnefo_public_network_gateway = 10.2.1.1
13
synnefo_public_network_type = CUSTOM
14

  
15
image_dir = /srv/okeanos
16

  
17
# To add another cluster repeat the above section
18
# with different header and nodes
b/ci/schemas/one_node/nodes.conf
1
# please note that currently is only supported deployment
2
# with nodes (both ganeti and synnefo) residing in the same subnet/domain
3
[network]
4
domain = synnefo.live
5
subnet = 192.168.0.0/28
6
gateway = 192.168.0.14
7

  
8

  
9
[hostnames]
10
node1 = auto1
11
# node2 = auto2
12

  
13
[ips]
14
node1 = 192.168.0.1
15
# node2 = 192.168.0.2
16

  
17
# This is used only in case of vcluster
18
# needed to pass the correct dhcp responces to the virtual nodes
19
[macs]
20
node1 = 52:54:00:00:00:01
21
# node2 = 52:54:00:00:00:02
22

  
23
[info]
24
# Here we define which nodes from the predefined ones to use
25
nodes = node1
26

  
27
# login credentials for the nodes
28
# please note that in case of vcluster these are preconfigured
29
# and not editable.
30
# in case of physical nodes all nodes should have the same login account
31
user = root
32
password = 12345
33

  
34
public_iface = eth0
35
vm_public_iface = eth1
36
vm_private_iface = eth2
37

  
38
# extra disk name inside the nodes
39
# if defined, snf-deploy will create a VG for ganeti in order to support lvm storage
40
# if not then only file disk template will be supported
41
extra_disk = /dev/vdb
b/ci/schemas/one_node/packages.conf
1
[debian]
2
rabbitmq-server = testing
3
gunicorn = squeeze-backports
4
qemu-kvm = squeeze-backports
5
qemu = squeeze-backports
6
python-gevent = squeeze-backports
7
apache2 =
8
postgresql =
9
python-psycopg2 =
10
python-argparse =
11
nfs-kernel-server = squeeze-backports
12
nfs-common = squeeze-backports
13
bind9 =
14
vlan =
15
vlan =
16
lvm2 =
17
curl =
18
memcached =
19
python-memcache =
20
bridge-utils =
21
python-progress =
22
ganeti-instance-debootstrap =
23

  
24

  
25
[synnefo]
26
snf-astakos-app = stable
27
snf-common = stable
28
snf-cyclades-app = stable
29
snf-cyclades-gtools = stable
30
snf-django-lib = stable
31
python-astakosclient = stable
32
python-objpool = stable
33
snf-branding = stable
34
snf-webproject = stable
35
snf-pithos-app = stable
36
snf-pithos-backend = stable
37
snf-tools = stable
38

  
39

  
40
[ganeti]
41
snf-ganeti = 2.6.2+ippool11+hotplug5+extstorage3+rbdfix1+kvmfix2+nolvm+netxen-1~squeeze
42
ganeti-htools = 2.6.2+ippool11+hotplug5+extstorage3+rbdfix1+kvmfix2+nolvm+netxen-1~squeeze
43

  
44
[other]
45
snf-cloudcms = stable
46
snf-vncauthproxy = stable
47
snf-pithos-webclient = stable
48
snf-image = stable
49
snf-network = stable
50
nfdhcpd = stable
51
kamaki = stable
52
python-bitarray = stable
53
nfqueue-bindings-python = stable
54

  
b/ci/schemas/one_node/synnefo.conf
1
[cred]
2
synnefo_user = synnefo
3
synnefo_db_passwd = example_passw0rd
4
synnefo_rapi_passwd = example_rapi_passw0rd
5
synnefo_rabbitmq_passwd = example_rabbitmq_passw0rd
6
user_email = user@synnefo.org
7
user_name = John
8
user_lastname = Doe
9
user_passwd = 12345
10

  
11

  
12
[roles]
13
accounts = node1
14
compute = node1
15
object-store = node1
16
cyclades = node1
17
pithos = node1
18
cms = node1
19
db = node1
20
mq = node1
21
ns = node1
22
client = node1
23
router = node1
24

  
25

  
26
[synnefo]
27
pithos_dir = /srv/pithos
28

  
29
vm_public_bridge = br0
30
vm_private_bridge = prv0
31
common_bridge = br0
32

  
33
debian_base_url = https://pithos.okeanos.grnet.gr/public/RDISy7sNVIJ9KIm4JkmbX4
b/ci/schemas/one_node/vcluster.conf
1
[image]
2
# url to get the base image. This is a debian base image with preconfigured
3
# root password and installed rsa/dsa keys. Plus a NetworkManager hook that
4
# changes the VM's name based on info provided by dhcp response.
5
# To create it run: snf-deploy image
6
squeeze_image_url = https://pithos.okeanos.grnet.gr/public/832xv
7
ubuntu_image_url =
8

  
9
# in order ganeti nodes to support lvm storage (plain disk template) it will
10
# be needed an extra disk to eventually be able to create a VG. Ganeti requires
11
# this VG to be at least of 30GB. To this end in order the virtual nodes to have
12
# this extra disk an image should be created locally. There are three options:
13
# 1. not create an extra disk (only file storage template will be supported)
14
# 2. create an image of 30G in image dir (default /var/lib/snf-deploy/images)
15
#    using dd if=/dev/zero of=squeeze.disk1
16
# 3. create this image in a local VG using lvgreate -L30G squeeze.disk1 lvg
17
#    and create a symbolic link in /var/lib/snf-deploy/images
18

  
19
# Whether to create an extra disk or not
20
create_extra_disk = False
21
# lvg is the name of the local VG if any
22
lvg =
23

  
24
# OS istalled in the virtual cluster
25
os = squeeze
26

  
27

  
28
[cluster]
29
# the bridge to use for the virtual cluster
30
# on this bridge we will launch a dnsnmasq and provide
31
# fqdns needed to the cluster.
32
# In ordrer cluster nodes to have internet access, host must do NAT.
33
# iptables -t nat -A POSTROUTING -s 192.0.0.0/28 -j MASQUERADE
34
# ip addr add 192.0.0.14/28 dev auto_nodes_br
35
# To create run: snf-deploy cluster
36
bridge = auto_nodes_br
b/ci/setup_slave.py
1
#!/usr/bin/env python
2

  
3
"""
4
Setup slave server
5
"""
6

  
7
from utils import SynnefoCI
8

  
9

  
10
def setup_slave():
11
    """Setup slave server"""
12
    synnefo_ci = SynnefoCI(cleanup_config=True)
13
    # Get token from /nfs/token
14
    try:
15
        token = open("/nfs/synnefo_token").read().strip()
16
        synnefo_ci.write_config('token', token, 'Deployment')
17
    except:
18
        pass
19
    # Build slave server
20
    synnefo_ci.create_server()
21
    # Copy synnefo repo to server
22
    synnefo_ci.clone_repo()
23

  
24

  
25
if __name__ == "__main__":
26
    setup_slave()
b/ci/unit_tests.py
1
#!/usr/bin/env python
2

  
3
"""
4
Run Synnefo unit test suite
5
"""
6

  
7
from utils import SynnefoCI
8

  
9

  
10
def unit_test():
11
    """Run Synnefo unit test suite"""
12
    synnefo_ci = SynnefoCI()
13
    synnefo_ci.unit_test()
14

  
15

  
16
if __name__ == "__main__":
17
    unit_test()
b/ci/utils.py
1
#!/usr/bin/env python
2

  
3
"""
4
Synnefo ci utils module
5
"""
6

  
7
import os
8
import sys
9
import time
10
import logging
11
import fabric.api as fabric
12
from ConfigParser import ConfigParser, DuplicateSectionError
13

  
14
from kamaki.clients.astakos import AstakosClient
15
from kamaki.clients.cyclades import CycladesClient
16
from kamaki.clients.image import ImageClient
17

  
18

  
19
def _run(cmd, verbose):
20
    """Run fabric with verbose level"""
21
    if verbose:
22
        args = ('running',)
23
    else:
24
        args = ('running', 'stdout',)
25
    with fabric.hide(*args):
26
        return fabric.run(cmd)
27

  
28

  
29
def _red(msg):
30
    """Red color"""
31
    #return "\x1b[31m" + str(msg) + "\x1b[0m"
32
    return str(msg)
33

  
34

  
35
def _yellow(msg):
36
    """Yellow color"""
37
    #return "\x1b[33m" + str(msg) + "\x1b[0m"
38
    return str(msg)
39

  
40

  
41
def _green(msg):
42
    """Green color"""
43
    #return "\x1b[32m" + str(msg) + "\x1b[0m"
44
    return str(msg)
45

  
46

  
47
def _check_fabric(fun):
48
    """Check if fabric env has been set"""
49
    def wrapper(self, *args):
50
        """wrapper function"""
51
        if not self.fabric_installed:
52
            self.setup_fabric()
53
        return fun(self, *args)
54
    return wrapper
55

  
56

  
57
def _check_kamaki(fun):
58
    """Check if kamaki has been initialized"""
59
    def wrapper(self, *args):
60
        """wrapper function"""
61
        if not self.kamaki_installed:
62
            self.setup_kamaki()
63
        return fun(self, *args)
64
    return wrapper
65

  
66

  
67
class _MyFormatter(logging.Formatter):
68
    """Logging Formatter"""
69
    def format(self, record):
70
        format_orig = self._fmt
71
        if record.levelno == logging.DEBUG:
72
            self._fmt = "  %(msg)s"
73
        elif record.levelno == logging.INFO:
74
            self._fmt = "%(msg)s"
75
        elif record.levelno == logging.WARNING:
76
            self._fmt = _yellow("[W] %(msg)s")
77
        elif record.levelno == logging.ERROR:
78
            self._fmt = _red("[E] %(msg)s")
79
        result = logging.Formatter.format(self, record)
80
        self._fmt = format_orig
81
        return result
82

  
83

  
84
class SynnefoCI(object):
85
    """SynnefoCI python class"""
86

  
87
    def __init__(self, cleanup_config=False):
88
        """ Initialize SynnefoCI python class
89

  
90
        Setup logger, local_dir, config and kamaki
91
        """
92
        # Setup logger
93
        self.logger = logging.getLogger('synnefo-ci')
94
        self.logger.setLevel(logging.DEBUG)
95
        handler = logging.StreamHandler()
96
        handler.setFormatter(_MyFormatter())
97
        self.logger.addHandler(handler)
98

  
99
        # Get our local dir
100
        self.ci_dir = os.path.dirname(os.path.abspath(__file__))
101
        self.repo_dir = os.path.dirname(self.ci_dir)
102

  
103
        # Read config file
104
        self.conffile = os.path.join(self.ci_dir, "new_config")
105
        self.config = ConfigParser()
106
        self.config.optionxform = str
107
        self.config.read(self.conffile)
108
        temp_config = self.config.get('Global', 'temporary_config')
109
        if cleanup_config:
110
            try:
111
                os.remove(temp_config)
112
            except:
113
                pass
114
        else:
115
            self.config.read(self.config.get('Global', 'temporary_config'))
116

  
117
        # Initialize variables
118
        self.fabric_installed = False
119
        self.kamaki_installed = False
120
        self.cyclades_client = None
121
        self.image_client = None
122

  
123
    def setup_kamaki(self):
124
        """Initialize kamaki
125

  
126
        Setup cyclades_client and image_client
127
        """
128
        self.logger.info("Setup kamaki client..")
129
        auth_url = self.config.get('Deployment', 'auth_url')
130
        self.logger.debug("Authentication URL is %s" % _green(auth_url))
131
        token = self.config.get('Deployment', 'token')
132
        #self.logger.debug("Token is %s" % _green(token))
133

  
134
        astakos_client = AstakosClient(auth_url, token)
135

  
136
        cyclades_url = \
137
            astakos_client.get_service_endpoints('compute')['publicURL']
138
        self.logger.debug("Cyclades API url is %s" % _green(cyclades_url))
139
        self.cyclades_client = CycladesClient(cyclades_url, token)
140
        self.cyclades_client.CONNECTION_RETRY_LIMIT = 2
141

  
142
        image_url = \
143
            astakos_client.get_service_endpoints('image')['publicURL']
144
        self.logger.debug("Images API url is %s" % _green(image_url))
145
        self.image_client = ImageClient(cyclades_url, token)
146
        self.image_client.CONNECTION_RETRY_LIMIT = 2
147

  
148
    def _wait_transition(self, server_id, current_status, new_status):
149
        """Wait for server to go from current_status to new_status"""
150
        self.logger.debug("Waiting for server to become %s" % new_status)
151
        timeout = self.config.getint('Global', 'build_timeout')
152
        sleep_time = 5
153
        while True:
154
            server = self.cyclades_client.get_server_details(server_id)
155
            if server['status'] == new_status:
156
                return server
157
            elif timeout < 0:
158
                self.logger.error(
159
                    "Waiting for server to become %s timed out" % new_status)
160
                self.destroy_server(False)
161
                sys.exit(-1)
162
            elif server['status'] == current_status:
163
                # Sleep for #n secs and continue
164
                timeout = timeout - sleep_time
165
                time.sleep(sleep_time)
166
            else:
167
                self.logger.error(
168
                    "Server failed with status %s" % server['status'])
169
                self.destroy_server(False)
170
                sys.exit(-1)
171

  
172
    @_check_kamaki
173
    def destroy_server(self, wait=True):
174
        """Destroy slave server"""
175
        server_id = self.config.getint('Temporary Options', 'server_id')
176
        self.logger.info("Destoying server with id %s " % server_id)
177
        self.cyclades_client.delete_server(server_id)
178
        if wait:
179
            self._wait_transition(server_id, "ACTIVE", "DELETED")
180

  
181
    @_check_kamaki
182
    def create_server(self):
183
        """Create slave server"""
184
        self.logger.info("Create a new server..")
185
        image = self._find_image()
186
        self.logger.debug("Will use image \"%s\"" % _green(image['name']))
187
        self.logger.debug("Image has id %s" % _green(image['id']))
188
        server = self.cyclades_client.create_server(
189
            self.config.get('Deployment', 'server_name'),
190
            self.config.getint('Deployment', 'flavor_id'),
191
            image['id'])
192
        server_id = server['id']
193
        self.write_config('server_id', server_id)
194
        self.logger.debug("Server got id %s" % _green(server_id))
195
        server_user = server['metadata']['users']
196
        self.write_config('server_user', server_user)
197
        self.logger.debug("Server's admin user is %s" % _green(server_user))
198
        server_passwd = server['adminPass']
199
        self.write_config('server_passwd', server_passwd)
200
        self.logger.debug(
201
            "Server's admin password is %s" % _green(server_passwd))
202

  
203
        server = self._wait_transition(server_id, "BUILD", "ACTIVE")
204
        self._get_server_ip_and_port(server)
205

  
206
        self.setup_fabric()
207
        self.logger.info("Setup firewall")
208
        accept_ssh_from = self.config.get('Global', 'filter_access_network')
209
        self.logger.debug("Block ssh except from %s" % accept_ssh_from)
210
        cmd = """
211
        iptables -A INPUT -s localhost -j ACCEPT
212
        iptables -A INPUT -s {0} -p tcp --dport 22 -j ACCEPT
213
        iptables -A INPUT -p tcp --dport 22 -j DROP
214
        """.format(accept_ssh_from)
215
        _run(cmd, False)
216

  
217
    def _find_image(self):
218
        """Find a suitable image to use
219

  
220
        It has to belong to the `system_uuid' user and
221
        contain the word `image_name'
222
        """
223
        system_uuid = self.config.get('Deployment', 'system_uuid')
224
        image_name = self.config.get('Deployment', 'image_name').lower()
225
        images = self.image_client.list_public(detail=True)['images']
226
        # Select images by `system_uuid' user
227
        images = [x for x in images if x['user_id'] == system_uuid]
228
        # Select images with `image_name' in their names
229
        images = \
230
            [x for x in images if x['name'].lower().find(image_name) != -1]
231
        # Let's select the first one
232
        return images[0]
233

  
234
    def _get_server_ip_and_port(self, server):
235
        """Compute server's IPv4 and ssh port number"""
236
        self.logger.info("Get server connection details..")
237
        # XXX: check if this IP is from public network
238
        server_ip = server['attachments'][0]['ipv4']
239
        if eval(self.config.get('Deployment', 'deploy_on_io')):
240
            tmp1 = int(server_ip.split(".")[2])
241
            tmp2 = int(server_ip.split(".")[3])
242
            server_ip = "gate.okeanos.io"
243
            server_port = 10000 + tmp1 * 256 + tmp2
244
        else:
245
            server_port = 22
246
        self.write_config('server_ip', server_ip)
247
        self.logger.debug("Server's IPv4 is %s" % _green(server_ip))
248
        self.write_config('server_port', server_port)
249
        self.logger.debug("Server's ssh port is %s" % _green(server_port))
250

  
251
    def write_config(self, option, value, section="Temporary Options"):
252
        """Write changes back to config file"""
253
        try:
254
            self.config.add_section(section)
255
        except DuplicateSectionError:
256
            pass
257
        self.config.set(section, option, str(value))
258
        temp_conf_file = self.config.get('Global', 'temporary_config')
259
        with open(temp_conf_file, 'wb') as tcf:
260
            self.config.write(tcf)
261

  
262
    def setup_fabric(self):
263
        """Setup fabric environment"""
264
        self.logger.info("Setup fabric parameters..")
265
        fabric.env.user = self.config.get('Temporary Options', 'server_user')
266
        fabric.env.host_string = \
267
            self.config.get('Temporary Options', 'server_ip')
268
        fabric.env.port = self.config.getint('Temporary Options', 'server_port')
269
        fabric.env.password = \
270
            self.config.get('Temporary Options', 'server_passwd')
271
        fabric.env.connection_attempts = 10
272
        fabric.env.shell = "/bin/bash -c"
273
        fabric.env.disable_known_hosts = True
274
        fabric.env.output_prefix = None
275

  
276
    def _check_hash_sum(self, localfile, remotefile):
277
        """Check hash sums of two files"""
278
        self.logger.debug("Check hash sum for local file %s" % localfile)
279
        hash1 = os.popen("sha256sum %s" % localfile).read().split(' ')[0]
280
        self.logger.debug("Local file has sha256 hash %s" % hash1)
281
        self.logger.debug("Check hash sum for remote file %s" % remotefile)
282
        hash2 = _run("sha256sum %s" % remotefile, False)
283
        hash2 = hash2.split(' ')[0]
284
        self.logger.debug("Remote file has sha256 hash %s" % hash2)
285
        if hash1 != hash2:
286
            self.logger.error("Hashes differ.. aborting")
287
            sys.exit(-1)
288

  
289
    @_check_fabric
290
    def clone_repo(self):
291
        """Clone Synnefo repo from slave server"""
292
        self.logger.info("Configure repositories on remote server..")
293
        self.logger.debug("Setup apt, install curl and git")
294
        cmd = """
295
        echo 'APT::Install-Suggests "false";' >> /etc/apt/apt.conf
296
        apt-get update
297
        apt-get install curl git --yes
298
        echo -e "\n\ndeb {0}" >> /etc/apt/sources.list
299
        curl https://dev.grnet.gr/files/apt-grnetdev.pub | apt-key add -
300
        apt-get update
301
        git config --global user.name {1}
302
        git config --global user.mail {2}
303
        """.format(self.config.get('Global', 'apt_repo'),
304
                   self.config.get('Global', 'git_config_name'),
305
                   self.config.get('Global', 'git_config_mail'))
306
        _run(cmd, False)
307

  
308
        synnefo_repo = self.config.get('Global', 'synnefo_repo')
309
        # Currently clonning synnefo can fail unexpectedly
310
        for i in range(3):
311
            self.logger.debug("Clone synnefo from %s" % synnefo_repo)
312
            try:
313
                _run("git clone %s" % synnefo_repo, False)
314
                break
315
            except:
316
                self.logger.warning("Clonning synnefo failed.. retrying %s" % i)
317

  
318
        synnefo_branch = self.config.get('Global', 'synnefo_branch')
319
        if synnefo_branch == "HEAD":
320
            # Get current branch
321
            synnefo_branch = os.popen("git rev-parse HEAD").read().strip()
322
            self.logger.debug(
323
                "Checkout %s in feature-ci branch" % synnefo_branch)
324
            with fabric.cd("synnefo"):
325
                _run("git checkout -b feature-ci %s" % synnefo_branch, False)
326
        elif synnefo_branch == "origin/master":
327
            pass
328
        elif "origin" in synnefo_branch:
329
            self.logger.debug("Checkout %s branch" % synnefo_branch)
330
            with fabric.cd("synnefo"):
331
                _run("git checkout -t %s" % synnefo_branch, False)
332
        else:
333
            self.logger.debug(
334
                "Checkout %s in feature-ci branch" % synnefo_branch)
335
            with fabric.cd("synnefo"):
336
                _run("git checkout -b feature-ci %s" % synnefo_branch, False)
337

  
338
        deploy_repo = self.config.get('Global', 'deploy_repo')
339
        self.logger.debug("Clone snf-deploy from %s" % deploy_repo)
340
        _run("git clone %s" % deploy_repo, False)
341

  
342
    @_check_fabric
343
    def build_synnefo(self):
344
        """Build Synnefo packages"""
345
        self.logger.info("Build Synnefo packages..")
346
        self.logger.debug("Install development packages")
347
        cmd = """
348
        apt-get update
349
        apt-get install zlib1g-dev dpkg-dev debhelper git-buildpackage \
350
                python-dev python-all python-pip --yes
351
        pip install devflow
352
        """
353
        _run(cmd, False)
354

  
355
        if eval(self.config.get('Global', 'patch_pydist')):
356
            self.logger.debug("Patch pydist.py module")
357
            cmd = r"""
358
            sed -r -i 's/(\(\?P<name>\[A-Za-z\]\[A-Za-z0-9_\.)/\1\\\-/' \
359
                /usr/share/python/debpython/pydist.py
360
            """
361
            _run(cmd, False)
362

  
363
        self.logger.debug("Build snf-deploy package")
364
        cmd = """
365
        git checkout -t origin/debian
366
        git-buildpackage --git-upstream-branch=master \
367
                --git-debian-branch=debian \
368
                --git-export-dir=../snf-deploy_build-area \
369
                -uc -us
370
        """
371
        with fabric.cd("snf-deploy"):
372
            _run(cmd, True)
373

  
374
        self.logger.debug("Install snf-deploy package")
375
        cmd = """
376
        dpkg -i snf-deploy*.deb
377
        apt-get -f install --yes
378
        """
379
        with fabric.cd("snf-deploy_build-area"):
380
            with fabric.settings(warn_only=True):
381
                _run(cmd, True)
382

  
383
        self.logger.debug("Build synnefo packages")
384
        cmd = """
385
        devflow-autopkg snapshot -b ~/synnefo_build-area --no-sign
386
        """
387
        with fabric.cd("synnefo"):
388
            _run(cmd, True)
389

  
390
        self.logger.debug("Copy synnefo debs to snf-deploy packages dir")
391
        cmd = """
392
        cp ~/synnefo_build-area/*.deb /var/lib/snf-deploy/packages/
393
        """
394
        _run(cmd, False)
395

  
396
    @_check_fabric
397
    def deploy_synnefo(self):
398
        """Deploy Synnefo using snf-deploy"""
399
        self.logger.info("Deploy Synnefo..")
400
        schema = self.config.get('Global', 'schema')
401
        schema_files = os.path.join(self.ci_dir, "schemas/%s/*" % schema)
402
        self.logger.debug("Will use %s schema" % schema)
403

  
404
        self.logger.debug("Upload schema files to server")
405
        with fabric.quiet():
406
            fabric.put(schema_files, "/etc/snf-deploy/")
407

  
408
        self.logger.debug("Change password in nodes.conf file")
409
        cmd = """
410
        sed -i 's/^password =.*/password = {0}/' /etc/snf-deploy/nodes.conf
411
        """.format(fabric.env.password)
412
        _run(cmd, False)
413

  
414
        self.logger.debug("Run snf-deploy")
415
        cmd = """
416
        snf-deploy all --autoconf
417
        """
418
        _run(cmd, True)
419

  
420
    @_check_fabric
421
    def unit_test(self):
422
        """Run Synnefo unit test suite"""
423
        self.logger.info("Run Synnefo unit test suite")
424
        component = self.config.get('Unit Tests', 'component')
425

  
426
        self.logger.debug("Install needed packages")
427
        cmd = """
428
        pip install mock
429
        pip install factory_boy
430
        """
431
        _run(cmd, False)
432

  
433
        self.logger.debug("Upload local_unit_tests.sh file")
434
        unit_tests_file = os.path.join(self.ci_dir, "local_unit_tests.sh")
435
        with fabric.quiet():
436
            fabric.put(unit_tests_file, ".")
437

  
438
        self.logger.debug("Run unit tests")
439
        cmd = """
440
        bash local_unit_tests.sh {0}
441
        """.format(component)
442
        _run(cmd, True)
443

  
444
    @_check_fabric
445
    def run_burnin(self):
446
        """Run burnin functional test suite"""
447
        self.logger.info("Run Burnin functional test suite")
448
        cmd = """
449
        auth_url=$(grep -e '^url =' .kamakirc | cut -d' ' -f3)
450
        token=$(grep -e '^token =' .kamakirc | cut -d' ' -f3)
451
        images_user=$(kamaki image list -l | grep owner | \
452
                      cut -d':' -f2 | tr -d ' ')
453
        snf-burnin --auth-url=$auth_url --token=$token \
454
            --force-flavor=2 --image-id=all \
455
            --system-images-user=$images_user \
456
            {0}
457
        log_folder=$(ls -1d /var/log/burnin/* | tail -n1)
458
        for i in $(ls $log_folder/*/details*); do
459
            echo -e "\\n\\n"
460
            echo -e "***** $i\\n"
461
            cat $i
462
        done
463
        """.format(self.config.get('Burnin', 'cmd_options'))
464
        _run(cmd, True)
465

  
466
    @_check_fabric
467
    def fetch_packages(self):
468
        """Download Synnefo packages"""
469
        self.logger.info("Download Synnefo packages")
470
        self.logger.debug("Create tarball with packages")
471
        cmd = """
472
        tar czf synnefo_build-area.tgz synnefo_build-area
473
        """
474
        _run(cmd, False)
475

  
476
        pkgs_dir = self.config.get('Global', 'pkgs_dir')
477
        self.logger.debug("Fetch packages to local dir %s" % pkgs_dir)
478
        os.makedirs(pkgs_dir)
479
        with fabric.quiet():
480
            fabric.get("synnefo_build-area.tgz", pkgs_dir)
481

  
482
        pkgs_file = os.path.join(pkgs_dir, "synnefo_build-area.tgz")
483
        self._check_hash_sum(pkgs_file, "synnefo_build-area.tgz")
484

  
485
        self.logger.debug("Untar packages file %s" % pkgs_file)
486
        os.system("cd %s; tar xzf synnefo_build-area.tgz" % pkgs_dir)

Also available in: Unified diff