Statistics
| Branch: | Tag: | Revision:

root / ci / utils.py @ ac7b865d

History | View | Annotate | Download (40.9 kB)

1 09f7ad00 Ilias Tsitsimpis
#!/usr/bin/env python
2 c29ac11d Ilias Tsitsimpis
3 c29ac11d Ilias Tsitsimpis
"""
4 c29ac11d Ilias Tsitsimpis
Synnefo ci utils module
5 c29ac11d Ilias Tsitsimpis
"""
6 c29ac11d Ilias Tsitsimpis
7 c29ac11d Ilias Tsitsimpis
import os
8 ad21ffa9 Ilias Tsitsimpis
import re
9 c29ac11d Ilias Tsitsimpis
import sys
10 c29ac11d Ilias Tsitsimpis
import time
11 c29ac11d Ilias Tsitsimpis
import logging
12 c29ac11d Ilias Tsitsimpis
import fabric.api as fabric
13 b69b55ca Christos Stavrakakis
import subprocess
14 88e6558b Christos Stavrakakis
import tempfile
15 329705c8 Ilias Tsitsimpis
from ConfigParser import ConfigParser, DuplicateSectionError
16 c29ac11d Ilias Tsitsimpis
17 c1b1d444 Christos Stavrakakis
from kamaki.cli import config as kamaki_config
18 ee6eff28 Ilias Tsitsimpis
from kamaki.clients.astakos import AstakosClient, parse_endpoints
19 94c89c0e Ilias Tsitsimpis
from kamaki.clients.cyclades import CycladesClient, CycladesNetworkClient
20 c29ac11d Ilias Tsitsimpis
from kamaki.clients.image import ImageClient
21 ad21ffa9 Ilias Tsitsimpis
from kamaki.clients.compute import ComputeClient
22 49fb00cc Ilias Tsitsimpis
from kamaki.clients import ClientError
23 62fcf0e5 Ilias Tsitsimpis
import filelocker
24 c29ac11d Ilias Tsitsimpis
25 7585f464 Ilias Tsitsimpis
DEFAULT_CONFIG_FILE = "ci_wheezy.conf"
26 106659f1 Ilias Tsitsimpis
# Is our terminal a colorful one?
27 106659f1 Ilias Tsitsimpis
USE_COLORS = True
28 62f3f54f Ilias Tsitsimpis
# UUID of owner of system images
29 62f3f54f Ilias Tsitsimpis
DEFAULT_SYSTEM_IMAGES_UUID = [
30 62f3f54f Ilias Tsitsimpis
    "25ecced9-bf53-4145-91ee-cf47377e9fb2",  # production (okeanos.grnet.gr)
31 62f3f54f Ilias Tsitsimpis
    "04cbe33f-29b7-4ef1-94fb-015929e5fc06",  # testing (okeanos.io)
32 905bb10b Christos Stavrakakis
]
33 6ca8f81a Christos Stavrakakis
34 c29ac11d Ilias Tsitsimpis
35 c29ac11d Ilias Tsitsimpis
def _run(cmd, verbose):
36 c29ac11d Ilias Tsitsimpis
    """Run fabric with verbose level"""
37 c29ac11d Ilias Tsitsimpis
    if verbose:
38 c29ac11d Ilias Tsitsimpis
        args = ('running',)
39 c29ac11d Ilias Tsitsimpis
    else:
40 c29ac11d Ilias Tsitsimpis
        args = ('running', 'stdout',)
41 e2a0abb8 Ilias Tsitsimpis
    with fabric.hide(*args):  # Used * or ** magic. pylint: disable-msg=W0142
42 c29ac11d Ilias Tsitsimpis
        return fabric.run(cmd)
43 c29ac11d Ilias Tsitsimpis
44 c29ac11d Ilias Tsitsimpis
45 6868804a Ilias Tsitsimpis
def _put(local, remote):
46 6868804a Ilias Tsitsimpis
    """Run fabric put command without output"""
47 6868804a Ilias Tsitsimpis
    with fabric.quiet():
48 6868804a Ilias Tsitsimpis
        fabric.put(local, remote)
49 6868804a Ilias Tsitsimpis
50 6868804a Ilias Tsitsimpis
51 c29ac11d Ilias Tsitsimpis
def _red(msg):
52 c29ac11d Ilias Tsitsimpis
    """Red color"""
53 106659f1 Ilias Tsitsimpis
    ret = "\x1b[31m" + str(msg) + "\x1b[0m" if USE_COLORS else str(msg)
54 106659f1 Ilias Tsitsimpis
    return ret
55 c29ac11d Ilias Tsitsimpis
56 c29ac11d Ilias Tsitsimpis
57 c29ac11d Ilias Tsitsimpis
def _yellow(msg):
58 c29ac11d Ilias Tsitsimpis
    """Yellow color"""
59 106659f1 Ilias Tsitsimpis
    ret = "\x1b[33m" + str(msg) + "\x1b[0m" if USE_COLORS else str(msg)
60 106659f1 Ilias Tsitsimpis
    return ret
61 c29ac11d Ilias Tsitsimpis
62 c29ac11d Ilias Tsitsimpis
63 c29ac11d Ilias Tsitsimpis
def _green(msg):
64 c29ac11d Ilias Tsitsimpis
    """Green color"""
65 106659f1 Ilias Tsitsimpis
    ret = "\x1b[32m" + str(msg) + "\x1b[0m" if USE_COLORS else str(msg)
66 106659f1 Ilias Tsitsimpis
    return ret
67 c29ac11d Ilias Tsitsimpis
68 c29ac11d Ilias Tsitsimpis
69 c29ac11d Ilias Tsitsimpis
def _check_fabric(fun):
70 c29ac11d Ilias Tsitsimpis
    """Check if fabric env has been set"""
71 43a22402 Christos Stavrakakis
    def wrapper(self, *args, **kwargs):
72 c29ac11d Ilias Tsitsimpis
        """wrapper function"""
73 c29ac11d Ilias Tsitsimpis
        if not self.fabric_installed:
74 c29ac11d Ilias Tsitsimpis
            self.setup_fabric()
75 6c3cc77e Ilias Tsitsimpis
            self.fabric_installed = True
76 43a22402 Christos Stavrakakis
        return fun(self, *args, **kwargs)
77 c29ac11d Ilias Tsitsimpis
    return wrapper
78 c29ac11d Ilias Tsitsimpis
79 c29ac11d Ilias Tsitsimpis
80 c29ac11d Ilias Tsitsimpis
def _check_kamaki(fun):
81 c29ac11d Ilias Tsitsimpis
    """Check if kamaki has been initialized"""
82 43a22402 Christos Stavrakakis
    def wrapper(self, *args, **kwargs):
83 c29ac11d Ilias Tsitsimpis
        """wrapper function"""
84 c29ac11d Ilias Tsitsimpis
        if not self.kamaki_installed:
85 c29ac11d Ilias Tsitsimpis
            self.setup_kamaki()
86 6c3cc77e Ilias Tsitsimpis
            self.kamaki_installed = True
87 43a22402 Christos Stavrakakis
        return fun(self, *args, **kwargs)
88 c29ac11d Ilias Tsitsimpis
    return wrapper
89 c29ac11d Ilias Tsitsimpis
90 c29ac11d Ilias Tsitsimpis
91 c29ac11d Ilias Tsitsimpis
class _MyFormatter(logging.Formatter):
92 c29ac11d Ilias Tsitsimpis
    """Logging Formatter"""
93 c29ac11d Ilias Tsitsimpis
    def format(self, record):
94 c29ac11d Ilias Tsitsimpis
        format_orig = self._fmt
95 c29ac11d Ilias Tsitsimpis
        if record.levelno == logging.DEBUG:
96 94c89c0e Ilias Tsitsimpis
            self._fmt = "  %(message)s"
97 c29ac11d Ilias Tsitsimpis
        elif record.levelno == logging.INFO:
98 94c89c0e Ilias Tsitsimpis
            self._fmt = "%(message)s"
99 c29ac11d Ilias Tsitsimpis
        elif record.levelno == logging.WARNING:
100 94c89c0e Ilias Tsitsimpis
            self._fmt = _yellow("[W] %(message)s")
101 c29ac11d Ilias Tsitsimpis
        elif record.levelno == logging.ERROR:
102 94c89c0e Ilias Tsitsimpis
            self._fmt = _red("[E] %(message)s")
103 c29ac11d Ilias Tsitsimpis
        result = logging.Formatter.format(self, record)
104 c29ac11d Ilias Tsitsimpis
        self._fmt = format_orig
105 c29ac11d Ilias Tsitsimpis
        return result
106 c29ac11d Ilias Tsitsimpis
107 c29ac11d Ilias Tsitsimpis
108 56ee1bbc Ilias Tsitsimpis
# Too few public methods. pylint: disable-msg=R0903
109 56ee1bbc Ilias Tsitsimpis
class _InfoFilter(logging.Filter):
110 56ee1bbc Ilias Tsitsimpis
    """Logging Filter that allows DEBUG and INFO messages only"""
111 56ee1bbc Ilias Tsitsimpis
    def filter(self, rec):
112 56ee1bbc Ilias Tsitsimpis
        """The filter"""
113 56ee1bbc Ilias Tsitsimpis
        return rec.levelno in (logging.DEBUG, logging.INFO)
114 56ee1bbc Ilias Tsitsimpis
115 56ee1bbc Ilias Tsitsimpis
116 56ee1bbc Ilias Tsitsimpis
# Too many instance attributes. pylint: disable-msg=R0902
117 c29ac11d Ilias Tsitsimpis
class SynnefoCI(object):
118 c29ac11d Ilias Tsitsimpis
    """SynnefoCI python class"""
119 c29ac11d Ilias Tsitsimpis
120 79144a72 Ilias Tsitsimpis
    def __init__(self, config_file=None, build_id=None, cloud=None):
121 c29ac11d Ilias Tsitsimpis
        """ Initialize SynnefoCI python class
122 c29ac11d Ilias Tsitsimpis

123 c29ac11d Ilias Tsitsimpis
        Setup logger, local_dir, config and kamaki
124 c29ac11d Ilias Tsitsimpis
        """
125 c29ac11d Ilias Tsitsimpis
        # Setup logger
126 c29ac11d Ilias Tsitsimpis
        self.logger = logging.getLogger('synnefo-ci')
127 c29ac11d Ilias Tsitsimpis
        self.logger.setLevel(logging.DEBUG)
128 56ee1bbc Ilias Tsitsimpis
129 56ee1bbc Ilias Tsitsimpis
        handler1 = logging.StreamHandler(sys.stdout)
130 56ee1bbc Ilias Tsitsimpis
        handler1.setLevel(logging.DEBUG)
131 56ee1bbc Ilias Tsitsimpis
        handler1.addFilter(_InfoFilter())
132 56ee1bbc Ilias Tsitsimpis
        handler1.setFormatter(_MyFormatter())
133 56ee1bbc Ilias Tsitsimpis
        handler2 = logging.StreamHandler(sys.stderr)
134 56ee1bbc Ilias Tsitsimpis
        handler2.setLevel(logging.WARNING)
135 56ee1bbc Ilias Tsitsimpis
        handler2.setFormatter(_MyFormatter())
136 56ee1bbc Ilias Tsitsimpis
137 56ee1bbc Ilias Tsitsimpis
        self.logger.addHandler(handler1)
138 56ee1bbc Ilias Tsitsimpis
        self.logger.addHandler(handler2)
139 c29ac11d Ilias Tsitsimpis
140 c29ac11d Ilias Tsitsimpis
        # Get our local dir
141 c29ac11d Ilias Tsitsimpis
        self.ci_dir = os.path.dirname(os.path.abspath(__file__))
142 c29ac11d Ilias Tsitsimpis
        self.repo_dir = os.path.dirname(self.ci_dir)
143 c29ac11d Ilias Tsitsimpis
144 c29ac11d Ilias Tsitsimpis
        # Read config file
145 6ca8f81a Christos Stavrakakis
        if config_file is None:
146 2c4a641b Ilias Tsitsimpis
            config_file = os.path.join(self.ci_dir, DEFAULT_CONFIG_FILE)
147 2c4a641b Ilias Tsitsimpis
        config_file = os.path.abspath(config_file)
148 c29ac11d Ilias Tsitsimpis
        self.config = ConfigParser()
149 c29ac11d Ilias Tsitsimpis
        self.config.optionxform = str
150 6ca8f81a Christos Stavrakakis
        self.config.read(config_file)
151 79144a72 Ilias Tsitsimpis
152 79144a72 Ilias Tsitsimpis
        # Read temporary_config file
153 62fcf0e5 Ilias Tsitsimpis
        self.temp_config_file = \
154 62fcf0e5 Ilias Tsitsimpis
            os.path.expanduser(self.config.get('Global', 'temporary_config'))
155 79144a72 Ilias Tsitsimpis
        self.temp_config = ConfigParser()
156 79144a72 Ilias Tsitsimpis
        self.temp_config.optionxform = str
157 62fcf0e5 Ilias Tsitsimpis
        self.temp_config.read(self.temp_config_file)
158 b7496c88 Ilias Tsitsimpis
        self.build_id = build_id
159 99d39120 Ilias Tsitsimpis
        if build_id is not None:
160 99d39120 Ilias Tsitsimpis
            self.logger.info("Will use \"%s\" as build id" %
161 99d39120 Ilias Tsitsimpis
                             _green(self.build_id))
162 c29ac11d Ilias Tsitsimpis
163 c1b1d444 Christos Stavrakakis
        # Set kamaki cloud
164 c1b1d444 Christos Stavrakakis
        if cloud is not None:
165 c1b1d444 Christos Stavrakakis
            self.kamaki_cloud = cloud
166 c1b1d444 Christos Stavrakakis
        elif self.config.has_option("Deployment", "kamaki_cloud"):
167 c1b1d444 Christos Stavrakakis
            kamaki_cloud = self.config.get("Deployment", "kamaki_cloud")
168 c1b1d444 Christos Stavrakakis
            if kamaki_cloud == "":
169 c1b1d444 Christos Stavrakakis
                self.kamaki_cloud = None
170 c1b1d444 Christos Stavrakakis
        else:
171 c1b1d444 Christos Stavrakakis
            self.kamaki_cloud = None
172 c1b1d444 Christos Stavrakakis
173 c29ac11d Ilias Tsitsimpis
        # Initialize variables
174 c29ac11d Ilias Tsitsimpis
        self.fabric_installed = False
175 c29ac11d Ilias Tsitsimpis
        self.kamaki_installed = False
176 c29ac11d Ilias Tsitsimpis
        self.cyclades_client = None
177 94c89c0e Ilias Tsitsimpis
        self.network_client = None
178 ad21ffa9 Ilias Tsitsimpis
        self.compute_client = None
179 c29ac11d Ilias Tsitsimpis
        self.image_client = None
180 694c79bb Ilias Tsitsimpis
        self.astakos_client = None
181 c29ac11d Ilias Tsitsimpis
182 c29ac11d Ilias Tsitsimpis
    def setup_kamaki(self):
183 c29ac11d Ilias Tsitsimpis
        """Initialize kamaki
184 c29ac11d Ilias Tsitsimpis

185 ad21ffa9 Ilias Tsitsimpis
        Setup cyclades_client, image_client and compute_client
186 c29ac11d Ilias Tsitsimpis
        """
187 c1b1d444 Christos Stavrakakis
188 c1b1d444 Christos Stavrakakis
        config = kamaki_config.Config()
189 c1b1d444 Christos Stavrakakis
        if self.kamaki_cloud is None:
190 84d2db4f Ilias Tsitsimpis
            try:
191 84d2db4f Ilias Tsitsimpis
                self.kamaki_cloud = config.get("global", "default_cloud")
192 84d2db4f Ilias Tsitsimpis
            except AttributeError:
193 84d2db4f Ilias Tsitsimpis
                # Compatibility with kamaki version <=0.10
194 94c89c0e Ilias Tsitsimpis
                self.kamaki_cloud = config.get("global", "default_cloud")
195 c1b1d444 Christos Stavrakakis
196 c1b1d444 Christos Stavrakakis
        self.logger.info("Setup kamaki client, using cloud '%s'.." %
197 c1b1d444 Christos Stavrakakis
                         self.kamaki_cloud)
198 c1b1d444 Christos Stavrakakis
        auth_url = config.get_cloud(self.kamaki_cloud, "url")
199 c29ac11d Ilias Tsitsimpis
        self.logger.debug("Authentication URL is %s" % _green(auth_url))
200 c1b1d444 Christos Stavrakakis
        token = config.get_cloud(self.kamaki_cloud, "token")
201 c29ac11d Ilias Tsitsimpis
        #self.logger.debug("Token is %s" % _green(token))
202 c29ac11d Ilias Tsitsimpis
203 ee6eff28 Ilias Tsitsimpis
        self.astakos_client = AstakosClient(auth_url, token)
204 ee6eff28 Ilias Tsitsimpis
        endpoints = self.astakos_client.authenticate()
205 c29ac11d Ilias Tsitsimpis
206 ee6eff28 Ilias Tsitsimpis
        cyclades_url = get_endpoint_url(endpoints, "compute")
207 c29ac11d Ilias Tsitsimpis
        self.logger.debug("Cyclades API url is %s" % _green(cyclades_url))
208 c29ac11d Ilias Tsitsimpis
        self.cyclades_client = CycladesClient(cyclades_url, token)
209 c29ac11d Ilias Tsitsimpis
        self.cyclades_client.CONNECTION_RETRY_LIMIT = 2
210 c29ac11d Ilias Tsitsimpis
211 ee6eff28 Ilias Tsitsimpis
        network_url = get_endpoint_url(endpoints, "network")
212 94c89c0e Ilias Tsitsimpis
        self.logger.debug("Network API url is %s" % _green(network_url))
213 94c89c0e Ilias Tsitsimpis
        self.network_client = CycladesNetworkClient(network_url, token)
214 94c89c0e Ilias Tsitsimpis
        self.network_client.CONNECTION_RETRY_LIMIT = 2
215 94c89c0e Ilias Tsitsimpis
216 ee6eff28 Ilias Tsitsimpis
        image_url = get_endpoint_url(endpoints, "image")
217 c29ac11d Ilias Tsitsimpis
        self.logger.debug("Images API url is %s" % _green(image_url))
218 c29ac11d Ilias Tsitsimpis
        self.image_client = ImageClient(cyclades_url, token)
219 c29ac11d Ilias Tsitsimpis
        self.image_client.CONNECTION_RETRY_LIMIT = 2
220 c29ac11d Ilias Tsitsimpis
221 ee6eff28 Ilias Tsitsimpis
        compute_url = get_endpoint_url(endpoints, "compute")
222 ad21ffa9 Ilias Tsitsimpis
        self.logger.debug("Compute API url is %s" % _green(compute_url))
223 ad21ffa9 Ilias Tsitsimpis
        self.compute_client = ComputeClient(compute_url, token)
224 ad21ffa9 Ilias Tsitsimpis
        self.compute_client.CONNECTION_RETRY_LIMIT = 2
225 ad21ffa9 Ilias Tsitsimpis
226 c29ac11d Ilias Tsitsimpis
    def _wait_transition(self, server_id, current_status, new_status):
227 c29ac11d Ilias Tsitsimpis
        """Wait for server to go from current_status to new_status"""
228 c29ac11d Ilias Tsitsimpis
        self.logger.debug("Waiting for server to become %s" % new_status)
229 c29ac11d Ilias Tsitsimpis
        timeout = self.config.getint('Global', 'build_timeout')
230 c29ac11d Ilias Tsitsimpis
        sleep_time = 5
231 c29ac11d Ilias Tsitsimpis
        while True:
232 c29ac11d Ilias Tsitsimpis
            server = self.cyclades_client.get_server_details(server_id)
233 c29ac11d Ilias Tsitsimpis
            if server['status'] == new_status:
234 c29ac11d Ilias Tsitsimpis
                return server
235 c29ac11d Ilias Tsitsimpis
            elif timeout < 0:
236 c29ac11d Ilias Tsitsimpis
                self.logger.error(
237 c29ac11d Ilias Tsitsimpis
                    "Waiting for server to become %s timed out" % new_status)
238 c29ac11d Ilias Tsitsimpis
                self.destroy_server(False)
239 99d39120 Ilias Tsitsimpis
                sys.exit(1)
240 c29ac11d Ilias Tsitsimpis
            elif server['status'] == current_status:
241 c29ac11d Ilias Tsitsimpis
                # Sleep for #n secs and continue
242 c29ac11d Ilias Tsitsimpis
                timeout = timeout - sleep_time
243 c29ac11d Ilias Tsitsimpis
                time.sleep(sleep_time)
244 c29ac11d Ilias Tsitsimpis
            else:
245 c29ac11d Ilias Tsitsimpis
                self.logger.error(
246 c29ac11d Ilias Tsitsimpis
                    "Server failed with status %s" % server['status'])
247 c29ac11d Ilias Tsitsimpis
                self.destroy_server(False)
248 99d39120 Ilias Tsitsimpis
                sys.exit(1)
249 c29ac11d Ilias Tsitsimpis
250 c29ac11d Ilias Tsitsimpis
    @_check_kamaki
251 c29ac11d Ilias Tsitsimpis
    def destroy_server(self, wait=True):
252 c29ac11d Ilias Tsitsimpis
        """Destroy slave server"""
253 b7496c88 Ilias Tsitsimpis
        server_id = int(self.read_temp_config('server_id'))
254 9dc74d37 Ilias Tsitsimpis
        fips = [f for f in self.network_client.list_floatingips()
255 9dc74d37 Ilias Tsitsimpis
                if str(f['instance_id']) == str(server_id)]
256 c29ac11d Ilias Tsitsimpis
        self.logger.info("Destoying server with id %s " % server_id)
257 c29ac11d Ilias Tsitsimpis
        self.cyclades_client.delete_server(server_id)
258 c29ac11d Ilias Tsitsimpis
        if wait:
259 c29ac11d Ilias Tsitsimpis
            self._wait_transition(server_id, "ACTIVE", "DELETED")
260 9dc74d37 Ilias Tsitsimpis
        for fip in fips:
261 9dc74d37 Ilias Tsitsimpis
            self.logger.info("Destroying floating ip %s",
262 9dc74d37 Ilias Tsitsimpis
                             fip['floating_ip_address'])
263 9dc74d37 Ilias Tsitsimpis
            self.network_client.delete_floatingip(fip['id'])
264 c29ac11d Ilias Tsitsimpis
265 2eb2f223 Ilias Tsitsimpis
    # pylint: disable= no-self-use
266 2eb2f223 Ilias Tsitsimpis
    @_check_fabric
267 2eb2f223 Ilias Tsitsimpis
    def shell_connect(self):
268 2eb2f223 Ilias Tsitsimpis
        """Open shell to remote server"""
269 2eb2f223 Ilias Tsitsimpis
        fabric.open_shell("export TERM=xterm")
270 2eb2f223 Ilias Tsitsimpis
271 94c89c0e Ilias Tsitsimpis
    def _create_floating_ip(self):
272 94c89c0e Ilias Tsitsimpis
        """Create a new floating ip"""
273 94c89c0e Ilias Tsitsimpis
        networks = self.network_client.list_networks(detail=True)
274 49fb00cc Ilias Tsitsimpis
        pub_nets = [n for n in networks
275 49fb00cc Ilias Tsitsimpis
                    if n['SNF:floating_ip_pool'] and n['public']]
276 49fb00cc Ilias Tsitsimpis
        for pub_net in pub_nets:
277 49fb00cc Ilias Tsitsimpis
            # Try until we find a public network that is not full
278 49fb00cc Ilias Tsitsimpis
            try:
279 49fb00cc Ilias Tsitsimpis
                fip = self.network_client.create_floatingip(pub_net['id'])
280 49fb00cc Ilias Tsitsimpis
            except ClientError as err:
281 49fb00cc Ilias Tsitsimpis
                self.logger.warning("%s: %s", err.message, err.details)
282 49fb00cc Ilias Tsitsimpis
                continue
283 49fb00cc Ilias Tsitsimpis
            self.logger.debug("Floating IP %s with id %s created",
284 49fb00cc Ilias Tsitsimpis
                              fip['floating_ip_address'], fip['id'])
285 49fb00cc Ilias Tsitsimpis
            return fip
286 49fb00cc Ilias Tsitsimpis
        self.logger.error("No mor IP addresses available")
287 49fb00cc Ilias Tsitsimpis
        sys.exit(1)
288 94c89c0e Ilias Tsitsimpis
289 94c89c0e Ilias Tsitsimpis
    def _create_port(self, floating_ip):
290 94c89c0e Ilias Tsitsimpis
        """Create a new port for our floating IP"""
291 94c89c0e Ilias Tsitsimpis
        net_id = floating_ip['floating_network_id']
292 94c89c0e Ilias Tsitsimpis
        self.logger.debug("Creating a new port to network with id %s", net_id)
293 94c89c0e Ilias Tsitsimpis
        fixed_ips = [{'ip_address': floating_ip['floating_ip_address']}]
294 94c89c0e Ilias Tsitsimpis
        port = self.network_client.create_port(
295 94c89c0e Ilias Tsitsimpis
            net_id, device_id=None, fixed_ips=fixed_ips)
296 94c89c0e Ilias Tsitsimpis
        return port
297 94c89c0e Ilias Tsitsimpis
298 c29ac11d Ilias Tsitsimpis
    @_check_kamaki
299 ee6eff28 Ilias Tsitsimpis
    # Too many local variables. pylint: disable-msg=R0914
300 2945e7ed Christos Stavrakakis
    def create_server(self, image=None, flavor=None, ssh_keys=None,
301 2945e7ed Christos Stavrakakis
                      server_name=None):
302 c29ac11d Ilias Tsitsimpis
        """Create slave server"""
303 c29ac11d Ilias Tsitsimpis
        self.logger.info("Create a new server..")
304 b7496c88 Ilias Tsitsimpis
305 b7496c88 Ilias Tsitsimpis
        # Find a build_id to use
306 c7828946 Ilias Tsitsimpis
        self._create_new_build_id()
307 b7496c88 Ilias Tsitsimpis
308 b7496c88 Ilias Tsitsimpis
        # Find an image to use
309 358a19bc Ilias Tsitsimpis
        image_id = self._find_image(image)
310 b7496c88 Ilias Tsitsimpis
        # Find a flavor to use
311 358a19bc Ilias Tsitsimpis
        flavor_id = self._find_flavor(flavor)
312 358a19bc Ilias Tsitsimpis
313 358a19bc Ilias Tsitsimpis
        # Create Server
314 0368716b Christos Stavrakakis
        networks = []
315 0368716b Christos Stavrakakis
        if self.config.get("Deployment", "allocate_floating_ip") == "True":
316 0368716b Christos Stavrakakis
            fip = self._create_floating_ip()
317 0368716b Christos Stavrakakis
            port = self._create_port(fip)
318 0368716b Christos Stavrakakis
            networks.append({'port': port['id']})
319 0368716b Christos Stavrakakis
        private_networks = self.config.get('Deployment', 'private_networks')
320 0368716b Christos Stavrakakis
        if private_networks:
321 bbc826ca Ilias Tsitsimpis
            private_networks = [p.strip() for p in private_networks.split(",")]
322 bbc826ca Ilias Tsitsimpis
            networks.extend([{"uuid": uuid} for uuid in private_networks])
323 2945e7ed Christos Stavrakakis
        if server_name is None:
324 2945e7ed Christos Stavrakakis
            server_name = self.config.get("Deployment", "server_name")
325 2945e7ed Christos Stavrakakis
            server_name = "%s(BID: %s)" % (server_name, self.build_id)
326 94c89c0e Ilias Tsitsimpis
        server = self.cyclades_client.create_server(
327 94c89c0e Ilias Tsitsimpis
            server_name, flavor_id, image_id, networks=networks)
328 c29ac11d Ilias Tsitsimpis
        server_id = server['id']
329 b7496c88 Ilias Tsitsimpis
        self.write_temp_config('server_id', server_id)
330 c29ac11d Ilias Tsitsimpis
        self.logger.debug("Server got id %s" % _green(server_id))
331 c29ac11d Ilias Tsitsimpis
        server_user = server['metadata']['users']
332 b7496c88 Ilias Tsitsimpis
        self.write_temp_config('server_user', server_user)
333 c29ac11d Ilias Tsitsimpis
        self.logger.debug("Server's admin user is %s" % _green(server_user))
334 c29ac11d Ilias Tsitsimpis
        server_passwd = server['adminPass']
335 b7496c88 Ilias Tsitsimpis
        self.write_temp_config('server_passwd', server_passwd)
336 c29ac11d Ilias Tsitsimpis
337 c29ac11d Ilias Tsitsimpis
        server = self._wait_transition(server_id, "BUILD", "ACTIVE")
338 bbc826ca Ilias Tsitsimpis
        self._get_server_ip_and_port(server, private_networks)
339 6868804a Ilias Tsitsimpis
        self._copy_ssh_keys(ssh_keys)
340 c29ac11d Ilias Tsitsimpis
341 6c3cc77e Ilias Tsitsimpis
        # Setup Firewall
342 c29ac11d Ilias Tsitsimpis
        self.setup_fabric()
343 c29ac11d Ilias Tsitsimpis
        self.logger.info("Setup firewall")
344 6868804a Ilias Tsitsimpis
        accept_ssh_from = self.config.get('Global', 'accept_ssh_from')
345 1c75c4dd Christos Stavrakakis
        if accept_ssh_from != "":
346 1c75c4dd Christos Stavrakakis
            self.logger.debug("Block ssh except from %s" % accept_ssh_from)
347 1c75c4dd Christos Stavrakakis
            cmd = """
348 1c75c4dd Christos Stavrakakis
            local_ip=$(/sbin/ifconfig eth0 | grep 'inet addr:' | \
349 1c75c4dd Christos Stavrakakis
                cut -d':' -f2 | cut -d' ' -f1)
350 1c75c4dd Christos Stavrakakis
            iptables -A INPUT -s localhost -j ACCEPT
351 1c75c4dd Christos Stavrakakis
            iptables -A INPUT -s $local_ip -j ACCEPT
352 1c75c4dd Christos Stavrakakis
            iptables -A INPUT -s {0} -p tcp --dport 22 -j ACCEPT
353 1c75c4dd Christos Stavrakakis
            iptables -A INPUT -p tcp --dport 22 -j DROP
354 1c75c4dd Christos Stavrakakis
            """.format(accept_ssh_from)
355 1c75c4dd Christos Stavrakakis
            _run(cmd, False)
356 c29ac11d Ilias Tsitsimpis
357 6c3cc77e Ilias Tsitsimpis
        # Setup apt, download packages
358 6c3cc77e Ilias Tsitsimpis
        self.logger.debug("Setup apt. Install x2goserver and firefox")
359 6c3cc77e Ilias Tsitsimpis
        cmd = """
360 6c3cc77e Ilias Tsitsimpis
        echo 'APT::Install-Suggests "false";' >> /etc/apt/apt.conf
361 a7d32e21 Ilias Tsitsimpis
        echo 'precedence ::ffff:0:0/96  100' >> /etc/gai.conf
362 6c3cc77e Ilias Tsitsimpis
        apt-get update
363 71f2649f Ilias Tsitsimpis
        apt-get install curl --yes --force-yes
364 6c3cc77e Ilias Tsitsimpis
        echo -e "\n\n{0}" >> /etc/apt/sources.list
365 6c3cc77e Ilias Tsitsimpis
        # Synnefo repo's key
366 6c3cc77e Ilias Tsitsimpis
        curl https://dev.grnet.gr/files/apt-grnetdev.pub | apt-key add -
367 8ddb541b Ilias Tsitsimpis

368 6c3cc77e Ilias Tsitsimpis
        # X2GO Key
369 6c3cc77e Ilias Tsitsimpis
        apt-key adv --recv-keys --keyserver keys.gnupg.net E1F958385BFE2B6E
370 71f2649f Ilias Tsitsimpis
        apt-get install x2go-keyring --yes --force-yes
371 6c3cc77e Ilias Tsitsimpis
        apt-get update
372 71f2649f Ilias Tsitsimpis
        apt-get install x2goserver x2goserver-xsession \
373 71f2649f Ilias Tsitsimpis
                iceweasel --yes --force-yes
374 8ddb541b Ilias Tsitsimpis

375 8ddb541b Ilias Tsitsimpis
        # xterm published application
376 8ddb541b Ilias Tsitsimpis
        echo '[Desktop Entry]' > /usr/share/applications/xterm.desktop
377 8ddb541b Ilias Tsitsimpis
        echo 'Name=XTerm' >> /usr/share/applications/xterm.desktop
378 8ddb541b Ilias Tsitsimpis
        echo 'Comment=standard terminal emulator for the X window system' >> \
379 8ddb541b Ilias Tsitsimpis
            /usr/share/applications/xterm.desktop
380 8ddb541b Ilias Tsitsimpis
        echo 'Exec=xterm' >> /usr/share/applications/xterm.desktop
381 8ddb541b Ilias Tsitsimpis
        echo 'Terminal=false' >> /usr/share/applications/xterm.desktop
382 8ddb541b Ilias Tsitsimpis
        echo 'Type=Application' >> /usr/share/applications/xterm.desktop
383 8ddb541b Ilias Tsitsimpis
        echo 'Encoding=UTF-8' >> /usr/share/applications/xterm.desktop
384 8ddb541b Ilias Tsitsimpis
        echo 'Icon=xterm-color_48x48' >> /usr/share/applications/xterm.desktop
385 8ddb541b Ilias Tsitsimpis
        echo 'Categories=System;TerminalEmulator;' >> \
386 8ddb541b Ilias Tsitsimpis
                /usr/share/applications/xterm.desktop
387 6c3cc77e Ilias Tsitsimpis
        """.format(self.config.get('Global', 'apt_repo'))
388 6c3cc77e Ilias Tsitsimpis
        _run(cmd, False)
389 6c3cc77e Ilias Tsitsimpis
390 358a19bc Ilias Tsitsimpis
    def _find_flavor(self, flavor=None):
391 358a19bc Ilias Tsitsimpis
        """Find a suitable flavor to use
392 358a19bc Ilias Tsitsimpis

393 358a19bc Ilias Tsitsimpis
        Search by name (reg expression) or by id
394 358a19bc Ilias Tsitsimpis
        """
395 358a19bc Ilias Tsitsimpis
        # Get a list of flavors from config file
396 358a19bc Ilias Tsitsimpis
        flavors = self.config.get('Deployment', 'flavors').split(",")
397 358a19bc Ilias Tsitsimpis
        if flavor is not None:
398 a8bfeffc Ilias Tsitsimpis
            # If we have a flavor_name to use, add it to our list
399 358a19bc Ilias Tsitsimpis
            flavors.insert(0, flavor)
400 358a19bc Ilias Tsitsimpis
401 358a19bc Ilias Tsitsimpis
        list_flavors = self.compute_client.list_flavors()
402 358a19bc Ilias Tsitsimpis
        for flv in flavors:
403 905bb10b Christos Stavrakakis
            flv_type, flv_value = parse_typed_option(option="flavor",
404 905bb10b Christos Stavrakakis
                                                     value=flv)
405 358a19bc Ilias Tsitsimpis
            if flv_type == "name":
406 358a19bc Ilias Tsitsimpis
                # Filter flavors by name
407 358a19bc Ilias Tsitsimpis
                self.logger.debug(
408 358a19bc Ilias Tsitsimpis
                    "Trying to find a flavor with name \"%s\"" % flv_value)
409 358a19bc Ilias Tsitsimpis
                list_flvs = \
410 358a19bc Ilias Tsitsimpis
                    [f for f in list_flavors
411 905bb10b Christos Stavrakakis
                     if re.search(flv_value, f['name'], flags=re.I)
412 905bb10b Christos Stavrakakis
                     is not None]
413 358a19bc Ilias Tsitsimpis
            elif flv_type == "id":
414 358a19bc Ilias Tsitsimpis
                # Filter flavors by id
415 358a19bc Ilias Tsitsimpis
                self.logger.debug(
416 358a19bc Ilias Tsitsimpis
                    "Trying to find a flavor with id \"%s\"" % flv_value)
417 358a19bc Ilias Tsitsimpis
                list_flvs = \
418 358a19bc Ilias Tsitsimpis
                    [f for f in list_flavors
419 694c79bb Ilias Tsitsimpis
                     if str(f['id']) == flv_value]
420 358a19bc Ilias Tsitsimpis
            else:
421 358a19bc Ilias Tsitsimpis
                self.logger.error("Unrecognized flavor type %s" % flv_type)
422 358a19bc Ilias Tsitsimpis
423 358a19bc Ilias Tsitsimpis
            # Check if we found one
424 358a19bc Ilias Tsitsimpis
            if list_flvs:
425 358a19bc Ilias Tsitsimpis
                self.logger.debug("Will use \"%s\" with id \"%s\""
426 106659f1 Ilias Tsitsimpis
                                  % (_green(list_flvs[0]['name']),
427 106659f1 Ilias Tsitsimpis
                                     _green(list_flvs[0]['id'])))
428 358a19bc Ilias Tsitsimpis
                return list_flvs[0]['id']
429 a8bfeffc Ilias Tsitsimpis
430 a8bfeffc Ilias Tsitsimpis
        self.logger.error("No matching flavor found.. aborting")
431 a8bfeffc Ilias Tsitsimpis
        sys.exit(1)
432 ad21ffa9 Ilias Tsitsimpis
433 358a19bc Ilias Tsitsimpis
    def _find_image(self, image=None):
434 c29ac11d Ilias Tsitsimpis
        """Find a suitable image to use
435 c29ac11d Ilias Tsitsimpis

436 358a19bc Ilias Tsitsimpis
        In case of search by name, the image has to belong to one
437 358a19bc Ilias Tsitsimpis
        of the `DEFAULT_SYSTEM_IMAGES_UUID' users.
438 358a19bc Ilias Tsitsimpis
        In case of search by id it only has to exist.
439 c29ac11d Ilias Tsitsimpis
        """
440 358a19bc Ilias Tsitsimpis
        # Get a list of images from config file
441 358a19bc Ilias Tsitsimpis
        images = self.config.get('Deployment', 'images').split(",")
442 358a19bc Ilias Tsitsimpis
        if image is not None:
443 358a19bc Ilias Tsitsimpis
            # If we have an image from command line, add it to our list
444 358a19bc Ilias Tsitsimpis
            images.insert(0, image)
445 358a19bc Ilias Tsitsimpis
446 905bb10b Christos Stavrakakis
        auth = self.astakos_client.authenticate()
447 905bb10b Christos Stavrakakis
        user_uuid = auth["access"]["token"]["tenant"]["id"]
448 358a19bc Ilias Tsitsimpis
        list_images = self.image_client.list_public(detail=True)['images']
449 358a19bc Ilias Tsitsimpis
        for img in images:
450 6958ffc6 Christos Stavrakakis
            img_type, img_value = parse_typed_option(option="image", value=img)
451 358a19bc Ilias Tsitsimpis
            if img_type == "name":
452 358a19bc Ilias Tsitsimpis
                # Filter images by name
453 358a19bc Ilias Tsitsimpis
                self.logger.debug(
454 358a19bc Ilias Tsitsimpis
                    "Trying to find an image with name \"%s\"" % img_value)
455 905bb10b Christos Stavrakakis
                accepted_uuids = DEFAULT_SYSTEM_IMAGES_UUID + [user_uuid]
456 358a19bc Ilias Tsitsimpis
                list_imgs = \
457 905bb10b Christos Stavrakakis
                    [i for i in list_images if i['user_id'] in accepted_uuids
458 905bb10b Christos Stavrakakis
                     and
459 905bb10b Christos Stavrakakis
                     re.search(img_value, i['name'], flags=re.I) is not None]
460 358a19bc Ilias Tsitsimpis
            elif img_type == "id":
461 358a19bc Ilias Tsitsimpis
                # Filter images by id
462 358a19bc Ilias Tsitsimpis
                self.logger.debug(
463 358a19bc Ilias Tsitsimpis
                    "Trying to find an image with id \"%s\"" % img_value)
464 358a19bc Ilias Tsitsimpis
                list_imgs = \
465 358a19bc Ilias Tsitsimpis
                    [i for i in list_images
466 358a19bc Ilias Tsitsimpis
                     if i['id'].lower() == img_value.lower()]
467 358a19bc Ilias Tsitsimpis
            else:
468 358a19bc Ilias Tsitsimpis
                self.logger.error("Unrecognized image type %s" % img_type)
469 358a19bc Ilias Tsitsimpis
                sys.exit(1)
470 358a19bc Ilias Tsitsimpis
471 358a19bc Ilias Tsitsimpis
            # Check if we found one
472 358a19bc Ilias Tsitsimpis
            if list_imgs:
473 358a19bc Ilias Tsitsimpis
                self.logger.debug("Will use \"%s\" with id \"%s\""
474 106659f1 Ilias Tsitsimpis
                                  % (_green(list_imgs[0]['name']),
475 106659f1 Ilias Tsitsimpis
                                     _green(list_imgs[0]['id'])))
476 358a19bc Ilias Tsitsimpis
                return list_imgs[0]['id']
477 358a19bc Ilias Tsitsimpis
478 358a19bc Ilias Tsitsimpis
        # We didn't found one
479 358a19bc Ilias Tsitsimpis
        self.logger.error("No matching image found.. aborting")
480 358a19bc Ilias Tsitsimpis
        sys.exit(1)
481 c29ac11d Ilias Tsitsimpis
482 bbc826ca Ilias Tsitsimpis
    def _get_server_ip_and_port(self, server, private_networks):
483 c29ac11d Ilias Tsitsimpis
        """Compute server's IPv4 and ssh port number"""
484 c29ac11d Ilias Tsitsimpis
        self.logger.info("Get server connection details..")
485 bbc826ca Ilias Tsitsimpis
        if private_networks:
486 bbc826ca Ilias Tsitsimpis
            # Choose the networks that belong to private_networks
487 bbc826ca Ilias Tsitsimpis
            networks = [n for n in server['attachments']
488 bbc826ca Ilias Tsitsimpis
                        if n['network_id'] in private_networks]
489 bbc826ca Ilias Tsitsimpis
        else:
490 bbc826ca Ilias Tsitsimpis
            # Choose the networks that are public
491 6be99b24 Ilias Tsitsimpis
            networks = [n for n in server['attachments']
492 6be99b24 Ilias Tsitsimpis
                        if self.network_client.
493 6be99b24 Ilias Tsitsimpis
                        get_network_details(n['network_id'])['public']]
494 bbc826ca Ilias Tsitsimpis
        # Choose the networks with IPv4
495 bbc826ca Ilias Tsitsimpis
        networks = [n for n in networks if n['ipv4']]
496 bbc826ca Ilias Tsitsimpis
        # Use the first network as IPv4
497 bbc826ca Ilias Tsitsimpis
        server_ip = networks[0]['ipv4']
498 bbc826ca Ilias Tsitsimpis
499 ac7b865d Alex Pyrgiotis
        # Check if config has ssh_port option and if so, use that port.
500 ac7b865d Alex Pyrgiotis
        if self.config.has_option("Deployment", "ssh_port"):
501 ac7b865d Alex Pyrgiotis
            server_port = self.config.get("Deployment", "ssh_port")
502 c29ac11d Ilias Tsitsimpis
        else:
503 ac7b865d Alex Pyrgiotis
            if (".okeanos.io" in self.cyclades_client.base_url or
504 ac7b865d Alex Pyrgiotis
            ".demo.synnefo.org" in self.cyclades_client.base_url):
505 ac7b865d Alex Pyrgiotis
                tmp1 = int(server_ip.split(".")[2])
506 ac7b865d Alex Pyrgiotis
                tmp2 = int(server_ip.split(".")[3])
507 ac7b865d Alex Pyrgiotis
                server_ip = "gate.okeanos.io"
508 ac7b865d Alex Pyrgiotis
                server_port = 10000 + tmp1 * 256 + tmp2
509 ac7b865d Alex Pyrgiotis
            else:
510 ac7b865d Alex Pyrgiotis
                server_port = 22
511 ac7b865d Alex Pyrgiotis
512 b7496c88 Ilias Tsitsimpis
        self.write_temp_config('server_ip', server_ip)
513 c29ac11d Ilias Tsitsimpis
        self.logger.debug("Server's IPv4 is %s" % _green(server_ip))
514 b7496c88 Ilias Tsitsimpis
        self.write_temp_config('server_port', server_port)
515 c29ac11d Ilias Tsitsimpis
        self.logger.debug("Server's ssh port is %s" % _green(server_port))
516 106659f1 Ilias Tsitsimpis
        ssh_command = "ssh -p %s %s@%s" \
517 106659f1 Ilias Tsitsimpis
            % (server_port, server['metadata']['users'], server_ip)
518 106659f1 Ilias Tsitsimpis
        self.logger.debug("Access server using \"%s\"" %
519 106659f1 Ilias Tsitsimpis
                          (_green(ssh_command)))
520 c29ac11d Ilias Tsitsimpis
521 e480f0ce Christos Stavrakakis
    @_check_fabric
522 6868804a Ilias Tsitsimpis
    def _copy_ssh_keys(self, ssh_keys):
523 e2a0abb8 Ilias Tsitsimpis
        """Upload/Install ssh keys to server"""
524 4b743650 Ilias Tsitsimpis
        self.logger.debug("Check for authentication keys to use")
525 6868804a Ilias Tsitsimpis
        if ssh_keys is None:
526 6868804a Ilias Tsitsimpis
            ssh_keys = self.config.get("Deployment", "ssh_keys")
527 6868804a Ilias Tsitsimpis
528 4b743650 Ilias Tsitsimpis
        if ssh_keys != "":
529 6058ae72 Ilias Tsitsimpis
            ssh_keys = os.path.expanduser(ssh_keys)
530 106659f1 Ilias Tsitsimpis
            self.logger.debug("Will use \"%s\" authentication keys file" %
531 106659f1 Ilias Tsitsimpis
                              _green(ssh_keys))
532 e480f0ce Christos Stavrakakis
            keyfile = '/tmp/%s.pub' % fabric.env.user
533 e480f0ce Christos Stavrakakis
            _run('mkdir -p ~/.ssh && chmod 700 ~/.ssh', False)
534 4b743650 Ilias Tsitsimpis
            if ssh_keys.startswith("http://") or \
535 4b743650 Ilias Tsitsimpis
                    ssh_keys.startswith("https://") or \
536 4b743650 Ilias Tsitsimpis
                    ssh_keys.startswith("ftp://"):
537 4b743650 Ilias Tsitsimpis
                cmd = """
538 4b743650 Ilias Tsitsimpis
                apt-get update
539 71f2649f Ilias Tsitsimpis
                apt-get install wget --yes --force-yes
540 4b743650 Ilias Tsitsimpis
                wget {0} -O {1} --no-check-certificate
541 4b743650 Ilias Tsitsimpis
                """.format(ssh_keys, keyfile)
542 4b743650 Ilias Tsitsimpis
                _run(cmd, False)
543 4b743650 Ilias Tsitsimpis
            elif os.path.exists(ssh_keys):
544 4b743650 Ilias Tsitsimpis
                _put(ssh_keys, keyfile)
545 4b743650 Ilias Tsitsimpis
            else:
546 4b743650 Ilias Tsitsimpis
                self.logger.debug("No ssh keys found")
547 6058ae72 Ilias Tsitsimpis
                return
548 e480f0ce Christos Stavrakakis
            _run('cat %s >> ~/.ssh/authorized_keys' % keyfile, False)
549 e480f0ce Christos Stavrakakis
            _run('rm %s' % keyfile, False)
550 e480f0ce Christos Stavrakakis
            self.logger.debug("Uploaded ssh authorized keys")
551 e480f0ce Christos Stavrakakis
        else:
552 e480f0ce Christos Stavrakakis
            self.logger.debug("No ssh keys found")
553 e480f0ce Christos Stavrakakis
554 2cbdb63f Ilias Tsitsimpis
    def _create_new_build_id(self):
555 2cbdb63f Ilias Tsitsimpis
        """Find a uniq build_id to use"""
556 2cbdb63f Ilias Tsitsimpis
        with filelocker.lock("%s.lock" % self.temp_config_file,
557 2cbdb63f Ilias Tsitsimpis
                             filelocker.LOCK_EX):
558 2cbdb63f Ilias Tsitsimpis
            # Read temp_config again to get any new entries
559 2cbdb63f Ilias Tsitsimpis
            self.temp_config.read(self.temp_config_file)
560 2cbdb63f Ilias Tsitsimpis
561 2cbdb63f Ilias Tsitsimpis
            # Find a uniq build_id to use
562 c7828946 Ilias Tsitsimpis
            if self.build_id is None:
563 c7828946 Ilias Tsitsimpis
                ids = self.temp_config.sections()
564 c7828946 Ilias Tsitsimpis
                if ids:
565 c7828946 Ilias Tsitsimpis
                    max_id = int(max(self.temp_config.sections(), key=int))
566 c7828946 Ilias Tsitsimpis
                    self.build_id = max_id + 1
567 c7828946 Ilias Tsitsimpis
                else:
568 c7828946 Ilias Tsitsimpis
                    self.build_id = 1
569 c7828946 Ilias Tsitsimpis
            self.logger.debug("Will use \"%s\" as build id"
570 2cbdb63f Ilias Tsitsimpis
                              % _green(self.build_id))
571 2cbdb63f Ilias Tsitsimpis
572 2cbdb63f Ilias Tsitsimpis
            # Create a new section
573 329705c8 Ilias Tsitsimpis
            try:
574 329705c8 Ilias Tsitsimpis
                self.temp_config.add_section(str(self.build_id))
575 329705c8 Ilias Tsitsimpis
            except DuplicateSectionError:
576 329705c8 Ilias Tsitsimpis
                msg = ("Build id \"%s\" already in use. " +
577 329705c8 Ilias Tsitsimpis
                       "Please use a uniq one or cleanup \"%s\" file.\n") \
578 329705c8 Ilias Tsitsimpis
                    % (self.build_id, self.temp_config_file)
579 329705c8 Ilias Tsitsimpis
                self.logger.error(msg)
580 329705c8 Ilias Tsitsimpis
                sys.exit(1)
581 2cbdb63f Ilias Tsitsimpis
            creation_time = \
582 2cbdb63f Ilias Tsitsimpis
                time.strftime("%a, %d %b %Y %X", time.localtime())
583 2cbdb63f Ilias Tsitsimpis
            self.temp_config.set(str(self.build_id),
584 2cbdb63f Ilias Tsitsimpis
                                 "created", str(creation_time))
585 2cbdb63f Ilias Tsitsimpis
586 2cbdb63f Ilias Tsitsimpis
            # Write changes back to temp config file
587 2cbdb63f Ilias Tsitsimpis
            with open(self.temp_config_file, 'wb') as tcf:
588 2cbdb63f Ilias Tsitsimpis
                self.temp_config.write(tcf)
589 2cbdb63f Ilias Tsitsimpis
590 b7496c88 Ilias Tsitsimpis
    def write_temp_config(self, option, value):
591 c29ac11d Ilias Tsitsimpis
        """Write changes back to config file"""
592 62fcf0e5 Ilias Tsitsimpis
        # Acquire the lock to write to temp_config_file
593 62fcf0e5 Ilias Tsitsimpis
        with filelocker.lock("%s.lock" % self.temp_config_file,
594 62fcf0e5 Ilias Tsitsimpis
                             filelocker.LOCK_EX):
595 62fcf0e5 Ilias Tsitsimpis
596 62fcf0e5 Ilias Tsitsimpis
            # Read temp_config again to get any new entries
597 62fcf0e5 Ilias Tsitsimpis
            self.temp_config.read(self.temp_config_file)
598 62fcf0e5 Ilias Tsitsimpis
599 62fcf0e5 Ilias Tsitsimpis
            self.temp_config.set(str(self.build_id), option, str(value))
600 62fcf0e5 Ilias Tsitsimpis
            curr_time = time.strftime("%a, %d %b %Y %X", time.localtime())
601 62fcf0e5 Ilias Tsitsimpis
            self.temp_config.set(str(self.build_id), "modified", curr_time)
602 2cbdb63f Ilias Tsitsimpis
603 2cbdb63f Ilias Tsitsimpis
            # Write changes back to temp config file
604 62fcf0e5 Ilias Tsitsimpis
            with open(self.temp_config_file, 'wb') as tcf:
605 62fcf0e5 Ilias Tsitsimpis
                self.temp_config.write(tcf)
606 c29ac11d Ilias Tsitsimpis
607 b7496c88 Ilias Tsitsimpis
    def read_temp_config(self, option):
608 b7496c88 Ilias Tsitsimpis
        """Read from temporary_config file"""
609 b7496c88 Ilias Tsitsimpis
        # If build_id is None use the latest one
610 b7496c88 Ilias Tsitsimpis
        if self.build_id is None:
611 b7496c88 Ilias Tsitsimpis
            ids = self.temp_config.sections()
612 b7496c88 Ilias Tsitsimpis
            if ids:
613 b7496c88 Ilias Tsitsimpis
                self.build_id = int(ids[-1])
614 b7496c88 Ilias Tsitsimpis
            else:
615 b7496c88 Ilias Tsitsimpis
                self.logger.error("No sections in temporary config file")
616 b7496c88 Ilias Tsitsimpis
                sys.exit(1)
617 b7496c88 Ilias Tsitsimpis
            self.logger.debug("Will use \"%s\" as build id"
618 b7496c88 Ilias Tsitsimpis
                              % _green(self.build_id))
619 b7496c88 Ilias Tsitsimpis
        # Read specified option
620 b7496c88 Ilias Tsitsimpis
        return self.temp_config.get(str(self.build_id), option)
621 b7496c88 Ilias Tsitsimpis
622 c29ac11d Ilias Tsitsimpis
    def setup_fabric(self):
623 c29ac11d Ilias Tsitsimpis
        """Setup fabric environment"""
624 c29ac11d Ilias Tsitsimpis
        self.logger.info("Setup fabric parameters..")
625 b7496c88 Ilias Tsitsimpis
        fabric.env.user = self.read_temp_config('server_user')
626 b7496c88 Ilias Tsitsimpis
        fabric.env.host_string = self.read_temp_config('server_ip')
627 b7496c88 Ilias Tsitsimpis
        fabric.env.port = int(self.read_temp_config('server_port'))
628 b7496c88 Ilias Tsitsimpis
        fabric.env.password = self.read_temp_config('server_passwd')
629 c29ac11d Ilias Tsitsimpis
        fabric.env.connection_attempts = 10
630 c29ac11d Ilias Tsitsimpis
        fabric.env.shell = "/bin/bash -c"
631 c29ac11d Ilias Tsitsimpis
        fabric.env.disable_known_hosts = True
632 c29ac11d Ilias Tsitsimpis
        fabric.env.output_prefix = None
633 c29ac11d Ilias Tsitsimpis
634 c29ac11d Ilias Tsitsimpis
    def _check_hash_sum(self, localfile, remotefile):
635 c29ac11d Ilias Tsitsimpis
        """Check hash sums of two files"""
636 c29ac11d Ilias Tsitsimpis
        self.logger.debug("Check hash sum for local file %s" % localfile)
637 c29ac11d Ilias Tsitsimpis
        hash1 = os.popen("sha256sum %s" % localfile).read().split(' ')[0]
638 c29ac11d Ilias Tsitsimpis
        self.logger.debug("Local file has sha256 hash %s" % hash1)
639 c29ac11d Ilias Tsitsimpis
        self.logger.debug("Check hash sum for remote file %s" % remotefile)
640 c29ac11d Ilias Tsitsimpis
        hash2 = _run("sha256sum %s" % remotefile, False)
641 c29ac11d Ilias Tsitsimpis
        hash2 = hash2.split(' ')[0]
642 c29ac11d Ilias Tsitsimpis
        self.logger.debug("Remote file has sha256 hash %s" % hash2)
643 c29ac11d Ilias Tsitsimpis
        if hash1 != hash2:
644 c29ac11d Ilias Tsitsimpis
            self.logger.error("Hashes differ.. aborting")
645 99d39120 Ilias Tsitsimpis
            sys.exit(1)
646 c29ac11d Ilias Tsitsimpis
647 c29ac11d Ilias Tsitsimpis
    @_check_fabric
648 5323404f Ilias Tsitsimpis
    def clone_repo(self, local_repo=False):
649 c29ac11d Ilias Tsitsimpis
        """Clone Synnefo repo from slave server"""
650 c29ac11d Ilias Tsitsimpis
        self.logger.info("Configure repositories on remote server..")
651 6c3cc77e Ilias Tsitsimpis
        self.logger.debug("Install/Setup git")
652 c29ac11d Ilias Tsitsimpis
        cmd = """
653 71f2649f Ilias Tsitsimpis
        apt-get install git --yes --force-yes
654 6c3cc77e Ilias Tsitsimpis
        git config --global user.name {0}
655 6c3cc77e Ilias Tsitsimpis
        git config --global user.email {1}
656 6c3cc77e Ilias Tsitsimpis
        """.format(self.config.get('Global', 'git_config_name'),
657 c29ac11d Ilias Tsitsimpis
                   self.config.get('Global', 'git_config_mail'))
658 c29ac11d Ilias Tsitsimpis
        _run(cmd, False)
659 c29ac11d Ilias Tsitsimpis
660 99d39120 Ilias Tsitsimpis
        # Clone synnefo_repo
661 99d39120 Ilias Tsitsimpis
        synnefo_branch = self.clone_synnefo_repo(local_repo=local_repo)
662 99d39120 Ilias Tsitsimpis
        # Clone pithos-web-client
663 99d39120 Ilias Tsitsimpis
        self.clone_pithos_webclient_repo(synnefo_branch)
664 99d39120 Ilias Tsitsimpis
665 99d39120 Ilias Tsitsimpis
    @_check_fabric
666 99d39120 Ilias Tsitsimpis
    def clone_synnefo_repo(self, local_repo=False):
667 99d39120 Ilias Tsitsimpis
        """Clone Synnefo repo to remote server"""
668 5323404f Ilias Tsitsimpis
        # Find synnefo_repo and synnefo_branch to use
669 c29ac11d Ilias Tsitsimpis
        synnefo_repo = self.config.get('Global', 'synnefo_repo')
670 b69b55ca Christos Stavrakakis
        synnefo_branch = self.config.get("Global", "synnefo_branch")
671 b69b55ca Christos Stavrakakis
        if synnefo_branch == "":
672 e2a0abb8 Ilias Tsitsimpis
            synnefo_branch = \
673 e2a0abb8 Ilias Tsitsimpis
                subprocess.Popen(
674 e2a0abb8 Ilias Tsitsimpis
                    ["git", "rev-parse", "--abbrev-ref", "HEAD"],
675 b69b55ca Christos Stavrakakis
                    stdout=subprocess.PIPE).communicate()[0].strip()
676 b69b55ca Christos Stavrakakis
            if synnefo_branch == "HEAD":
677 b69b55ca Christos Stavrakakis
                synnefo_branch = \
678 e2a0abb8 Ilias Tsitsimpis
                    subprocess.Popen(
679 e2a0abb8 Ilias Tsitsimpis
                        ["git", "rev-parse", "--short", "HEAD"],
680 b69b55ca Christos Stavrakakis
                        stdout=subprocess.PIPE).communicate()[0].strip()
681 99d39120 Ilias Tsitsimpis
        self.logger.debug("Will use branch \"%s\"" % _green(synnefo_branch))
682 5323404f Ilias Tsitsimpis
683 99d39120 Ilias Tsitsimpis
        if local_repo or synnefo_repo == "":
684 5323404f Ilias Tsitsimpis
            # Use local_repo
685 5323404f Ilias Tsitsimpis
            self.logger.debug("Push local repo to server")
686 5323404f Ilias Tsitsimpis
            # Firstly create the remote repo
687 5323404f Ilias Tsitsimpis
            _run("git init synnefo", False)
688 5323404f Ilias Tsitsimpis
            # Then push our local repo over ssh
689 5323404f Ilias Tsitsimpis
            # We have to pass some arguments to ssh command
690 5323404f Ilias Tsitsimpis
            # namely to disable host checking.
691 5323404f Ilias Tsitsimpis
            (temp_ssh_file_handle, temp_ssh_file) = tempfile.mkstemp()
692 5323404f Ilias Tsitsimpis
            os.close(temp_ssh_file_handle)
693 62fcf0e5 Ilias Tsitsimpis
            # XXX: git push doesn't read the password
694 5323404f Ilias Tsitsimpis
            cmd = """
695 5323404f Ilias Tsitsimpis
            echo 'exec ssh -o "StrictHostKeyChecking no" \
696 5323404f Ilias Tsitsimpis
                           -o "UserKnownHostsFile /dev/null" \
697 5323404f Ilias Tsitsimpis
                           -q "$@"' > {4}
698 5323404f Ilias Tsitsimpis
            chmod u+x {4}
699 5323404f Ilias Tsitsimpis
            export GIT_SSH="{4}"
700 99d39120 Ilias Tsitsimpis
            echo "{0}" | git push --quiet --mirror ssh://{1}@{2}:{3}/~/synnefo
701 5323404f Ilias Tsitsimpis
            rm -f {4}
702 5323404f Ilias Tsitsimpis
            """.format(fabric.env.password,
703 5323404f Ilias Tsitsimpis
                       fabric.env.user,
704 5323404f Ilias Tsitsimpis
                       fabric.env.host_string,
705 5323404f Ilias Tsitsimpis
                       fabric.env.port,
706 5323404f Ilias Tsitsimpis
                       temp_ssh_file)
707 5323404f Ilias Tsitsimpis
            os.system(cmd)
708 5323404f Ilias Tsitsimpis
        else:
709 5323404f Ilias Tsitsimpis
            # Clone Synnefo from remote repo
710 99d39120 Ilias Tsitsimpis
            self.logger.debug("Clone synnefo from %s" % synnefo_repo)
711 99d39120 Ilias Tsitsimpis
            self._git_clone(synnefo_repo)
712 5323404f Ilias Tsitsimpis
713 5323404f Ilias Tsitsimpis
        # Checkout the desired synnefo_branch
714 07710fc3 Ilias Tsitsimpis
        self.logger.debug("Checkout \"%s\" branch/commit" % synnefo_branch)
715 e2a0abb8 Ilias Tsitsimpis
        cmd = """
716 b69b55ca Christos Stavrakakis
        cd synnefo
717 99d39120 Ilias Tsitsimpis
        for branch in `git branch -a | grep remotes | grep -v HEAD`; do
718 b69b55ca Christos Stavrakakis
            git branch --track ${branch##*/} $branch
719 b69b55ca Christos Stavrakakis
        done
720 b69b55ca Christos Stavrakakis
        git checkout %s
721 b69b55ca Christos Stavrakakis
        """ % (synnefo_branch)
722 b69b55ca Christos Stavrakakis
        _run(cmd, False)
723 b69b55ca Christos Stavrakakis
724 99d39120 Ilias Tsitsimpis
        return synnefo_branch
725 99d39120 Ilias Tsitsimpis
726 c29ac11d Ilias Tsitsimpis
    @_check_fabric
727 99d39120 Ilias Tsitsimpis
    def clone_pithos_webclient_repo(self, synnefo_branch):
728 99d39120 Ilias Tsitsimpis
        """Clone Pithos WebClient repo to remote server"""
729 99d39120 Ilias Tsitsimpis
        # Find pithos_webclient_repo and pithos_webclient_branch to use
730 99d39120 Ilias Tsitsimpis
        pithos_webclient_repo = \
731 99d39120 Ilias Tsitsimpis
            self.config.get('Global', 'pithos_webclient_repo')
732 99d39120 Ilias Tsitsimpis
        pithos_webclient_branch = \
733 99d39120 Ilias Tsitsimpis
            self.config.get('Global', 'pithos_webclient_branch')
734 99d39120 Ilias Tsitsimpis
735 99d39120 Ilias Tsitsimpis
        # Clone pithos-webclient from remote repo
736 99d39120 Ilias Tsitsimpis
        self.logger.debug("Clone pithos-webclient from %s" %
737 99d39120 Ilias Tsitsimpis
                          pithos_webclient_repo)
738 99d39120 Ilias Tsitsimpis
        self._git_clone(pithos_webclient_repo)
739 99d39120 Ilias Tsitsimpis
740 99d39120 Ilias Tsitsimpis
        # Track all pithos-webclient branches
741 99d39120 Ilias Tsitsimpis
        cmd = """
742 99d39120 Ilias Tsitsimpis
        cd pithos-web-client
743 99d39120 Ilias Tsitsimpis
        for branch in `git branch -a | grep remotes | grep -v HEAD`; do
744 99d39120 Ilias Tsitsimpis
            git branch --track ${branch##*/} $branch > /dev/null 2>&1
745 99d39120 Ilias Tsitsimpis
        done
746 cd7282a1 Ilias Tsitsimpis
        git --no-pager branch --no-color
747 99d39120 Ilias Tsitsimpis
        """
748 99d39120 Ilias Tsitsimpis
        webclient_branches = _run(cmd, False)
749 99d39120 Ilias Tsitsimpis
        webclient_branches = webclient_branches.split()
750 99d39120 Ilias Tsitsimpis
751 99d39120 Ilias Tsitsimpis
        # If we have pithos_webclient_branch in config file use this one
752 99d39120 Ilias Tsitsimpis
        # else try to use the same branch as synnefo_branch
753 99d39120 Ilias Tsitsimpis
        # else use an appropriate one.
754 99d39120 Ilias Tsitsimpis
        if pithos_webclient_branch == "":
755 99d39120 Ilias Tsitsimpis
            if synnefo_branch in webclient_branches:
756 99d39120 Ilias Tsitsimpis
                pithos_webclient_branch = synnefo_branch
757 99d39120 Ilias Tsitsimpis
            else:
758 99d39120 Ilias Tsitsimpis
                # If synnefo_branch starts with one of
759 99d39120 Ilias Tsitsimpis
                # 'master', 'hotfix'; use the master branch
760 99d39120 Ilias Tsitsimpis
                if synnefo_branch.startswith('master') or \
761 99d39120 Ilias Tsitsimpis
                        synnefo_branch.startswith('hotfix'):
762 99d39120 Ilias Tsitsimpis
                    pithos_webclient_branch = "master"
763 99d39120 Ilias Tsitsimpis
                # If synnefo_branch starts with one of
764 99d39120 Ilias Tsitsimpis
                # 'develop', 'feature'; use the develop branch
765 99d39120 Ilias Tsitsimpis
                elif synnefo_branch.startswith('develop') or \
766 99d39120 Ilias Tsitsimpis
                        synnefo_branch.startswith('feature'):
767 99d39120 Ilias Tsitsimpis
                    pithos_webclient_branch = "develop"
768 99d39120 Ilias Tsitsimpis
                else:
769 7a8e156f Ilias Tsitsimpis
                    self.logger.warning(
770 99d39120 Ilias Tsitsimpis
                        "Cannot determine which pithos-web-client branch to "
771 99d39120 Ilias Tsitsimpis
                        "use based on \"%s\" synnefo branch. "
772 99d39120 Ilias Tsitsimpis
                        "Will use develop." % synnefo_branch)
773 99d39120 Ilias Tsitsimpis
                    pithos_webclient_branch = "develop"
774 99d39120 Ilias Tsitsimpis
        # Checkout branch
775 99d39120 Ilias Tsitsimpis
        self.logger.debug("Checkout \"%s\" branch" %
776 99d39120 Ilias Tsitsimpis
                          _green(pithos_webclient_branch))
777 99d39120 Ilias Tsitsimpis
        cmd = """
778 99d39120 Ilias Tsitsimpis
        cd pithos-web-client
779 99d39120 Ilias Tsitsimpis
        git checkout {0}
780 99d39120 Ilias Tsitsimpis
        """.format(pithos_webclient_branch)
781 99d39120 Ilias Tsitsimpis
        _run(cmd, False)
782 99d39120 Ilias Tsitsimpis
783 99d39120 Ilias Tsitsimpis
    def _git_clone(self, repo):
784 99d39120 Ilias Tsitsimpis
        """Clone repo to remote server
785 99d39120 Ilias Tsitsimpis

786 99d39120 Ilias Tsitsimpis
        Currently clonning from code.grnet.gr can fail unexpectedly.
787 99d39120 Ilias Tsitsimpis
        So retry!!
788 99d39120 Ilias Tsitsimpis

789 99d39120 Ilias Tsitsimpis
        """
790 99d39120 Ilias Tsitsimpis
        cloned = False
791 99d39120 Ilias Tsitsimpis
        for i in range(1, 11):
792 99d39120 Ilias Tsitsimpis
            try:
793 99d39120 Ilias Tsitsimpis
                _run("git clone %s" % repo, False)
794 99d39120 Ilias Tsitsimpis
                cloned = True
795 99d39120 Ilias Tsitsimpis
                break
796 99d39120 Ilias Tsitsimpis
            except BaseException:
797 99d39120 Ilias Tsitsimpis
                self.logger.warning("Clonning failed.. retrying %s/10" % i)
798 99d39120 Ilias Tsitsimpis
        if not cloned:
799 99d39120 Ilias Tsitsimpis
            self.logger.error("Can not clone repo.")
800 99d39120 Ilias Tsitsimpis
            sys.exit(1)
801 99d39120 Ilias Tsitsimpis
802 99d39120 Ilias Tsitsimpis
    @_check_fabric
803 99d39120 Ilias Tsitsimpis
    def build_packages(self):
804 99d39120 Ilias Tsitsimpis
        """Build packages needed by Synnefo software"""
805 99d39120 Ilias Tsitsimpis
        self.logger.info("Install development packages")
806 c29ac11d Ilias Tsitsimpis
        cmd = """
807 c29ac11d Ilias Tsitsimpis
        apt-get update
808 c29ac11d Ilias Tsitsimpis
        apt-get install zlib1g-dev dpkg-dev debhelper git-buildpackage \
809 99d39120 Ilias Tsitsimpis
                python-dev python-all python-pip ant --yes --force-yes
810 6a99aca3 Ilias Tsitsimpis
        pip install -U devflow
811 6a99aca3 Ilias Tsitsimpis
        """
812 c29ac11d Ilias Tsitsimpis
        _run(cmd, False)
813 c29ac11d Ilias Tsitsimpis
814 99d39120 Ilias Tsitsimpis
        # Patch pydist bug
815 e908a902 Christos Stavrakakis
        if self.config.get('Global', 'patch_pydist') == "True":
816 c29ac11d Ilias Tsitsimpis
            self.logger.debug("Patch pydist.py module")
817 c29ac11d Ilias Tsitsimpis
            cmd = r"""
818 c29ac11d Ilias Tsitsimpis
            sed -r -i 's/(\(\?P<name>\[A-Za-z\]\[A-Za-z0-9_\.)/\1\\\-/' \
819 c29ac11d Ilias Tsitsimpis
                /usr/share/python/debpython/pydist.py
820 c29ac11d Ilias Tsitsimpis
            """
821 c29ac11d Ilias Tsitsimpis
            _run(cmd, False)
822 c29ac11d Ilias Tsitsimpis

823 1c230fd8 Ilias Tsitsimpis
        # Build synnefo packages
824 99d39120 Ilias Tsitsimpis
        self.build_synnefo()
825 99d39120 Ilias Tsitsimpis
        # Build pithos-web-client packages
826 99d39120 Ilias Tsitsimpis
        self.build_pithos_webclient()
827 99d39120 Ilias Tsitsimpis

828 99d39120 Ilias Tsitsimpis
    @_check_fabric
829 99d39120 Ilias Tsitsimpis
    def build_synnefo(self):
830 99d39120 Ilias Tsitsimpis
        """Build Synnefo packages"""
831 99d39120 Ilias Tsitsimpis
        self.logger.info("Build Synnefo packages..")
832 99d39120 Ilias Tsitsimpis

833 c29ac11d Ilias Tsitsimpis
        cmd = """
834 1c230fd8 Ilias Tsitsimpis
        devflow-autopkg snapshot -b ~/synnefo_build-area --no-sign
835 c29ac11d Ilias Tsitsimpis
        """
836 1c230fd8 Ilias Tsitsimpis
        with fabric.cd("synnefo"):
837 c29ac11d Ilias Tsitsimpis
            _run(cmd, True)
838 c29ac11d Ilias Tsitsimpis

839 1c230fd8 Ilias Tsitsimpis
        # Install snf-deploy package
840 c29ac11d Ilias Tsitsimpis
        self.logger.debug("Install snf-deploy package")
841 c29ac11d Ilias Tsitsimpis
        cmd = """
842 c29ac11d Ilias Tsitsimpis
        dpkg -i snf-deploy*.deb
843 71f2649f Ilias Tsitsimpis
        apt-get -f install --yes --force-yes
844 c29ac11d Ilias Tsitsimpis
        """
845 1c230fd8 Ilias Tsitsimpis
        with fabric.cd("synnefo_build-area"):
846 c29ac11d Ilias Tsitsimpis
            with fabric.settings(warn_only=True):
847 c29ac11d Ilias Tsitsimpis
                _run(cmd, True)
848 c29ac11d Ilias Tsitsimpis

849 1c230fd8 Ilias Tsitsimpis
        # Setup synnefo packages for snf-deploy
850 c29ac11d Ilias Tsitsimpis
        self.logger.debug("Copy synnefo debs to snf-deploy packages dir")
851 c29ac11d Ilias Tsitsimpis
        cmd = """
852 c29ac11d Ilias Tsitsimpis
        cp ~/synnefo_build-area/*.deb /var/lib/snf-deploy/packages/
853 c29ac11d Ilias Tsitsimpis
        """
854 c29ac11d Ilias Tsitsimpis
        _run(cmd, False)
855 c29ac11d Ilias Tsitsimpis

856 88e6558b Christos Stavrakakis
    @_check_fabric
857 99d39120 Ilias Tsitsimpis
    def build_pithos_webclient(self):
858 99d39120 Ilias Tsitsimpis
        """Build pithos-web-client packages"""
859 99d39120 Ilias Tsitsimpis
        self.logger.info("Build pithos-web-client packages..")
860 99d39120 Ilias Tsitsimpis

861 99d39120 Ilias Tsitsimpis
        cmd = """
862 99d39120 Ilias Tsitsimpis
        devflow-autopkg snapshot -b ~/webclient_build-area --no-sign
863 99d39120 Ilias Tsitsimpis
        """
864 99d39120 Ilias Tsitsimpis
        with fabric.cd("pithos-web-client"):
865 99d39120 Ilias Tsitsimpis
            _run(cmd, True)
866 99d39120 Ilias Tsitsimpis

867 99d39120 Ilias Tsitsimpis
        # Setup pithos-web-client packages for snf-deploy
868 99d39120 Ilias Tsitsimpis
        self.logger.debug("Copy webclient debs to snf-deploy packages dir")
869 99d39120 Ilias Tsitsimpis
        cmd = """
870 99d39120 Ilias Tsitsimpis
        cp ~/webclient_build-area/*.deb /var/lib/snf-deploy/packages/
871 99d39120 Ilias Tsitsimpis
        """
872 99d39120 Ilias Tsitsimpis
        _run(cmd, False)
873 99d39120 Ilias Tsitsimpis

874 99d39120 Ilias Tsitsimpis
    @_check_fabric
875 88e6558b Christos Stavrakakis
    def build_documentation(self):
876 e2a0abb8 Ilias Tsitsimpis
        """Build Synnefo documentation"""
877 88e6558b Christos Stavrakakis
        self.logger.info("Build Synnefo documentation..")
878 88e6558b Christos Stavrakakis
        _run("pip install -U Sphinx", False)
879 88e6558b Christos Stavrakakis
        with fabric.cd("synnefo"):
880 e2a0abb8 Ilias Tsitsimpis
            _run("devflow-update-version; "
881 e2a0abb8 Ilias Tsitsimpis
                 "./ci/make_docs.sh synnefo_documentation", False)
882 88e6558b Christos Stavrakakis

883 88e6558b Christos Stavrakakis
    def fetch_documentation(self, dest=None):
884 e2a0abb8 Ilias Tsitsimpis
        """Fetch Synnefo documentation"""
885 e2a0abb8 Ilias Tsitsimpis
        self.logger.info("Fetch Synnefo documentation..")
886 88e6558b Christos Stavrakakis
        if dest is None:
887 88e6558b Christos Stavrakakis
            dest = "synnefo_documentation"
888 88e6558b Christos Stavrakakis
        dest = os.path.abspath(dest)
889 88e6558b Christos Stavrakakis
        if not os.path.exists(dest):
890 88e6558b Christos Stavrakakis
            os.makedirs(dest)
891 88e6558b Christos Stavrakakis
        self.fetch_compressed("synnefo/synnefo_documentation", dest)
892 88e6558b Christos Stavrakakis
        self.logger.info("Downloaded documentation to %s" %
893 88e6558b Christos Stavrakakis
                         _green(dest))
894 88e6558b Christos Stavrakakis

895 c29ac11d Ilias Tsitsimpis
    @_check_fabric
896 ec8bc030 Christos Stavrakakis
    def deploy_synnefo(self, schema=None):
897 c29ac11d Ilias Tsitsimpis
        """Deploy Synnefo using snf-deploy"""
898 c29ac11d Ilias Tsitsimpis
        self.logger.info("Deploy Synnefo..")
899 ec8bc030 Christos Stavrakakis
        if schema is None:
900 ec8bc030 Christos Stavrakakis
            schema = self.config.get('Global', 'schema')
901 106659f1 Ilias Tsitsimpis
        self.logger.debug("Will use \"%s\" schema" % _green(schema))
902 c29ac11d Ilias Tsitsimpis

903 ec8bc030 Christos Stavrakakis
        schema_dir = os.path.join(self.ci_dir, "schemas/%s" % schema)
904 ec8bc030 Christos Stavrakakis
        if not (os.path.exists(schema_dir) and os.path.isdir(schema_dir)):
905 ec8bc030 Christos Stavrakakis
            raise ValueError("Unknown schema: %s" % schema)
906 ec8bc030 Christos Stavrakakis

907 c29ac11d Ilias Tsitsimpis
        self.logger.debug("Upload schema files to server")
908 6868804a Ilias Tsitsimpis
        _put(os.path.join(schema_dir, "*"), "/etc/snf-deploy/")
909 c29ac11d Ilias Tsitsimpis

910 c29ac11d Ilias Tsitsimpis
        self.logger.debug("Change password in nodes.conf file")
911 c29ac11d Ilias Tsitsimpis
        cmd = """
912 c29ac11d Ilias Tsitsimpis
        sed -i 's/^password =.*/password = {0}/' /etc/snf-deploy/nodes.conf
913 3c3bccab Dimitris Aragiorgis
        sed -i 's/12345/{0}/' /etc/snf-deploy/nodes.conf
914 c29ac11d Ilias Tsitsimpis
        """.format(fabric.env.password)
915 c29ac11d Ilias Tsitsimpis
        _run(cmd, False)
916 c29ac11d Ilias Tsitsimpis

917 c29ac11d Ilias Tsitsimpis
        self.logger.debug("Run snf-deploy")
918 c29ac11d Ilias Tsitsimpis
        cmd = """
919 985442f5 Ilias Tsitsimpis
        snf-deploy keygen --force
920 f82ce32a Christos Stavrakakis
        snf-deploy --disable-colors --autoconf all
921 c29ac11d Ilias Tsitsimpis
        """
922 c29ac11d Ilias Tsitsimpis
        _run(cmd, True)
923 c29ac11d Ilias Tsitsimpis

924 c29ac11d Ilias Tsitsimpis
    @_check_fabric
925 c29ac11d Ilias Tsitsimpis
    def unit_test(self):
926 c29ac11d Ilias Tsitsimpis
        """Run Synnefo unit test suite"""
927 c29ac11d Ilias Tsitsimpis
        self.logger.info("Run Synnefo unit test suite")
928 c29ac11d Ilias Tsitsimpis
        component = self.config.get('Unit Tests', 'component')
929 c29ac11d Ilias Tsitsimpis

930 c29ac11d Ilias Tsitsimpis
        self.logger.debug("Install needed packages")
931 c29ac11d Ilias Tsitsimpis
        cmd = """
932 8d2c08e4 Christos Stavrakakis
        pip install -U mock
933 8d2c08e4 Christos Stavrakakis
        pip install -U factory_boy
934 cb1060f2 Ilias Tsitsimpis
        pip install -U nose
935 c29ac11d Ilias Tsitsimpis
        """
936 c29ac11d Ilias Tsitsimpis
        _run(cmd, False)
937 c29ac11d Ilias Tsitsimpis

938 861b437e Ilias Tsitsimpis
        self.logger.debug("Upload tests.sh file")
939 861b437e Ilias Tsitsimpis
        unit_tests_file = os.path.join(self.ci_dir, "tests.sh")
940 6868804a Ilias Tsitsimpis
        _put(unit_tests_file, ".")
941 c29ac11d Ilias Tsitsimpis

942 c29ac11d Ilias Tsitsimpis
        self.logger.debug("Run unit tests")
943 c29ac11d Ilias Tsitsimpis
        cmd = """
944 861b437e Ilias Tsitsimpis
        bash tests.sh {0}
945 c29ac11d Ilias Tsitsimpis
        """.format(component)
946 c29ac11d Ilias Tsitsimpis
        _run(cmd, True)
947 c29ac11d Ilias Tsitsimpis

948 c29ac11d Ilias Tsitsimpis
    @_check_fabric
949 c29ac11d Ilias Tsitsimpis
    def run_burnin(self):
950 c29ac11d Ilias Tsitsimpis
        """Run burnin functional test suite"""
951 c29ac11d Ilias Tsitsimpis
        self.logger.info("Run Burnin functional test suite")
952 c29ac11d Ilias Tsitsimpis
        cmd = """
953 c29ac11d Ilias Tsitsimpis
        auth_url=$(grep -e '^url =' .kamakirc | cut -d' ' -f3)
954 c29ac11d Ilias Tsitsimpis
        token=$(grep -e '^token =' .kamakirc | cut -d' ' -f3)
955 c29ac11d Ilias Tsitsimpis
        images_user=$(kamaki image list -l | grep owner | \
956 c29ac11d Ilias Tsitsimpis
                      cut -d':' -f2 | tr -d ' ')
957 2afd10bf Ilias Tsitsimpis
        snf-burnin --auth-url=$auth_url --token=$token {0}
958 281c7634 Ilias Tsitsimpis
        BurninExitStatus=$?
959 281c7634 Ilias Tsitsimpis
        exit $BurninExitStatus
960 c29ac11d Ilias Tsitsimpis
        """.format(self.config.get('Burnin', 'cmd_options'))
961 c29ac11d Ilias Tsitsimpis
        _run(cmd, True)
962 c29ac11d Ilias Tsitsimpis

963 c29ac11d Ilias Tsitsimpis
    @_check_fabric
964 88e6558b Christos Stavrakakis
    def fetch_compressed(self, src, dest=None):
965 e2a0abb8 Ilias Tsitsimpis
        """Create a tarball and fetch it locally"""
966 88e6558b Christos Stavrakakis
        self.logger.debug("Creating tarball of %s" % src)
967 88e6558b Christos Stavrakakis
        basename = os.path.basename(src)
968 88e6558b Christos Stavrakakis
        tar_file = basename + ".tgz"
969 88e6558b Christos Stavrakakis
        cmd = "tar czf %s %s" % (tar_file, src)
970 c29ac11d Ilias Tsitsimpis
        _run(cmd, False)
971 88e6558b Christos Stavrakakis
        if not os.path.exists(dest):
972 88e6558b Christos Stavrakakis
            os.makedirs(dest)
973 c29ac11d Ilias Tsitsimpis

974 88e6558b Christos Stavrakakis
        tmp_dir = tempfile.mkdtemp()
975 88e6558b Christos Stavrakakis
        fabric.get(tar_file, tmp_dir)
976 c29ac11d Ilias Tsitsimpis

977 88e6558b Christos Stavrakakis
        dest_file = os.path.join(tmp_dir, tar_file)
978 88e6558b Christos Stavrakakis
        self._check_hash_sum(dest_file, tar_file)
979 88e6558b Christos Stavrakakis
        self.logger.debug("Untar packages file %s" % dest_file)
980 e2dc37b2 Christos Stavrakakis
        cmd = """
981 e2dc37b2 Christos Stavrakakis
        cd %s
982 88e6558b Christos Stavrakakis
        tar xzf %s
983 88e6558b Christos Stavrakakis
        cp -r %s/* %s
984 88e6558b Christos Stavrakakis
        rm -r %s
985 88e6558b Christos Stavrakakis
        """ % (tmp_dir, tar_file, src, dest, tmp_dir)
986 e2dc37b2 Christos Stavrakakis
        os.system(cmd)
987 88e6558b Christos Stavrakakis
        self.logger.info("Downloaded %s to %s" %
988 88e6558b Christos Stavrakakis
                         (src, _green(dest)))
989 88e6558b Christos Stavrakakis

990 88e6558b Christos Stavrakakis
    @_check_fabric
991 88e6558b Christos Stavrakakis
    def fetch_packages(self, dest=None):
992 e2a0abb8 Ilias Tsitsimpis
        """Fetch Synnefo packages"""
993 88e6558b Christos Stavrakakis
        if dest is None:
994 88e6558b Christos Stavrakakis
            dest = self.config.get('Global', 'pkgs_dir')
995 6058ae72 Ilias Tsitsimpis
        dest = os.path.abspath(os.path.expanduser(dest))
996 88e6558b Christos Stavrakakis
        if not os.path.exists(dest):
997 88e6558b Christos Stavrakakis
            os.makedirs(dest)
998 88e6558b Christos Stavrakakis
        self.fetch_compressed("synnefo_build-area", dest)
999 99d39120 Ilias Tsitsimpis
        self.fetch_compressed("webclient_build-area", dest)
1000 464e58e9 Christos Stavrakakis
        self.logger.info("Downloaded debian packages to %s" %
1001 88e6558b Christos Stavrakakis
                         _green(dest))
1002 6958ffc6 Christos Stavrakakis

1003 cf99ca80 Ilias Tsitsimpis
    def x2go_plugin(self, dest=None):
1004 cf99ca80 Ilias Tsitsimpis
        """Produce an html page which will use the x2goplugin
1005 cf99ca80 Ilias Tsitsimpis
1006 cf99ca80 Ilias Tsitsimpis
        Arguments:
1007 cf99ca80 Ilias Tsitsimpis
          dest  -- The file where to save the page (String)
1008 cf99ca80 Ilias Tsitsimpis
1009 cf99ca80 Ilias Tsitsimpis
        """
1010 cf99ca80 Ilias Tsitsimpis
        output_str = """
1011 cf99ca80 Ilias Tsitsimpis
        <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
1012 cf99ca80 Ilias Tsitsimpis
        <html>
1013 cf99ca80 Ilias Tsitsimpis
        <head>
1014 cf99ca80 Ilias Tsitsimpis
        <title>X2Go SynnefoCI Service</title>
1015 cf99ca80 Ilias Tsitsimpis
        </head>
1016 cf99ca80 Ilias Tsitsimpis
        <body onload="checkPlugin()">
1017 cf99ca80 Ilias Tsitsimpis
        <div id="x2goplugin">
1018 cf99ca80 Ilias Tsitsimpis
            <object
1019 cf99ca80 Ilias Tsitsimpis
                src="location"
1020 cf99ca80 Ilias Tsitsimpis
                type="application/x2go"
1021 cf99ca80 Ilias Tsitsimpis
                name="x2goplugin"
1022 cf99ca80 Ilias Tsitsimpis
                palette="background"
1023 cf99ca80 Ilias Tsitsimpis
                height="100%"
1024 cf99ca80 Ilias Tsitsimpis
                hspace="0"
1025 cf99ca80 Ilias Tsitsimpis
                vspace="0"
1026 cf99ca80 Ilias Tsitsimpis
                width="100%"
1027 cf99ca80 Ilias Tsitsimpis
                x2goconfig="
1028 cf99ca80 Ilias Tsitsimpis
                    session=X2Go-SynnefoCI-Session
1029 cf99ca80 Ilias Tsitsimpis
                    server={0}
1030 cf99ca80 Ilias Tsitsimpis
                    user={1}
1031 cf99ca80 Ilias Tsitsimpis
                    sshport={2}
1032 cf99ca80 Ilias Tsitsimpis
                    published=true
1033 cf99ca80 Ilias Tsitsimpis
                    autologin=true
1034 cf99ca80 Ilias Tsitsimpis
                ">
1035 cf99ca80 Ilias Tsitsimpis
            </object>
1036 cf99ca80 Ilias Tsitsimpis
        </div>
1037 cf99ca80 Ilias Tsitsimpis
        </body>
1038 cf99ca80 Ilias Tsitsimpis
        </html>
1039 cf99ca80 Ilias Tsitsimpis
        """.format(self.read_temp_config('server_ip'),
1040 cf99ca80 Ilias Tsitsimpis
                   self.read_temp_config('server_user'),
1041 cf99ca80 Ilias Tsitsimpis
                   self.read_temp_config('server_port'))
1042 cf99ca80 Ilias Tsitsimpis
        if dest is None:
1043 cf99ca80 Ilias Tsitsimpis
            dest = self.config.get('Global', 'x2go_plugin_file')
1044 cf99ca80 Ilias Tsitsimpis

1045 cf99ca80 Ilias Tsitsimpis
        self.logger.info("Writting x2go plugin html file to %s" % dest)
1046 cf99ca80 Ilias Tsitsimpis
        fid = open(dest, 'w')
1047 cf99ca80 Ilias Tsitsimpis
        fid.write(output_str)
1048 cf99ca80 Ilias Tsitsimpis
        fid.close()
1049 cf99ca80 Ilias Tsitsimpis

1050 6958ffc6 Christos Stavrakakis

1051 6958ffc6 Christos Stavrakakis
def parse_typed_option(option, value):
1052 694c79bb Ilias Tsitsimpis
    """Parsed typed options (flavors and images)"""
1053 6958ffc6 Christos Stavrakakis
    try:
1054 6958ffc6 Christos Stavrakakis
        [type_, val] = value.strip().split(':')
1055 6958ffc6 Christos Stavrakakis
        if type_ not in ["id", "name"]:
1056 6958ffc6 Christos Stavrakakis
            raise ValueError
1057 6958ffc6 Christos Stavrakakis
        return type_, val
1058 6958ffc6 Christos Stavrakakis
    except ValueError:
1059 6958ffc6 Christos Stavrakakis
        msg = "Invalid %s format. Must be [id|name]:.+" % option
1060 6958ffc6 Christos Stavrakakis
        raise ValueError(msg)
1061 ee6eff28 Ilias Tsitsimpis

1062 ee6eff28 Ilias Tsitsimpis

1063 ee6eff28 Ilias Tsitsimpis
def get_endpoint_url(endpoints, endpoint_type):
1064 ee6eff28 Ilias Tsitsimpis
    """Get the publicURL for the specified endpoint"""
1065 ee6eff28 Ilias Tsitsimpis

1066 ee6eff28 Ilias Tsitsimpis
    service_catalog = parse_endpoints(endpoints, ep_type=endpoint_type)
1067 ee6eff28 Ilias Tsitsimpis
    return service_catalog[0]['endpoints'][0]['publicURL']