Statistics
| Branch: | Tag: | Revision:

root / ci / utils.py @ 09f7ad00

History | View | Annotate | Download (37.1 kB)

1 09f7ad00 Ilias Tsitsimpis
#!/usr/bin/env python
2 c29ac11d Ilias Tsitsimpis
3 c29ac11d Ilias Tsitsimpis
"""
4 c29ac11d Ilias Tsitsimpis
Synnefo ci utils module
5 c29ac11d Ilias Tsitsimpis
"""
6 c29ac11d Ilias Tsitsimpis
7 c29ac11d Ilias Tsitsimpis
import os
8 ad21ffa9 Ilias Tsitsimpis
import re
9 c29ac11d Ilias Tsitsimpis
import sys
10 c29ac11d Ilias Tsitsimpis
import time
11 c29ac11d Ilias Tsitsimpis
import logging
12 c29ac11d Ilias Tsitsimpis
import fabric.api as fabric
13 b69b55ca Christos Stavrakakis
import subprocess
14 88e6558b Christos Stavrakakis
import tempfile
15 329705c8 Ilias Tsitsimpis
from ConfigParser import ConfigParser, DuplicateSectionError
16 c29ac11d Ilias Tsitsimpis
17 c1b1d444 Christos Stavrakakis
from kamaki.cli import config as kamaki_config
18 c29ac11d Ilias Tsitsimpis
from kamaki.clients.astakos import AstakosClient
19 c29ac11d Ilias Tsitsimpis
from kamaki.clients.cyclades import CycladesClient
20 c29ac11d Ilias Tsitsimpis
from kamaki.clients.image import ImageClient
21 ad21ffa9 Ilias Tsitsimpis
from kamaki.clients.compute import ComputeClient
22 62fcf0e5 Ilias Tsitsimpis
import filelocker
23 c29ac11d Ilias Tsitsimpis
24 7585f464 Ilias Tsitsimpis
DEFAULT_CONFIG_FILE = "ci_wheezy.conf"
25 106659f1 Ilias Tsitsimpis
# Is our terminal a colorful one?
26 106659f1 Ilias Tsitsimpis
USE_COLORS = True
27 62f3f54f Ilias Tsitsimpis
# UUID of owner of system images
28 62f3f54f Ilias Tsitsimpis
DEFAULT_SYSTEM_IMAGES_UUID = [
29 62f3f54f Ilias Tsitsimpis
    "25ecced9-bf53-4145-91ee-cf47377e9fb2",  # production (okeanos.grnet.gr)
30 62f3f54f Ilias Tsitsimpis
    "04cbe33f-29b7-4ef1-94fb-015929e5fc06",  # testing (okeanos.io)
31 905bb10b Christos Stavrakakis
]
32 6ca8f81a Christos Stavrakakis
33 c29ac11d Ilias Tsitsimpis
34 c29ac11d Ilias Tsitsimpis
def _run(cmd, verbose):
35 c29ac11d Ilias Tsitsimpis
    """Run fabric with verbose level"""
36 c29ac11d Ilias Tsitsimpis
    if verbose:
37 c29ac11d Ilias Tsitsimpis
        args = ('running',)
38 c29ac11d Ilias Tsitsimpis
    else:
39 c29ac11d Ilias Tsitsimpis
        args = ('running', 'stdout',)
40 e2a0abb8 Ilias Tsitsimpis
    with fabric.hide(*args):  # Used * or ** magic. pylint: disable-msg=W0142
41 c29ac11d Ilias Tsitsimpis
        return fabric.run(cmd)
42 c29ac11d Ilias Tsitsimpis
43 c29ac11d Ilias Tsitsimpis
44 6868804a Ilias Tsitsimpis
def _put(local, remote):
45 6868804a Ilias Tsitsimpis
    """Run fabric put command without output"""
46 6868804a Ilias Tsitsimpis
    with fabric.quiet():
47 6868804a Ilias Tsitsimpis
        fabric.put(local, remote)
48 6868804a Ilias Tsitsimpis
49 6868804a Ilias Tsitsimpis
50 c29ac11d Ilias Tsitsimpis
def _red(msg):
51 c29ac11d Ilias Tsitsimpis
    """Red color"""
52 106659f1 Ilias Tsitsimpis
    ret = "\x1b[31m" + str(msg) + "\x1b[0m" if USE_COLORS else str(msg)
53 106659f1 Ilias Tsitsimpis
    return ret
54 c29ac11d Ilias Tsitsimpis
55 c29ac11d Ilias Tsitsimpis
56 c29ac11d Ilias Tsitsimpis
def _yellow(msg):
57 c29ac11d Ilias Tsitsimpis
    """Yellow color"""
58 106659f1 Ilias Tsitsimpis
    ret = "\x1b[33m" + str(msg) + "\x1b[0m" if USE_COLORS else str(msg)
59 106659f1 Ilias Tsitsimpis
    return ret
60 c29ac11d Ilias Tsitsimpis
61 c29ac11d Ilias Tsitsimpis
62 c29ac11d Ilias Tsitsimpis
def _green(msg):
63 c29ac11d Ilias Tsitsimpis
    """Green color"""
64 106659f1 Ilias Tsitsimpis
    ret = "\x1b[32m" + str(msg) + "\x1b[0m" if USE_COLORS else str(msg)
65 106659f1 Ilias Tsitsimpis
    return ret
66 c29ac11d Ilias Tsitsimpis
67 c29ac11d Ilias Tsitsimpis
68 c29ac11d Ilias Tsitsimpis
def _check_fabric(fun):
69 c29ac11d Ilias Tsitsimpis
    """Check if fabric env has been set"""
70 43a22402 Christos Stavrakakis
    def wrapper(self, *args, **kwargs):
71 c29ac11d Ilias Tsitsimpis
        """wrapper function"""
72 c29ac11d Ilias Tsitsimpis
        if not self.fabric_installed:
73 c29ac11d Ilias Tsitsimpis
            self.setup_fabric()
74 6c3cc77e Ilias Tsitsimpis
            self.fabric_installed = True
75 43a22402 Christos Stavrakakis
        return fun(self, *args, **kwargs)
76 c29ac11d Ilias Tsitsimpis
    return wrapper
77 c29ac11d Ilias Tsitsimpis
78 c29ac11d Ilias Tsitsimpis
79 c29ac11d Ilias Tsitsimpis
def _check_kamaki(fun):
80 c29ac11d Ilias Tsitsimpis
    """Check if kamaki has been initialized"""
81 43a22402 Christos Stavrakakis
    def wrapper(self, *args, **kwargs):
82 c29ac11d Ilias Tsitsimpis
        """wrapper function"""
83 c29ac11d Ilias Tsitsimpis
        if not self.kamaki_installed:
84 c29ac11d Ilias Tsitsimpis
            self.setup_kamaki()
85 6c3cc77e Ilias Tsitsimpis
            self.kamaki_installed = True
86 43a22402 Christos Stavrakakis
        return fun(self, *args, **kwargs)
87 c29ac11d Ilias Tsitsimpis
    return wrapper
88 c29ac11d Ilias Tsitsimpis
89 c29ac11d Ilias Tsitsimpis
90 c29ac11d Ilias Tsitsimpis
class _MyFormatter(logging.Formatter):
91 c29ac11d Ilias Tsitsimpis
    """Logging Formatter"""
92 c29ac11d Ilias Tsitsimpis
    def format(self, record):
93 c29ac11d Ilias Tsitsimpis
        format_orig = self._fmt
94 c29ac11d Ilias Tsitsimpis
        if record.levelno == logging.DEBUG:
95 c29ac11d Ilias Tsitsimpis
            self._fmt = "  %(msg)s"
96 c29ac11d Ilias Tsitsimpis
        elif record.levelno == logging.INFO:
97 c29ac11d Ilias Tsitsimpis
            self._fmt = "%(msg)s"
98 c29ac11d Ilias Tsitsimpis
        elif record.levelno == logging.WARNING:
99 c29ac11d Ilias Tsitsimpis
            self._fmt = _yellow("[W] %(msg)s")
100 c29ac11d Ilias Tsitsimpis
        elif record.levelno == logging.ERROR:
101 c29ac11d Ilias Tsitsimpis
            self._fmt = _red("[E] %(msg)s")
102 c29ac11d Ilias Tsitsimpis
        result = logging.Formatter.format(self, record)
103 c29ac11d Ilias Tsitsimpis
        self._fmt = format_orig
104 c29ac11d Ilias Tsitsimpis
        return result
105 c29ac11d Ilias Tsitsimpis
106 c29ac11d Ilias Tsitsimpis
107 56ee1bbc Ilias Tsitsimpis
# Too few public methods. pylint: disable-msg=R0903
108 56ee1bbc Ilias Tsitsimpis
class _InfoFilter(logging.Filter):
109 56ee1bbc Ilias Tsitsimpis
    """Logging Filter that allows DEBUG and INFO messages only"""
110 56ee1bbc Ilias Tsitsimpis
    def filter(self, rec):
111 56ee1bbc Ilias Tsitsimpis
        """The filter"""
112 56ee1bbc Ilias Tsitsimpis
        return rec.levelno in (logging.DEBUG, logging.INFO)
113 56ee1bbc Ilias Tsitsimpis
114 56ee1bbc Ilias Tsitsimpis
115 56ee1bbc Ilias Tsitsimpis
# Too many instance attributes. pylint: disable-msg=R0902
116 c29ac11d Ilias Tsitsimpis
class SynnefoCI(object):
117 c29ac11d Ilias Tsitsimpis
    """SynnefoCI python class"""
118 c29ac11d Ilias Tsitsimpis
119 79144a72 Ilias Tsitsimpis
    def __init__(self, config_file=None, build_id=None, cloud=None):
120 c29ac11d Ilias Tsitsimpis
        """ Initialize SynnefoCI python class
121 c29ac11d Ilias Tsitsimpis

122 c29ac11d Ilias Tsitsimpis
        Setup logger, local_dir, config and kamaki
123 c29ac11d Ilias Tsitsimpis
        """
124 c29ac11d Ilias Tsitsimpis
        # Setup logger
125 c29ac11d Ilias Tsitsimpis
        self.logger = logging.getLogger('synnefo-ci')
126 c29ac11d Ilias Tsitsimpis
        self.logger.setLevel(logging.DEBUG)
127 56ee1bbc Ilias Tsitsimpis
128 56ee1bbc Ilias Tsitsimpis
        handler1 = logging.StreamHandler(sys.stdout)
129 56ee1bbc Ilias Tsitsimpis
        handler1.setLevel(logging.DEBUG)
130 56ee1bbc Ilias Tsitsimpis
        handler1.addFilter(_InfoFilter())
131 56ee1bbc Ilias Tsitsimpis
        handler1.setFormatter(_MyFormatter())
132 56ee1bbc Ilias Tsitsimpis
        handler2 = logging.StreamHandler(sys.stderr)
133 56ee1bbc Ilias Tsitsimpis
        handler2.setLevel(logging.WARNING)
134 56ee1bbc Ilias Tsitsimpis
        handler2.setFormatter(_MyFormatter())
135 56ee1bbc Ilias Tsitsimpis
136 56ee1bbc Ilias Tsitsimpis
        self.logger.addHandler(handler1)
137 56ee1bbc Ilias Tsitsimpis
        self.logger.addHandler(handler2)
138 c29ac11d Ilias Tsitsimpis
139 c29ac11d Ilias Tsitsimpis
        # Get our local dir
140 c29ac11d Ilias Tsitsimpis
        self.ci_dir = os.path.dirname(os.path.abspath(__file__))
141 c29ac11d Ilias Tsitsimpis
        self.repo_dir = os.path.dirname(self.ci_dir)
142 c29ac11d Ilias Tsitsimpis
143 c29ac11d Ilias Tsitsimpis
        # Read config file
144 6ca8f81a Christos Stavrakakis
        if config_file is None:
145 2c4a641b Ilias Tsitsimpis
            config_file = os.path.join(self.ci_dir, DEFAULT_CONFIG_FILE)
146 2c4a641b Ilias Tsitsimpis
        config_file = os.path.abspath(config_file)
147 c29ac11d Ilias Tsitsimpis
        self.config = ConfigParser()
148 c29ac11d Ilias Tsitsimpis
        self.config.optionxform = str
149 6ca8f81a Christos Stavrakakis
        self.config.read(config_file)
150 79144a72 Ilias Tsitsimpis
151 79144a72 Ilias Tsitsimpis
        # Read temporary_config file
152 62fcf0e5 Ilias Tsitsimpis
        self.temp_config_file = \
153 62fcf0e5 Ilias Tsitsimpis
            os.path.expanduser(self.config.get('Global', 'temporary_config'))
154 79144a72 Ilias Tsitsimpis
        self.temp_config = ConfigParser()
155 79144a72 Ilias Tsitsimpis
        self.temp_config.optionxform = str
156 62fcf0e5 Ilias Tsitsimpis
        self.temp_config.read(self.temp_config_file)
157 b7496c88 Ilias Tsitsimpis
        self.build_id = build_id
158 99d39120 Ilias Tsitsimpis
        if build_id is not None:
159 99d39120 Ilias Tsitsimpis
            self.logger.info("Will use \"%s\" as build id" %
160 99d39120 Ilias Tsitsimpis
                             _green(self.build_id))
161 c29ac11d Ilias Tsitsimpis
162 c1b1d444 Christos Stavrakakis
        # Set kamaki cloud
163 c1b1d444 Christos Stavrakakis
        if cloud is not None:
164 c1b1d444 Christos Stavrakakis
            self.kamaki_cloud = cloud
165 c1b1d444 Christos Stavrakakis
        elif self.config.has_option("Deployment", "kamaki_cloud"):
166 c1b1d444 Christos Stavrakakis
            kamaki_cloud = self.config.get("Deployment", "kamaki_cloud")
167 c1b1d444 Christos Stavrakakis
            if kamaki_cloud == "":
168 c1b1d444 Christos Stavrakakis
                self.kamaki_cloud = None
169 c1b1d444 Christos Stavrakakis
        else:
170 c1b1d444 Christos Stavrakakis
            self.kamaki_cloud = None
171 c1b1d444 Christos Stavrakakis
172 c29ac11d Ilias Tsitsimpis
        # Initialize variables
173 c29ac11d Ilias Tsitsimpis
        self.fabric_installed = False
174 c29ac11d Ilias Tsitsimpis
        self.kamaki_installed = False
175 c29ac11d Ilias Tsitsimpis
        self.cyclades_client = None
176 ad21ffa9 Ilias Tsitsimpis
        self.compute_client = None
177 c29ac11d Ilias Tsitsimpis
        self.image_client = None
178 694c79bb Ilias Tsitsimpis
        self.astakos_client = None
179 c29ac11d Ilias Tsitsimpis
180 c29ac11d Ilias Tsitsimpis
    def setup_kamaki(self):
181 c29ac11d Ilias Tsitsimpis
        """Initialize kamaki
182 c29ac11d Ilias Tsitsimpis

183 ad21ffa9 Ilias Tsitsimpis
        Setup cyclades_client, image_client and compute_client
184 c29ac11d Ilias Tsitsimpis
        """
185 c1b1d444 Christos Stavrakakis
186 c1b1d444 Christos Stavrakakis
        config = kamaki_config.Config()
187 c1b1d444 Christos Stavrakakis
        if self.kamaki_cloud is None:
188 c1b1d444 Christos Stavrakakis
            self.kamaki_cloud = config.get_global("default_cloud")
189 c1b1d444 Christos Stavrakakis
190 c1b1d444 Christos Stavrakakis
        self.logger.info("Setup kamaki client, using cloud '%s'.." %
191 c1b1d444 Christos Stavrakakis
                         self.kamaki_cloud)
192 c1b1d444 Christos Stavrakakis
        auth_url = config.get_cloud(self.kamaki_cloud, "url")
193 c29ac11d Ilias Tsitsimpis
        self.logger.debug("Authentication URL is %s" % _green(auth_url))
194 c1b1d444 Christos Stavrakakis
        token = config.get_cloud(self.kamaki_cloud, "token")
195 c29ac11d Ilias Tsitsimpis
        #self.logger.debug("Token is %s" % _green(token))
196 c29ac11d Ilias Tsitsimpis
197 905bb10b Christos Stavrakakis
        self.astakos_client = AstakosClient(auth_url, token)
198 c29ac11d Ilias Tsitsimpis
199 c29ac11d Ilias Tsitsimpis
        cyclades_url = \
200 905bb10b Christos Stavrakakis
            self.astakos_client.get_service_endpoints('compute')['publicURL']
201 c29ac11d Ilias Tsitsimpis
        self.logger.debug("Cyclades API url is %s" % _green(cyclades_url))
202 c29ac11d Ilias Tsitsimpis
        self.cyclades_client = CycladesClient(cyclades_url, token)
203 c29ac11d Ilias Tsitsimpis
        self.cyclades_client.CONNECTION_RETRY_LIMIT = 2
204 c29ac11d Ilias Tsitsimpis
205 c29ac11d Ilias Tsitsimpis
        image_url = \
206 905bb10b Christos Stavrakakis
            self.astakos_client.get_service_endpoints('image')['publicURL']
207 c29ac11d Ilias Tsitsimpis
        self.logger.debug("Images API url is %s" % _green(image_url))
208 c29ac11d Ilias Tsitsimpis
        self.image_client = ImageClient(cyclades_url, token)
209 c29ac11d Ilias Tsitsimpis
        self.image_client.CONNECTION_RETRY_LIMIT = 2
210 c29ac11d Ilias Tsitsimpis
211 ad21ffa9 Ilias Tsitsimpis
        compute_url = \
212 905bb10b Christos Stavrakakis
            self.astakos_client.get_service_endpoints('compute')['publicURL']
213 ad21ffa9 Ilias Tsitsimpis
        self.logger.debug("Compute API url is %s" % _green(compute_url))
214 ad21ffa9 Ilias Tsitsimpis
        self.compute_client = ComputeClient(compute_url, token)
215 ad21ffa9 Ilias Tsitsimpis
        self.compute_client.CONNECTION_RETRY_LIMIT = 2
216 ad21ffa9 Ilias Tsitsimpis
217 c29ac11d Ilias Tsitsimpis
    def _wait_transition(self, server_id, current_status, new_status):
218 c29ac11d Ilias Tsitsimpis
        """Wait for server to go from current_status to new_status"""
219 c29ac11d Ilias Tsitsimpis
        self.logger.debug("Waiting for server to become %s" % new_status)
220 c29ac11d Ilias Tsitsimpis
        timeout = self.config.getint('Global', 'build_timeout')
221 c29ac11d Ilias Tsitsimpis
        sleep_time = 5
222 c29ac11d Ilias Tsitsimpis
        while True:
223 c29ac11d Ilias Tsitsimpis
            server = self.cyclades_client.get_server_details(server_id)
224 c29ac11d Ilias Tsitsimpis
            if server['status'] == new_status:
225 c29ac11d Ilias Tsitsimpis
                return server
226 c29ac11d Ilias Tsitsimpis
            elif timeout < 0:
227 c29ac11d Ilias Tsitsimpis
                self.logger.error(
228 c29ac11d Ilias Tsitsimpis
                    "Waiting for server to become %s timed out" % new_status)
229 c29ac11d Ilias Tsitsimpis
                self.destroy_server(False)
230 99d39120 Ilias Tsitsimpis
                sys.exit(1)
231 c29ac11d Ilias Tsitsimpis
            elif server['status'] == current_status:
232 c29ac11d Ilias Tsitsimpis
                # Sleep for #n secs and continue
233 c29ac11d Ilias Tsitsimpis
                timeout = timeout - sleep_time
234 c29ac11d Ilias Tsitsimpis
                time.sleep(sleep_time)
235 c29ac11d Ilias Tsitsimpis
            else:
236 c29ac11d Ilias Tsitsimpis
                self.logger.error(
237 c29ac11d Ilias Tsitsimpis
                    "Server failed with status %s" % server['status'])
238 c29ac11d Ilias Tsitsimpis
                self.destroy_server(False)
239 99d39120 Ilias Tsitsimpis
                sys.exit(1)
240 c29ac11d Ilias Tsitsimpis
241 c29ac11d Ilias Tsitsimpis
    @_check_kamaki
242 c29ac11d Ilias Tsitsimpis
    def destroy_server(self, wait=True):
243 c29ac11d Ilias Tsitsimpis
        """Destroy slave server"""
244 b7496c88 Ilias Tsitsimpis
        server_id = int(self.read_temp_config('server_id'))
245 c29ac11d Ilias Tsitsimpis
        self.logger.info("Destoying server with id %s " % server_id)
246 c29ac11d Ilias Tsitsimpis
        self.cyclades_client.delete_server(server_id)
247 c29ac11d Ilias Tsitsimpis
        if wait:
248 c29ac11d Ilias Tsitsimpis
            self._wait_transition(server_id, "ACTIVE", "DELETED")
249 c29ac11d Ilias Tsitsimpis
250 c29ac11d Ilias Tsitsimpis
    @_check_kamaki
251 358a19bc Ilias Tsitsimpis
    def create_server(self, image=None, flavor=None, ssh_keys=None):
252 c29ac11d Ilias Tsitsimpis
        """Create slave server"""
253 c29ac11d Ilias Tsitsimpis
        self.logger.info("Create a new server..")
254 b7496c88 Ilias Tsitsimpis
255 b7496c88 Ilias Tsitsimpis
        # Find a build_id to use
256 c7828946 Ilias Tsitsimpis
        self._create_new_build_id()
257 b7496c88 Ilias Tsitsimpis
258 b7496c88 Ilias Tsitsimpis
        # Find an image to use
259 358a19bc Ilias Tsitsimpis
        image_id = self._find_image(image)
260 b7496c88 Ilias Tsitsimpis
        # Find a flavor to use
261 358a19bc Ilias Tsitsimpis
        flavor_id = self._find_flavor(flavor)
262 358a19bc Ilias Tsitsimpis
263 358a19bc Ilias Tsitsimpis
        # Create Server
264 07c13754 Christos Stavrakakis
        server_name = self.config.get("Deployment", "server_name")
265 c29ac11d Ilias Tsitsimpis
        server = self.cyclades_client.create_server(
266 07c13754 Christos Stavrakakis
            "%s(BID: %s)" % (server_name, self.build_id),
267 43a22402 Christos Stavrakakis
            flavor_id,
268 43a22402 Christos Stavrakakis
            image_id)
269 c29ac11d Ilias Tsitsimpis
        server_id = server['id']
270 b7496c88 Ilias Tsitsimpis
        self.write_temp_config('server_id', server_id)
271 c29ac11d Ilias Tsitsimpis
        self.logger.debug("Server got id %s" % _green(server_id))
272 c29ac11d Ilias Tsitsimpis
        server_user = server['metadata']['users']
273 b7496c88 Ilias Tsitsimpis
        self.write_temp_config('server_user', server_user)
274 c29ac11d Ilias Tsitsimpis
        self.logger.debug("Server's admin user is %s" % _green(server_user))
275 c29ac11d Ilias Tsitsimpis
        server_passwd = server['adminPass']
276 b7496c88 Ilias Tsitsimpis
        self.write_temp_config('server_passwd', server_passwd)
277 c29ac11d Ilias Tsitsimpis
278 c29ac11d Ilias Tsitsimpis
        server = self._wait_transition(server_id, "BUILD", "ACTIVE")
279 c29ac11d Ilias Tsitsimpis
        self._get_server_ip_and_port(server)
280 6868804a Ilias Tsitsimpis
        self._copy_ssh_keys(ssh_keys)
281 c29ac11d Ilias Tsitsimpis
282 6c3cc77e Ilias Tsitsimpis
        # Setup Firewall
283 c29ac11d Ilias Tsitsimpis
        self.setup_fabric()
284 c29ac11d Ilias Tsitsimpis
        self.logger.info("Setup firewall")
285 6868804a Ilias Tsitsimpis
        accept_ssh_from = self.config.get('Global', 'accept_ssh_from')
286 1c75c4dd Christos Stavrakakis
        if accept_ssh_from != "":
287 1c75c4dd Christos Stavrakakis
            self.logger.debug("Block ssh except from %s" % accept_ssh_from)
288 1c75c4dd Christos Stavrakakis
            cmd = """
289 1c75c4dd Christos Stavrakakis
            local_ip=$(/sbin/ifconfig eth0 | grep 'inet addr:' | \
290 1c75c4dd Christos Stavrakakis
                cut -d':' -f2 | cut -d' ' -f1)
291 1c75c4dd Christos Stavrakakis
            iptables -A INPUT -s localhost -j ACCEPT
292 1c75c4dd Christos Stavrakakis
            iptables -A INPUT -s $local_ip -j ACCEPT
293 1c75c4dd Christos Stavrakakis
            iptables -A INPUT -s {0} -p tcp --dport 22 -j ACCEPT
294 1c75c4dd Christos Stavrakakis
            iptables -A INPUT -p tcp --dport 22 -j DROP
295 1c75c4dd Christos Stavrakakis
            """.format(accept_ssh_from)
296 1c75c4dd Christos Stavrakakis
            _run(cmd, False)
297 c29ac11d Ilias Tsitsimpis
298 6c3cc77e Ilias Tsitsimpis
        # Setup apt, download packages
299 6c3cc77e Ilias Tsitsimpis
        self.logger.debug("Setup apt. Install x2goserver and firefox")
300 6c3cc77e Ilias Tsitsimpis
        cmd = """
301 6c3cc77e Ilias Tsitsimpis
        echo 'APT::Install-Suggests "false";' >> /etc/apt/apt.conf
302 6c3cc77e Ilias Tsitsimpis
        apt-get update
303 71f2649f Ilias Tsitsimpis
        apt-get install curl --yes --force-yes
304 6c3cc77e Ilias Tsitsimpis
        echo -e "\n\n{0}" >> /etc/apt/sources.list
305 6c3cc77e Ilias Tsitsimpis
        # Synnefo repo's key
306 6c3cc77e Ilias Tsitsimpis
        curl https://dev.grnet.gr/files/apt-grnetdev.pub | apt-key add -
307 8ddb541b Ilias Tsitsimpis

308 6c3cc77e Ilias Tsitsimpis
        # X2GO Key
309 6c3cc77e Ilias Tsitsimpis
        apt-key adv --recv-keys --keyserver keys.gnupg.net E1F958385BFE2B6E
310 71f2649f Ilias Tsitsimpis
        apt-get install x2go-keyring --yes --force-yes
311 6c3cc77e Ilias Tsitsimpis
        apt-get update
312 71f2649f Ilias Tsitsimpis
        apt-get install x2goserver x2goserver-xsession \
313 71f2649f Ilias Tsitsimpis
                iceweasel --yes --force-yes
314 8ddb541b Ilias Tsitsimpis

315 8ddb541b Ilias Tsitsimpis
        # xterm published application
316 8ddb541b Ilias Tsitsimpis
        echo '[Desktop Entry]' > /usr/share/applications/xterm.desktop
317 8ddb541b Ilias Tsitsimpis
        echo 'Name=XTerm' >> /usr/share/applications/xterm.desktop
318 8ddb541b Ilias Tsitsimpis
        echo 'Comment=standard terminal emulator for the X window system' >> \
319 8ddb541b Ilias Tsitsimpis
            /usr/share/applications/xterm.desktop
320 8ddb541b Ilias Tsitsimpis
        echo 'Exec=xterm' >> /usr/share/applications/xterm.desktop
321 8ddb541b Ilias Tsitsimpis
        echo 'Terminal=false' >> /usr/share/applications/xterm.desktop
322 8ddb541b Ilias Tsitsimpis
        echo 'Type=Application' >> /usr/share/applications/xterm.desktop
323 8ddb541b Ilias Tsitsimpis
        echo 'Encoding=UTF-8' >> /usr/share/applications/xterm.desktop
324 8ddb541b Ilias Tsitsimpis
        echo 'Icon=xterm-color_48x48' >> /usr/share/applications/xterm.desktop
325 8ddb541b Ilias Tsitsimpis
        echo 'Categories=System;TerminalEmulator;' >> \
326 8ddb541b Ilias Tsitsimpis
                /usr/share/applications/xterm.desktop
327 6c3cc77e Ilias Tsitsimpis
        """.format(self.config.get('Global', 'apt_repo'))
328 6c3cc77e Ilias Tsitsimpis
        _run(cmd, False)
329 6c3cc77e Ilias Tsitsimpis
330 358a19bc Ilias Tsitsimpis
    def _find_flavor(self, flavor=None):
331 358a19bc Ilias Tsitsimpis
        """Find a suitable flavor to use
332 358a19bc Ilias Tsitsimpis

333 358a19bc Ilias Tsitsimpis
        Search by name (reg expression) or by id
334 358a19bc Ilias Tsitsimpis
        """
335 358a19bc Ilias Tsitsimpis
        # Get a list of flavors from config file
336 358a19bc Ilias Tsitsimpis
        flavors = self.config.get('Deployment', 'flavors').split(",")
337 358a19bc Ilias Tsitsimpis
        if flavor is not None:
338 a8bfeffc Ilias Tsitsimpis
            # If we have a flavor_name to use, add it to our list
339 358a19bc Ilias Tsitsimpis
            flavors.insert(0, flavor)
340 358a19bc Ilias Tsitsimpis
341 358a19bc Ilias Tsitsimpis
        list_flavors = self.compute_client.list_flavors()
342 358a19bc Ilias Tsitsimpis
        for flv in flavors:
343 905bb10b Christos Stavrakakis
            flv_type, flv_value = parse_typed_option(option="flavor",
344 905bb10b Christos Stavrakakis
                                                     value=flv)
345 358a19bc Ilias Tsitsimpis
            if flv_type == "name":
346 358a19bc Ilias Tsitsimpis
                # Filter flavors by name
347 358a19bc Ilias Tsitsimpis
                self.logger.debug(
348 358a19bc Ilias Tsitsimpis
                    "Trying to find a flavor with name \"%s\"" % flv_value)
349 358a19bc Ilias Tsitsimpis
                list_flvs = \
350 358a19bc Ilias Tsitsimpis
                    [f for f in list_flavors
351 905bb10b Christos Stavrakakis
                     if re.search(flv_value, f['name'], flags=re.I)
352 905bb10b Christos Stavrakakis
                     is not None]
353 358a19bc Ilias Tsitsimpis
            elif flv_type == "id":
354 358a19bc Ilias Tsitsimpis
                # Filter flavors by id
355 358a19bc Ilias Tsitsimpis
                self.logger.debug(
356 358a19bc Ilias Tsitsimpis
                    "Trying to find a flavor with id \"%s\"" % flv_value)
357 358a19bc Ilias Tsitsimpis
                list_flvs = \
358 358a19bc Ilias Tsitsimpis
                    [f for f in list_flavors
359 694c79bb Ilias Tsitsimpis
                     if str(f['id']) == flv_value]
360 358a19bc Ilias Tsitsimpis
            else:
361 358a19bc Ilias Tsitsimpis
                self.logger.error("Unrecognized flavor type %s" % flv_type)
362 358a19bc Ilias Tsitsimpis
363 358a19bc Ilias Tsitsimpis
            # Check if we found one
364 358a19bc Ilias Tsitsimpis
            if list_flvs:
365 358a19bc Ilias Tsitsimpis
                self.logger.debug("Will use \"%s\" with id \"%s\""
366 106659f1 Ilias Tsitsimpis
                                  % (_green(list_flvs[0]['name']),
367 106659f1 Ilias Tsitsimpis
                                     _green(list_flvs[0]['id'])))
368 358a19bc Ilias Tsitsimpis
                return list_flvs[0]['id']
369 a8bfeffc Ilias Tsitsimpis
370 a8bfeffc Ilias Tsitsimpis
        self.logger.error("No matching flavor found.. aborting")
371 a8bfeffc Ilias Tsitsimpis
        sys.exit(1)
372 ad21ffa9 Ilias Tsitsimpis
373 358a19bc Ilias Tsitsimpis
    def _find_image(self, image=None):
374 c29ac11d Ilias Tsitsimpis
        """Find a suitable image to use
375 c29ac11d Ilias Tsitsimpis

376 358a19bc Ilias Tsitsimpis
        In case of search by name, the image has to belong to one
377 358a19bc Ilias Tsitsimpis
        of the `DEFAULT_SYSTEM_IMAGES_UUID' users.
378 358a19bc Ilias Tsitsimpis
        In case of search by id it only has to exist.
379 c29ac11d Ilias Tsitsimpis
        """
380 358a19bc Ilias Tsitsimpis
        # Get a list of images from config file
381 358a19bc Ilias Tsitsimpis
        images = self.config.get('Deployment', 'images').split(",")
382 358a19bc Ilias Tsitsimpis
        if image is not None:
383 358a19bc Ilias Tsitsimpis
            # If we have an image from command line, add it to our list
384 358a19bc Ilias Tsitsimpis
            images.insert(0, image)
385 358a19bc Ilias Tsitsimpis
386 905bb10b Christos Stavrakakis
        auth = self.astakos_client.authenticate()
387 905bb10b Christos Stavrakakis
        user_uuid = auth["access"]["token"]["tenant"]["id"]
388 358a19bc Ilias Tsitsimpis
        list_images = self.image_client.list_public(detail=True)['images']
389 358a19bc Ilias Tsitsimpis
        for img in images:
390 6958ffc6 Christos Stavrakakis
            img_type, img_value = parse_typed_option(option="image", value=img)
391 358a19bc Ilias Tsitsimpis
            if img_type == "name":
392 358a19bc Ilias Tsitsimpis
                # Filter images by name
393 358a19bc Ilias Tsitsimpis
                self.logger.debug(
394 358a19bc Ilias Tsitsimpis
                    "Trying to find an image with name \"%s\"" % img_value)
395 905bb10b Christos Stavrakakis
                accepted_uuids = DEFAULT_SYSTEM_IMAGES_UUID + [user_uuid]
396 358a19bc Ilias Tsitsimpis
                list_imgs = \
397 905bb10b Christos Stavrakakis
                    [i for i in list_images if i['user_id'] in accepted_uuids
398 905bb10b Christos Stavrakakis
                     and
399 905bb10b Christos Stavrakakis
                     re.search(img_value, i['name'], flags=re.I) is not None]
400 358a19bc Ilias Tsitsimpis
            elif img_type == "id":
401 358a19bc Ilias Tsitsimpis
                # Filter images by id
402 358a19bc Ilias Tsitsimpis
                self.logger.debug(
403 358a19bc Ilias Tsitsimpis
                    "Trying to find an image with id \"%s\"" % img_value)
404 358a19bc Ilias Tsitsimpis
                list_imgs = \
405 358a19bc Ilias Tsitsimpis
                    [i for i in list_images
406 358a19bc Ilias Tsitsimpis
                     if i['id'].lower() == img_value.lower()]
407 358a19bc Ilias Tsitsimpis
            else:
408 358a19bc Ilias Tsitsimpis
                self.logger.error("Unrecognized image type %s" % img_type)
409 358a19bc Ilias Tsitsimpis
                sys.exit(1)
410 358a19bc Ilias Tsitsimpis
411 358a19bc Ilias Tsitsimpis
            # Check if we found one
412 358a19bc Ilias Tsitsimpis
            if list_imgs:
413 358a19bc Ilias Tsitsimpis
                self.logger.debug("Will use \"%s\" with id \"%s\""
414 106659f1 Ilias Tsitsimpis
                                  % (_green(list_imgs[0]['name']),
415 106659f1 Ilias Tsitsimpis
                                     _green(list_imgs[0]['id'])))
416 358a19bc Ilias Tsitsimpis
                return list_imgs[0]['id']
417 358a19bc Ilias Tsitsimpis
418 358a19bc Ilias Tsitsimpis
        # We didn't found one
419 358a19bc Ilias Tsitsimpis
        self.logger.error("No matching image found.. aborting")
420 358a19bc Ilias Tsitsimpis
        sys.exit(1)
421 c29ac11d Ilias Tsitsimpis
422 c29ac11d Ilias Tsitsimpis
    def _get_server_ip_and_port(self, server):
423 c29ac11d Ilias Tsitsimpis
        """Compute server's IPv4 and ssh port number"""
424 c29ac11d Ilias Tsitsimpis
        self.logger.info("Get server connection details..")
425 c29ac11d Ilias Tsitsimpis
        server_ip = server['attachments'][0]['ipv4']
426 705f70a9 Christos Stavrakakis
        if (".okeanos.io" in self.cyclades_client.base_url or
427 705f70a9 Christos Stavrakakis
           ".demo.synnefo.org" in self.cyclades_client.base_url):
428 c29ac11d Ilias Tsitsimpis
            tmp1 = int(server_ip.split(".")[2])
429 c29ac11d Ilias Tsitsimpis
            tmp2 = int(server_ip.split(".")[3])
430 c29ac11d Ilias Tsitsimpis
            server_ip = "gate.okeanos.io"
431 c29ac11d Ilias Tsitsimpis
            server_port = 10000 + tmp1 * 256 + tmp2
432 c29ac11d Ilias Tsitsimpis
        else:
433 c29ac11d Ilias Tsitsimpis
            server_port = 22
434 b7496c88 Ilias Tsitsimpis
        self.write_temp_config('server_ip', server_ip)
435 c29ac11d Ilias Tsitsimpis
        self.logger.debug("Server's IPv4 is %s" % _green(server_ip))
436 b7496c88 Ilias Tsitsimpis
        self.write_temp_config('server_port', server_port)
437 c29ac11d Ilias Tsitsimpis
        self.logger.debug("Server's ssh port is %s" % _green(server_port))
438 106659f1 Ilias Tsitsimpis
        ssh_command = "ssh -p %s %s@%s" \
439 106659f1 Ilias Tsitsimpis
            % (server_port, server['metadata']['users'], server_ip)
440 106659f1 Ilias Tsitsimpis
        self.logger.debug("Access server using \"%s\"" %
441 106659f1 Ilias Tsitsimpis
                          (_green(ssh_command)))
442 c29ac11d Ilias Tsitsimpis
443 e480f0ce Christos Stavrakakis
    @_check_fabric
444 6868804a Ilias Tsitsimpis
    def _copy_ssh_keys(self, ssh_keys):
445 e2a0abb8 Ilias Tsitsimpis
        """Upload/Install ssh keys to server"""
446 4b743650 Ilias Tsitsimpis
        self.logger.debug("Check for authentication keys to use")
447 6868804a Ilias Tsitsimpis
        if ssh_keys is None:
448 6868804a Ilias Tsitsimpis
            ssh_keys = self.config.get("Deployment", "ssh_keys")
449 6868804a Ilias Tsitsimpis
450 4b743650 Ilias Tsitsimpis
        if ssh_keys != "":
451 6058ae72 Ilias Tsitsimpis
            ssh_keys = os.path.expanduser(ssh_keys)
452 106659f1 Ilias Tsitsimpis
            self.logger.debug("Will use \"%s\" authentication keys file" %
453 106659f1 Ilias Tsitsimpis
                              _green(ssh_keys))
454 e480f0ce Christos Stavrakakis
            keyfile = '/tmp/%s.pub' % fabric.env.user
455 e480f0ce Christos Stavrakakis
            _run('mkdir -p ~/.ssh && chmod 700 ~/.ssh', False)
456 4b743650 Ilias Tsitsimpis
            if ssh_keys.startswith("http://") or \
457 4b743650 Ilias Tsitsimpis
                    ssh_keys.startswith("https://") or \
458 4b743650 Ilias Tsitsimpis
                    ssh_keys.startswith("ftp://"):
459 4b743650 Ilias Tsitsimpis
                cmd = """
460 4b743650 Ilias Tsitsimpis
                apt-get update
461 71f2649f Ilias Tsitsimpis
                apt-get install wget --yes --force-yes
462 4b743650 Ilias Tsitsimpis
                wget {0} -O {1} --no-check-certificate
463 4b743650 Ilias Tsitsimpis
                """.format(ssh_keys, keyfile)
464 4b743650 Ilias Tsitsimpis
                _run(cmd, False)
465 4b743650 Ilias Tsitsimpis
            elif os.path.exists(ssh_keys):
466 4b743650 Ilias Tsitsimpis
                _put(ssh_keys, keyfile)
467 4b743650 Ilias Tsitsimpis
            else:
468 4b743650 Ilias Tsitsimpis
                self.logger.debug("No ssh keys found")
469 6058ae72 Ilias Tsitsimpis
                return
470 e480f0ce Christos Stavrakakis
            _run('cat %s >> ~/.ssh/authorized_keys' % keyfile, False)
471 e480f0ce Christos Stavrakakis
            _run('rm %s' % keyfile, False)
472 e480f0ce Christos Stavrakakis
            self.logger.debug("Uploaded ssh authorized keys")
473 e480f0ce Christos Stavrakakis
        else:
474 e480f0ce Christos Stavrakakis
            self.logger.debug("No ssh keys found")
475 e480f0ce Christos Stavrakakis
476 2cbdb63f Ilias Tsitsimpis
    def _create_new_build_id(self):
477 2cbdb63f Ilias Tsitsimpis
        """Find a uniq build_id to use"""
478 2cbdb63f Ilias Tsitsimpis
        with filelocker.lock("%s.lock" % self.temp_config_file,
479 2cbdb63f Ilias Tsitsimpis
                             filelocker.LOCK_EX):
480 2cbdb63f Ilias Tsitsimpis
            # Read temp_config again to get any new entries
481 2cbdb63f Ilias Tsitsimpis
            self.temp_config.read(self.temp_config_file)
482 2cbdb63f Ilias Tsitsimpis
483 2cbdb63f Ilias Tsitsimpis
            # Find a uniq build_id to use
484 c7828946 Ilias Tsitsimpis
            if self.build_id is None:
485 c7828946 Ilias Tsitsimpis
                ids = self.temp_config.sections()
486 c7828946 Ilias Tsitsimpis
                if ids:
487 c7828946 Ilias Tsitsimpis
                    max_id = int(max(self.temp_config.sections(), key=int))
488 c7828946 Ilias Tsitsimpis
                    self.build_id = max_id + 1
489 c7828946 Ilias Tsitsimpis
                else:
490 c7828946 Ilias Tsitsimpis
                    self.build_id = 1
491 c7828946 Ilias Tsitsimpis
            self.logger.debug("Will use \"%s\" as build id"
492 2cbdb63f Ilias Tsitsimpis
                              % _green(self.build_id))
493 2cbdb63f Ilias Tsitsimpis
494 2cbdb63f Ilias Tsitsimpis
            # Create a new section
495 329705c8 Ilias Tsitsimpis
            try:
496 329705c8 Ilias Tsitsimpis
                self.temp_config.add_section(str(self.build_id))
497 329705c8 Ilias Tsitsimpis
            except DuplicateSectionError:
498 329705c8 Ilias Tsitsimpis
                msg = ("Build id \"%s\" already in use. " +
499 329705c8 Ilias Tsitsimpis
                       "Please use a uniq one or cleanup \"%s\" file.\n") \
500 329705c8 Ilias Tsitsimpis
                    % (self.build_id, self.temp_config_file)
501 329705c8 Ilias Tsitsimpis
                self.logger.error(msg)
502 329705c8 Ilias Tsitsimpis
                sys.exit(1)
503 2cbdb63f Ilias Tsitsimpis
            creation_time = \
504 2cbdb63f Ilias Tsitsimpis
                time.strftime("%a, %d %b %Y %X", time.localtime())
505 2cbdb63f Ilias Tsitsimpis
            self.temp_config.set(str(self.build_id),
506 2cbdb63f Ilias Tsitsimpis
                                 "created", str(creation_time))
507 2cbdb63f Ilias Tsitsimpis
508 2cbdb63f Ilias Tsitsimpis
            # Write changes back to temp config file
509 2cbdb63f Ilias Tsitsimpis
            with open(self.temp_config_file, 'wb') as tcf:
510 2cbdb63f Ilias Tsitsimpis
                self.temp_config.write(tcf)
511 2cbdb63f Ilias Tsitsimpis
512 b7496c88 Ilias Tsitsimpis
    def write_temp_config(self, option, value):
513 c29ac11d Ilias Tsitsimpis
        """Write changes back to config file"""
514 62fcf0e5 Ilias Tsitsimpis
        # Acquire the lock to write to temp_config_file
515 62fcf0e5 Ilias Tsitsimpis
        with filelocker.lock("%s.lock" % self.temp_config_file,
516 62fcf0e5 Ilias Tsitsimpis
                             filelocker.LOCK_EX):
517 62fcf0e5 Ilias Tsitsimpis
518 62fcf0e5 Ilias Tsitsimpis
            # Read temp_config again to get any new entries
519 62fcf0e5 Ilias Tsitsimpis
            self.temp_config.read(self.temp_config_file)
520 62fcf0e5 Ilias Tsitsimpis
521 62fcf0e5 Ilias Tsitsimpis
            self.temp_config.set(str(self.build_id), option, str(value))
522 62fcf0e5 Ilias Tsitsimpis
            curr_time = time.strftime("%a, %d %b %Y %X", time.localtime())
523 62fcf0e5 Ilias Tsitsimpis
            self.temp_config.set(str(self.build_id), "modified", curr_time)
524 2cbdb63f Ilias Tsitsimpis
525 2cbdb63f Ilias Tsitsimpis
            # Write changes back to temp config file
526 62fcf0e5 Ilias Tsitsimpis
            with open(self.temp_config_file, 'wb') as tcf:
527 62fcf0e5 Ilias Tsitsimpis
                self.temp_config.write(tcf)
528 c29ac11d Ilias Tsitsimpis
529 b7496c88 Ilias Tsitsimpis
    def read_temp_config(self, option):
530 b7496c88 Ilias Tsitsimpis
        """Read from temporary_config file"""
531 b7496c88 Ilias Tsitsimpis
        # If build_id is None use the latest one
532 b7496c88 Ilias Tsitsimpis
        if self.build_id is None:
533 b7496c88 Ilias Tsitsimpis
            ids = self.temp_config.sections()
534 b7496c88 Ilias Tsitsimpis
            if ids:
535 b7496c88 Ilias Tsitsimpis
                self.build_id = int(ids[-1])
536 b7496c88 Ilias Tsitsimpis
            else:
537 b7496c88 Ilias Tsitsimpis
                self.logger.error("No sections in temporary config file")
538 b7496c88 Ilias Tsitsimpis
                sys.exit(1)
539 b7496c88 Ilias Tsitsimpis
            self.logger.debug("Will use \"%s\" as build id"
540 b7496c88 Ilias Tsitsimpis
                              % _green(self.build_id))
541 b7496c88 Ilias Tsitsimpis
        # Read specified option
542 b7496c88 Ilias Tsitsimpis
        return self.temp_config.get(str(self.build_id), option)
543 b7496c88 Ilias Tsitsimpis
544 c29ac11d Ilias Tsitsimpis
    def setup_fabric(self):
545 c29ac11d Ilias Tsitsimpis
        """Setup fabric environment"""
546 c29ac11d Ilias Tsitsimpis
        self.logger.info("Setup fabric parameters..")
547 b7496c88 Ilias Tsitsimpis
        fabric.env.user = self.read_temp_config('server_user')
548 b7496c88 Ilias Tsitsimpis
        fabric.env.host_string = self.read_temp_config('server_ip')
549 b7496c88 Ilias Tsitsimpis
        fabric.env.port = int(self.read_temp_config('server_port'))
550 b7496c88 Ilias Tsitsimpis
        fabric.env.password = self.read_temp_config('server_passwd')
551 c29ac11d Ilias Tsitsimpis
        fabric.env.connection_attempts = 10
552 c29ac11d Ilias Tsitsimpis
        fabric.env.shell = "/bin/bash -c"
553 c29ac11d Ilias Tsitsimpis
        fabric.env.disable_known_hosts = True
554 c29ac11d Ilias Tsitsimpis
        fabric.env.output_prefix = None
555 c29ac11d Ilias Tsitsimpis
556 c29ac11d Ilias Tsitsimpis
    def _check_hash_sum(self, localfile, remotefile):
557 c29ac11d Ilias Tsitsimpis
        """Check hash sums of two files"""
558 c29ac11d Ilias Tsitsimpis
        self.logger.debug("Check hash sum for local file %s" % localfile)
559 c29ac11d Ilias Tsitsimpis
        hash1 = os.popen("sha256sum %s" % localfile).read().split(' ')[0]
560 c29ac11d Ilias Tsitsimpis
        self.logger.debug("Local file has sha256 hash %s" % hash1)
561 c29ac11d Ilias Tsitsimpis
        self.logger.debug("Check hash sum for remote file %s" % remotefile)
562 c29ac11d Ilias Tsitsimpis
        hash2 = _run("sha256sum %s" % remotefile, False)
563 c29ac11d Ilias Tsitsimpis
        hash2 = hash2.split(' ')[0]
564 c29ac11d Ilias Tsitsimpis
        self.logger.debug("Remote file has sha256 hash %s" % hash2)
565 c29ac11d Ilias Tsitsimpis
        if hash1 != hash2:
566 c29ac11d Ilias Tsitsimpis
            self.logger.error("Hashes differ.. aborting")
567 99d39120 Ilias Tsitsimpis
            sys.exit(1)
568 c29ac11d Ilias Tsitsimpis
569 c29ac11d Ilias Tsitsimpis
    @_check_fabric
570 5323404f Ilias Tsitsimpis
    def clone_repo(self, local_repo=False):
571 c29ac11d Ilias Tsitsimpis
        """Clone Synnefo repo from slave server"""
572 c29ac11d Ilias Tsitsimpis
        self.logger.info("Configure repositories on remote server..")
573 6c3cc77e Ilias Tsitsimpis
        self.logger.debug("Install/Setup git")
574 c29ac11d Ilias Tsitsimpis
        cmd = """
575 71f2649f Ilias Tsitsimpis
        apt-get install git --yes --force-yes
576 6c3cc77e Ilias Tsitsimpis
        git config --global user.name {0}
577 6c3cc77e Ilias Tsitsimpis
        git config --global user.email {1}
578 6c3cc77e Ilias Tsitsimpis
        """.format(self.config.get('Global', 'git_config_name'),
579 c29ac11d Ilias Tsitsimpis
                   self.config.get('Global', 'git_config_mail'))
580 c29ac11d Ilias Tsitsimpis
        _run(cmd, False)
581 c29ac11d Ilias Tsitsimpis
582 99d39120 Ilias Tsitsimpis
        # Clone synnefo_repo
583 99d39120 Ilias Tsitsimpis
        synnefo_branch = self.clone_synnefo_repo(local_repo=local_repo)
584 99d39120 Ilias Tsitsimpis
        # Clone pithos-web-client
585 99d39120 Ilias Tsitsimpis
        self.clone_pithos_webclient_repo(synnefo_branch)
586 99d39120 Ilias Tsitsimpis
587 99d39120 Ilias Tsitsimpis
    @_check_fabric
588 99d39120 Ilias Tsitsimpis
    def clone_synnefo_repo(self, local_repo=False):
589 99d39120 Ilias Tsitsimpis
        """Clone Synnefo repo to remote server"""
590 5323404f Ilias Tsitsimpis
        # Find synnefo_repo and synnefo_branch to use
591 c29ac11d Ilias Tsitsimpis
        synnefo_repo = self.config.get('Global', 'synnefo_repo')
592 b69b55ca Christos Stavrakakis
        synnefo_branch = self.config.get("Global", "synnefo_branch")
593 b69b55ca Christos Stavrakakis
        if synnefo_branch == "":
594 e2a0abb8 Ilias Tsitsimpis
            synnefo_branch = \
595 e2a0abb8 Ilias Tsitsimpis
                subprocess.Popen(
596 e2a0abb8 Ilias Tsitsimpis
                    ["git", "rev-parse", "--abbrev-ref", "HEAD"],
597 b69b55ca Christos Stavrakakis
                    stdout=subprocess.PIPE).communicate()[0].strip()
598 b69b55ca Christos Stavrakakis
            if synnefo_branch == "HEAD":
599 b69b55ca Christos Stavrakakis
                synnefo_branch = \
600 e2a0abb8 Ilias Tsitsimpis
                    subprocess.Popen(
601 e2a0abb8 Ilias Tsitsimpis
                        ["git", "rev-parse", "--short", "HEAD"],
602 b69b55ca Christos Stavrakakis
                        stdout=subprocess.PIPE).communicate()[0].strip()
603 99d39120 Ilias Tsitsimpis
        self.logger.debug("Will use branch \"%s\"" % _green(synnefo_branch))
604 5323404f Ilias Tsitsimpis
605 99d39120 Ilias Tsitsimpis
        if local_repo or synnefo_repo == "":
606 5323404f Ilias Tsitsimpis
            # Use local_repo
607 5323404f Ilias Tsitsimpis
            self.logger.debug("Push local repo to server")
608 5323404f Ilias Tsitsimpis
            # Firstly create the remote repo
609 5323404f Ilias Tsitsimpis
            _run("git init synnefo", False)
610 5323404f Ilias Tsitsimpis
            # Then push our local repo over ssh
611 5323404f Ilias Tsitsimpis
            # We have to pass some arguments to ssh command
612 5323404f Ilias Tsitsimpis
            # namely to disable host checking.
613 5323404f Ilias Tsitsimpis
            (temp_ssh_file_handle, temp_ssh_file) = tempfile.mkstemp()
614 5323404f Ilias Tsitsimpis
            os.close(temp_ssh_file_handle)
615 62fcf0e5 Ilias Tsitsimpis
            # XXX: git push doesn't read the password
616 5323404f Ilias Tsitsimpis
            cmd = """
617 5323404f Ilias Tsitsimpis
            echo 'exec ssh -o "StrictHostKeyChecking no" \
618 5323404f Ilias Tsitsimpis
                           -o "UserKnownHostsFile /dev/null" \
619 5323404f Ilias Tsitsimpis
                           -q "$@"' > {4}
620 5323404f Ilias Tsitsimpis
            chmod u+x {4}
621 5323404f Ilias Tsitsimpis
            export GIT_SSH="{4}"
622 99d39120 Ilias Tsitsimpis
            echo "{0}" | git push --quiet --mirror ssh://{1}@{2}:{3}/~/synnefo
623 5323404f Ilias Tsitsimpis
            rm -f {4}
624 5323404f Ilias Tsitsimpis
            """.format(fabric.env.password,
625 5323404f Ilias Tsitsimpis
                       fabric.env.user,
626 5323404f Ilias Tsitsimpis
                       fabric.env.host_string,
627 5323404f Ilias Tsitsimpis
                       fabric.env.port,
628 5323404f Ilias Tsitsimpis
                       temp_ssh_file)
629 5323404f Ilias Tsitsimpis
            os.system(cmd)
630 5323404f Ilias Tsitsimpis
        else:
631 5323404f Ilias Tsitsimpis
            # Clone Synnefo from remote repo
632 99d39120 Ilias Tsitsimpis
            self.logger.debug("Clone synnefo from %s" % synnefo_repo)
633 99d39120 Ilias Tsitsimpis
            self._git_clone(synnefo_repo)
634 5323404f Ilias Tsitsimpis
635 5323404f Ilias Tsitsimpis
        # Checkout the desired synnefo_branch
636 07710fc3 Ilias Tsitsimpis
        self.logger.debug("Checkout \"%s\" branch/commit" % synnefo_branch)
637 e2a0abb8 Ilias Tsitsimpis
        cmd = """
638 b69b55ca Christos Stavrakakis
        cd synnefo
639 99d39120 Ilias Tsitsimpis
        for branch in `git branch -a | grep remotes | grep -v HEAD`; do
640 b69b55ca Christos Stavrakakis
            git branch --track ${branch##*/} $branch
641 b69b55ca Christos Stavrakakis
        done
642 b69b55ca Christos Stavrakakis
        git checkout %s
643 b69b55ca Christos Stavrakakis
        """ % (synnefo_branch)
644 b69b55ca Christos Stavrakakis
        _run(cmd, False)
645 b69b55ca Christos Stavrakakis
646 99d39120 Ilias Tsitsimpis
        return synnefo_branch
647 99d39120 Ilias Tsitsimpis
648 c29ac11d Ilias Tsitsimpis
    @_check_fabric
649 99d39120 Ilias Tsitsimpis
    def clone_pithos_webclient_repo(self, synnefo_branch):
650 99d39120 Ilias Tsitsimpis
        """Clone Pithos WebClient repo to remote server"""
651 99d39120 Ilias Tsitsimpis
        # Find pithos_webclient_repo and pithos_webclient_branch to use
652 99d39120 Ilias Tsitsimpis
        pithos_webclient_repo = \
653 99d39120 Ilias Tsitsimpis
            self.config.get('Global', 'pithos_webclient_repo')
654 99d39120 Ilias Tsitsimpis
        pithos_webclient_branch = \
655 99d39120 Ilias Tsitsimpis
            self.config.get('Global', 'pithos_webclient_branch')
656 99d39120 Ilias Tsitsimpis
657 99d39120 Ilias Tsitsimpis
        # Clone pithos-webclient from remote repo
658 99d39120 Ilias Tsitsimpis
        self.logger.debug("Clone pithos-webclient from %s" %
659 99d39120 Ilias Tsitsimpis
                          pithos_webclient_repo)
660 99d39120 Ilias Tsitsimpis
        self._git_clone(pithos_webclient_repo)
661 99d39120 Ilias Tsitsimpis
662 99d39120 Ilias Tsitsimpis
        # Track all pithos-webclient branches
663 99d39120 Ilias Tsitsimpis
        cmd = """
664 99d39120 Ilias Tsitsimpis
        cd pithos-web-client
665 99d39120 Ilias Tsitsimpis
        for branch in `git branch -a | grep remotes | grep -v HEAD`; do
666 99d39120 Ilias Tsitsimpis
            git branch --track ${branch##*/} $branch > /dev/null 2>&1
667 99d39120 Ilias Tsitsimpis
        done
668 99d39120 Ilias Tsitsimpis
        git branch
669 99d39120 Ilias Tsitsimpis
        """
670 99d39120 Ilias Tsitsimpis
        webclient_branches = _run(cmd, False)
671 99d39120 Ilias Tsitsimpis
        webclient_branches = webclient_branches.split()
672 99d39120 Ilias Tsitsimpis
673 99d39120 Ilias Tsitsimpis
        # If we have pithos_webclient_branch in config file use this one
674 99d39120 Ilias Tsitsimpis
        # else try to use the same branch as synnefo_branch
675 99d39120 Ilias Tsitsimpis
        # else use an appropriate one.
676 99d39120 Ilias Tsitsimpis
        if pithos_webclient_branch == "":
677 99d39120 Ilias Tsitsimpis
            if synnefo_branch in webclient_branches:
678 99d39120 Ilias Tsitsimpis
                pithos_webclient_branch = synnefo_branch
679 99d39120 Ilias Tsitsimpis
            else:
680 99d39120 Ilias Tsitsimpis
                # If synnefo_branch starts with one of
681 99d39120 Ilias Tsitsimpis
                # 'master', 'hotfix'; use the master branch
682 99d39120 Ilias Tsitsimpis
                if synnefo_branch.startswith('master') or \
683 99d39120 Ilias Tsitsimpis
                        synnefo_branch.startswith('hotfix'):
684 99d39120 Ilias Tsitsimpis
                    pithos_webclient_branch = "master"
685 99d39120 Ilias Tsitsimpis
                # If synnefo_branch starts with one of
686 99d39120 Ilias Tsitsimpis
                # 'develop', 'feature'; use the develop branch
687 99d39120 Ilias Tsitsimpis
                elif synnefo_branch.startswith('develop') or \
688 99d39120 Ilias Tsitsimpis
                        synnefo_branch.startswith('feature'):
689 99d39120 Ilias Tsitsimpis
                    pithos_webclient_branch = "develop"
690 99d39120 Ilias Tsitsimpis
                else:
691 99d39120 Ilias Tsitsimpis
                    self.logger.waring(
692 99d39120 Ilias Tsitsimpis
                        "Cannot determine which pithos-web-client branch to "
693 99d39120 Ilias Tsitsimpis
                        "use based on \"%s\" synnefo branch. "
694 99d39120 Ilias Tsitsimpis
                        "Will use develop." % synnefo_branch)
695 99d39120 Ilias Tsitsimpis
                    pithos_webclient_branch = "develop"
696 99d39120 Ilias Tsitsimpis
        # Checkout branch
697 99d39120 Ilias Tsitsimpis
        self.logger.debug("Checkout \"%s\" branch" %
698 99d39120 Ilias Tsitsimpis
                          _green(pithos_webclient_branch))
699 99d39120 Ilias Tsitsimpis
        cmd = """
700 99d39120 Ilias Tsitsimpis
        cd pithos-web-client
701 99d39120 Ilias Tsitsimpis
        git checkout {0}
702 99d39120 Ilias Tsitsimpis
        """.format(pithos_webclient_branch)
703 99d39120 Ilias Tsitsimpis
        _run(cmd, False)
704 99d39120 Ilias Tsitsimpis
705 99d39120 Ilias Tsitsimpis
    def _git_clone(self, repo):
706 99d39120 Ilias Tsitsimpis
        """Clone repo to remote server
707 99d39120 Ilias Tsitsimpis

708 99d39120 Ilias Tsitsimpis
        Currently clonning from code.grnet.gr can fail unexpectedly.
709 99d39120 Ilias Tsitsimpis
        So retry!!
710 99d39120 Ilias Tsitsimpis

711 99d39120 Ilias Tsitsimpis
        """
712 99d39120 Ilias Tsitsimpis
        cloned = False
713 99d39120 Ilias Tsitsimpis
        for i in range(1, 11):
714 99d39120 Ilias Tsitsimpis
            try:
715 99d39120 Ilias Tsitsimpis
                _run("git clone %s" % repo, False)
716 99d39120 Ilias Tsitsimpis
                cloned = True
717 99d39120 Ilias Tsitsimpis
                break
718 99d39120 Ilias Tsitsimpis
            except BaseException:
719 99d39120 Ilias Tsitsimpis
                self.logger.warning("Clonning failed.. retrying %s/10" % i)
720 99d39120 Ilias Tsitsimpis
        if not cloned:
721 99d39120 Ilias Tsitsimpis
            self.logger.error("Can not clone repo.")
722 99d39120 Ilias Tsitsimpis
            sys.exit(1)
723 99d39120 Ilias Tsitsimpis
724 99d39120 Ilias Tsitsimpis
    @_check_fabric
725 99d39120 Ilias Tsitsimpis
    def build_packages(self):
726 99d39120 Ilias Tsitsimpis
        """Build packages needed by Synnefo software"""
727 99d39120 Ilias Tsitsimpis
        self.logger.info("Install development packages")
728 c29ac11d Ilias Tsitsimpis
        cmd = """
729 c29ac11d Ilias Tsitsimpis
        apt-get update
730 c29ac11d Ilias Tsitsimpis
        apt-get install zlib1g-dev dpkg-dev debhelper git-buildpackage \
731 99d39120 Ilias Tsitsimpis
                python-dev python-all python-pip ant --yes --force-yes
732 8d2c08e4 Christos Stavrakakis
        pip install -U devflow
733 c29ac11d Ilias Tsitsimpis
        """
734 c29ac11d Ilias Tsitsimpis
        _run(cmd, False)
735 c29ac11d Ilias Tsitsimpis
736 99d39120 Ilias Tsitsimpis
        # Patch pydist bug
737 e908a902 Christos Stavrakakis
        if self.config.get('Global', 'patch_pydist') == "True":
738 c29ac11d Ilias Tsitsimpis
            self.logger.debug("Patch pydist.py module")
739 c29ac11d Ilias Tsitsimpis
            cmd = r"""
740 c29ac11d Ilias Tsitsimpis
            sed -r -i 's/(\(\?P<name>\[A-Za-z\]\[A-Za-z0-9_\.)/\1\\\-/' \
741 c29ac11d Ilias Tsitsimpis
                /usr/share/python/debpython/pydist.py
742 c29ac11d Ilias Tsitsimpis
            """
743 c29ac11d Ilias Tsitsimpis
            _run(cmd, False)
744 c29ac11d Ilias Tsitsimpis

745 1c230fd8 Ilias Tsitsimpis
        # Build synnefo packages
746 99d39120 Ilias Tsitsimpis
        self.build_synnefo()
747 99d39120 Ilias Tsitsimpis
        # Build pithos-web-client packages
748 99d39120 Ilias Tsitsimpis
        self.build_pithos_webclient()
749 99d39120 Ilias Tsitsimpis

750 99d39120 Ilias Tsitsimpis
    @_check_fabric
751 99d39120 Ilias Tsitsimpis
    def build_synnefo(self):
752 99d39120 Ilias Tsitsimpis
        """Build Synnefo packages"""
753 99d39120 Ilias Tsitsimpis
        self.logger.info("Build Synnefo packages..")
754 99d39120 Ilias Tsitsimpis

755 c29ac11d Ilias Tsitsimpis
        cmd = """
756 1c230fd8 Ilias Tsitsimpis
        devflow-autopkg snapshot -b ~/synnefo_build-area --no-sign
757 c29ac11d Ilias Tsitsimpis
        """
758 1c230fd8 Ilias Tsitsimpis
        with fabric.cd("synnefo"):
759 c29ac11d Ilias Tsitsimpis
            _run(cmd, True)
760 c29ac11d Ilias Tsitsimpis

761 1c230fd8 Ilias Tsitsimpis
        # Install snf-deploy package
762 c29ac11d Ilias Tsitsimpis
        self.logger.debug("Install snf-deploy package")
763 c29ac11d Ilias Tsitsimpis
        cmd = """
764 c29ac11d Ilias Tsitsimpis
        dpkg -i snf-deploy*.deb
765 71f2649f Ilias Tsitsimpis
        apt-get -f install --yes --force-yes
766 c29ac11d Ilias Tsitsimpis
        """
767 1c230fd8 Ilias Tsitsimpis
        with fabric.cd("synnefo_build-area"):
768 c29ac11d Ilias Tsitsimpis
            with fabric.settings(warn_only=True):
769 c29ac11d Ilias Tsitsimpis
                _run(cmd, True)
770 c29ac11d Ilias Tsitsimpis

771 1c230fd8 Ilias Tsitsimpis
        # Setup synnefo packages for snf-deploy
772 c29ac11d Ilias Tsitsimpis
        self.logger.debug("Copy synnefo debs to snf-deploy packages dir")
773 c29ac11d Ilias Tsitsimpis
        cmd = """
774 c29ac11d Ilias Tsitsimpis
        cp ~/synnefo_build-area/*.deb /var/lib/snf-deploy/packages/
775 c29ac11d Ilias Tsitsimpis
        """
776 c29ac11d Ilias Tsitsimpis
        _run(cmd, False)
777 c29ac11d Ilias Tsitsimpis

778 88e6558b Christos Stavrakakis
    @_check_fabric
779 99d39120 Ilias Tsitsimpis
    def build_pithos_webclient(self):
780 99d39120 Ilias Tsitsimpis
        """Build pithos-web-client packages"""
781 99d39120 Ilias Tsitsimpis
        self.logger.info("Build pithos-web-client packages..")
782 99d39120 Ilias Tsitsimpis

783 99d39120 Ilias Tsitsimpis
        cmd = """
784 99d39120 Ilias Tsitsimpis
        devflow-autopkg snapshot -b ~/webclient_build-area --no-sign
785 99d39120 Ilias Tsitsimpis
        """
786 99d39120 Ilias Tsitsimpis
        with fabric.cd("pithos-web-client"):
787 99d39120 Ilias Tsitsimpis
            _run(cmd, True)
788 99d39120 Ilias Tsitsimpis

789 99d39120 Ilias Tsitsimpis
        # Setup pithos-web-client packages for snf-deploy
790 99d39120 Ilias Tsitsimpis
        self.logger.debug("Copy webclient debs to snf-deploy packages dir")
791 99d39120 Ilias Tsitsimpis
        cmd = """
792 99d39120 Ilias Tsitsimpis
        cp ~/webclient_build-area/*.deb /var/lib/snf-deploy/packages/
793 99d39120 Ilias Tsitsimpis
        """
794 99d39120 Ilias Tsitsimpis
        _run(cmd, False)
795 99d39120 Ilias Tsitsimpis

796 99d39120 Ilias Tsitsimpis
    @_check_fabric
797 88e6558b Christos Stavrakakis
    def build_documentation(self):
798 e2a0abb8 Ilias Tsitsimpis
        """Build Synnefo documentation"""
799 88e6558b Christos Stavrakakis
        self.logger.info("Build Synnefo documentation..")
800 88e6558b Christos Stavrakakis
        _run("pip install -U Sphinx", False)
801 88e6558b Christos Stavrakakis
        with fabric.cd("synnefo"):
802 e2a0abb8 Ilias Tsitsimpis
            _run("devflow-update-version; "
803 e2a0abb8 Ilias Tsitsimpis
                 "./ci/make_docs.sh synnefo_documentation", False)
804 88e6558b Christos Stavrakakis

805 88e6558b Christos Stavrakakis
    def fetch_documentation(self, dest=None):
806 e2a0abb8 Ilias Tsitsimpis
        """Fetch Synnefo documentation"""
807 e2a0abb8 Ilias Tsitsimpis
        self.logger.info("Fetch Synnefo documentation..")
808 88e6558b Christos Stavrakakis
        if dest is None:
809 88e6558b Christos Stavrakakis
            dest = "synnefo_documentation"
810 88e6558b Christos Stavrakakis
        dest = os.path.abspath(dest)
811 88e6558b Christos Stavrakakis
        if not os.path.exists(dest):
812 88e6558b Christos Stavrakakis
            os.makedirs(dest)
813 88e6558b Christos Stavrakakis
        self.fetch_compressed("synnefo/synnefo_documentation", dest)
814 88e6558b Christos Stavrakakis
        self.logger.info("Downloaded documentation to %s" %
815 88e6558b Christos Stavrakakis
                         _green(dest))
816 88e6558b Christos Stavrakakis

817 c29ac11d Ilias Tsitsimpis
    @_check_fabric
818 ec8bc030 Christos Stavrakakis
    def deploy_synnefo(self, schema=None):
819 c29ac11d Ilias Tsitsimpis
        """Deploy Synnefo using snf-deploy"""
820 c29ac11d Ilias Tsitsimpis
        self.logger.info("Deploy Synnefo..")
821 ec8bc030 Christos Stavrakakis
        if schema is None:
822 ec8bc030 Christos Stavrakakis
            schema = self.config.get('Global', 'schema')
823 106659f1 Ilias Tsitsimpis
        self.logger.debug("Will use \"%s\" schema" % _green(schema))
824 c29ac11d Ilias Tsitsimpis

825 ec8bc030 Christos Stavrakakis
        schema_dir = os.path.join(self.ci_dir, "schemas/%s" % schema)
826 ec8bc030 Christos Stavrakakis
        if not (os.path.exists(schema_dir) and os.path.isdir(schema_dir)):
827 ec8bc030 Christos Stavrakakis
            raise ValueError("Unknown schema: %s" % schema)
828 ec8bc030 Christos Stavrakakis

829 c29ac11d Ilias Tsitsimpis
        self.logger.debug("Upload schema files to server")
830 6868804a Ilias Tsitsimpis
        _put(os.path.join(schema_dir, "*"), "/etc/snf-deploy/")
831 c29ac11d Ilias Tsitsimpis

832 c29ac11d Ilias Tsitsimpis
        self.logger.debug("Change password in nodes.conf file")
833 c29ac11d Ilias Tsitsimpis
        cmd = """
834 c29ac11d Ilias Tsitsimpis
        sed -i 's/^password =.*/password = {0}/' /etc/snf-deploy/nodes.conf
835 c29ac11d Ilias Tsitsimpis
        """.format(fabric.env.password)
836 c29ac11d Ilias Tsitsimpis
        _run(cmd, False)
837 c29ac11d Ilias Tsitsimpis

838 c29ac11d Ilias Tsitsimpis
        self.logger.debug("Run snf-deploy")
839 c29ac11d Ilias Tsitsimpis
        cmd = """
840 985442f5 Ilias Tsitsimpis
        snf-deploy keygen --force
841 f82ce32a Christos Stavrakakis
        snf-deploy --disable-colors --autoconf all
842 c29ac11d Ilias Tsitsimpis
        """
843 c29ac11d Ilias Tsitsimpis
        _run(cmd, True)
844 c29ac11d Ilias Tsitsimpis

845 c29ac11d Ilias Tsitsimpis
    @_check_fabric
846 c29ac11d Ilias Tsitsimpis
    def unit_test(self):
847 c29ac11d Ilias Tsitsimpis
        """Run Synnefo unit test suite"""
848 c29ac11d Ilias Tsitsimpis
        self.logger.info("Run Synnefo unit test suite")
849 c29ac11d Ilias Tsitsimpis
        component = self.config.get('Unit Tests', 'component')
850 c29ac11d Ilias Tsitsimpis

851 c29ac11d Ilias Tsitsimpis
        self.logger.debug("Install needed packages")
852 c29ac11d Ilias Tsitsimpis
        cmd = """
853 8d2c08e4 Christos Stavrakakis
        pip install -U mock
854 8d2c08e4 Christos Stavrakakis
        pip install -U factory_boy
855 c29ac11d Ilias Tsitsimpis
        """
856 c29ac11d Ilias Tsitsimpis
        _run(cmd, False)
857 c29ac11d Ilias Tsitsimpis

858 861b437e Ilias Tsitsimpis
        self.logger.debug("Upload tests.sh file")
859 861b437e Ilias Tsitsimpis
        unit_tests_file = os.path.join(self.ci_dir, "tests.sh")
860 6868804a Ilias Tsitsimpis
        _put(unit_tests_file, ".")
861 c29ac11d Ilias Tsitsimpis

862 c29ac11d Ilias Tsitsimpis
        self.logger.debug("Run unit tests")
863 c29ac11d Ilias Tsitsimpis
        cmd = """
864 861b437e Ilias Tsitsimpis
        bash tests.sh {0}
865 c29ac11d Ilias Tsitsimpis
        """.format(component)
866 c29ac11d Ilias Tsitsimpis
        _run(cmd, True)
867 c29ac11d Ilias Tsitsimpis

868 c29ac11d Ilias Tsitsimpis
    @_check_fabric
869 c29ac11d Ilias Tsitsimpis
    def run_burnin(self):
870 c29ac11d Ilias Tsitsimpis
        """Run burnin functional test suite"""
871 c29ac11d Ilias Tsitsimpis
        self.logger.info("Run Burnin functional test suite")
872 c29ac11d Ilias Tsitsimpis
        cmd = """
873 c29ac11d Ilias Tsitsimpis
        auth_url=$(grep -e '^url =' .kamakirc | cut -d' ' -f3)
874 c29ac11d Ilias Tsitsimpis
        token=$(grep -e '^token =' .kamakirc | cut -d' ' -f3)
875 c29ac11d Ilias Tsitsimpis
        images_user=$(kamaki image list -l | grep owner | \
876 c29ac11d Ilias Tsitsimpis
                      cut -d':' -f2 | tr -d ' ')
877 c29ac11d Ilias Tsitsimpis
        snf-burnin --auth-url=$auth_url --token=$token \
878 c29ac11d Ilias Tsitsimpis
            --force-flavor=2 --image-id=all \
879 c29ac11d Ilias Tsitsimpis
            --system-images-user=$images_user \
880 c29ac11d Ilias Tsitsimpis
            {0}
881 281c7634 Ilias Tsitsimpis
        BurninExitStatus=$?
882 c29ac11d Ilias Tsitsimpis
        log_folder=$(ls -1d /var/log/burnin/* | tail -n1)
883 c29ac11d Ilias Tsitsimpis
        for i in $(ls $log_folder/*/details*); do
884 c29ac11d Ilias Tsitsimpis
            echo -e "\\n\\n"
885 c29ac11d Ilias Tsitsimpis
            echo -e "***** $i\\n"
886 c29ac11d Ilias Tsitsimpis
            cat $i
887 c29ac11d Ilias Tsitsimpis
        done
888 281c7634 Ilias Tsitsimpis
        exit $BurninExitStatus
889 c29ac11d Ilias Tsitsimpis
        """.format(self.config.get('Burnin', 'cmd_options'))
890 c29ac11d Ilias Tsitsimpis
        _run(cmd, True)
891 c29ac11d Ilias Tsitsimpis

892 c29ac11d Ilias Tsitsimpis
    @_check_fabric
893 88e6558b Christos Stavrakakis
    def fetch_compressed(self, src, dest=None):
894 e2a0abb8 Ilias Tsitsimpis
        """Create a tarball and fetch it locally"""
895 88e6558b Christos Stavrakakis
        self.logger.debug("Creating tarball of %s" % src)
896 88e6558b Christos Stavrakakis
        basename = os.path.basename(src)
897 88e6558b Christos Stavrakakis
        tar_file = basename + ".tgz"
898 88e6558b Christos Stavrakakis
        cmd = "tar czf %s %s" % (tar_file, src)
899 c29ac11d Ilias Tsitsimpis
        _run(cmd, False)
900 88e6558b Christos Stavrakakis
        if not os.path.exists(dest):
901 88e6558b Christos Stavrakakis
            os.makedirs(dest)
902 c29ac11d Ilias Tsitsimpis

903 88e6558b Christos Stavrakakis
        tmp_dir = tempfile.mkdtemp()
904 88e6558b Christos Stavrakakis
        fabric.get(tar_file, tmp_dir)
905 c29ac11d Ilias Tsitsimpis

906 88e6558b Christos Stavrakakis
        dest_file = os.path.join(tmp_dir, tar_file)
907 88e6558b Christos Stavrakakis
        self._check_hash_sum(dest_file, tar_file)
908 88e6558b Christos Stavrakakis
        self.logger.debug("Untar packages file %s" % dest_file)
909 e2dc37b2 Christos Stavrakakis
        cmd = """
910 e2dc37b2 Christos Stavrakakis
        cd %s
911 88e6558b Christos Stavrakakis
        tar xzf %s
912 88e6558b Christos Stavrakakis
        cp -r %s/* %s
913 88e6558b Christos Stavrakakis
        rm -r %s
914 88e6558b Christos Stavrakakis
        """ % (tmp_dir, tar_file, src, dest, tmp_dir)
915 e2dc37b2 Christos Stavrakakis
        os.system(cmd)
916 88e6558b Christos Stavrakakis
        self.logger.info("Downloaded %s to %s" %
917 88e6558b Christos Stavrakakis
                         (src, _green(dest)))
918 88e6558b Christos Stavrakakis

919 88e6558b Christos Stavrakakis
    @_check_fabric
920 88e6558b Christos Stavrakakis
    def fetch_packages(self, dest=None):
921 e2a0abb8 Ilias Tsitsimpis
        """Fetch Synnefo packages"""
922 88e6558b Christos Stavrakakis
        if dest is None:
923 88e6558b Christos Stavrakakis
            dest = self.config.get('Global', 'pkgs_dir')
924 6058ae72 Ilias Tsitsimpis
        dest = os.path.abspath(os.path.expanduser(dest))
925 88e6558b Christos Stavrakakis
        if not os.path.exists(dest):
926 88e6558b Christos Stavrakakis
            os.makedirs(dest)
927 88e6558b Christos Stavrakakis
        self.fetch_compressed("synnefo_build-area", dest)
928 99d39120 Ilias Tsitsimpis
        self.fetch_compressed("webclient_build-area", dest)
929 464e58e9 Christos Stavrakakis
        self.logger.info("Downloaded debian packages to %s" %
930 88e6558b Christos Stavrakakis
                         _green(dest))
931 6958ffc6 Christos Stavrakakis

932 cf99ca80 Ilias Tsitsimpis
    def x2go_plugin(self, dest=None):
933 cf99ca80 Ilias Tsitsimpis
        """Produce an html page which will use the x2goplugin
934 cf99ca80 Ilias Tsitsimpis
935 cf99ca80 Ilias Tsitsimpis
        Arguments:
936 cf99ca80 Ilias Tsitsimpis
          dest  -- The file where to save the page (String)
937 cf99ca80 Ilias Tsitsimpis
938 cf99ca80 Ilias Tsitsimpis
        """
939 cf99ca80 Ilias Tsitsimpis
        output_str = """
940 cf99ca80 Ilias Tsitsimpis
        <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
941 cf99ca80 Ilias Tsitsimpis
        <html>
942 cf99ca80 Ilias Tsitsimpis
        <head>
943 cf99ca80 Ilias Tsitsimpis
        <title>X2Go SynnefoCI Service</title>
944 cf99ca80 Ilias Tsitsimpis
        </head>
945 cf99ca80 Ilias Tsitsimpis
        <body onload="checkPlugin()">
946 cf99ca80 Ilias Tsitsimpis
        <div id="x2goplugin">
947 cf99ca80 Ilias Tsitsimpis
            <object
948 cf99ca80 Ilias Tsitsimpis
                src="location"
949 cf99ca80 Ilias Tsitsimpis
                type="application/x2go"
950 cf99ca80 Ilias Tsitsimpis
                name="x2goplugin"
951 cf99ca80 Ilias Tsitsimpis
                palette="background"
952 cf99ca80 Ilias Tsitsimpis
                height="100%"
953 cf99ca80 Ilias Tsitsimpis
                hspace="0"
954 cf99ca80 Ilias Tsitsimpis
                vspace="0"
955 cf99ca80 Ilias Tsitsimpis
                width="100%"
956 cf99ca80 Ilias Tsitsimpis
                x2goconfig="
957 cf99ca80 Ilias Tsitsimpis
                    session=X2Go-SynnefoCI-Session
958 cf99ca80 Ilias Tsitsimpis
                    server={0}
959 cf99ca80 Ilias Tsitsimpis
                    user={1}
960 cf99ca80 Ilias Tsitsimpis
                    sshport={2}
961 cf99ca80 Ilias Tsitsimpis
                    published=true
962 cf99ca80 Ilias Tsitsimpis
                    autologin=true
963 cf99ca80 Ilias Tsitsimpis
                ">
964 cf99ca80 Ilias Tsitsimpis
            </object>
965 cf99ca80 Ilias Tsitsimpis
        </div>
966 cf99ca80 Ilias Tsitsimpis
        </body>
967 cf99ca80 Ilias Tsitsimpis
        </html>
968 cf99ca80 Ilias Tsitsimpis
        """.format(self.read_temp_config('server_ip'),
969 cf99ca80 Ilias Tsitsimpis
                   self.read_temp_config('server_user'),
970 cf99ca80 Ilias Tsitsimpis
                   self.read_temp_config('server_port'))
971 cf99ca80 Ilias Tsitsimpis
        if dest is None:
972 cf99ca80 Ilias Tsitsimpis
            dest = self.config.get('Global', 'x2go_plugin_file')
973 cf99ca80 Ilias Tsitsimpis

974 cf99ca80 Ilias Tsitsimpis
        self.logger.info("Writting x2go plugin html file to %s" % dest)
975 cf99ca80 Ilias Tsitsimpis
        fid = open(dest, 'w')
976 cf99ca80 Ilias Tsitsimpis
        fid.write(output_str)
977 cf99ca80 Ilias Tsitsimpis
        fid.close()
978 cf99ca80 Ilias Tsitsimpis

979 6958ffc6 Christos Stavrakakis

980 6958ffc6 Christos Stavrakakis
def parse_typed_option(option, value):
981 694c79bb Ilias Tsitsimpis
    """Parsed typed options (flavors and images)"""
982 6958ffc6 Christos Stavrakakis
    try:
983 6958ffc6 Christos Stavrakakis
        [type_, val] = value.strip().split(':')
984 6958ffc6 Christos Stavrakakis
        if type_ not in ["id", "name"]:
985 6958ffc6 Christos Stavrakakis
            raise ValueError
986 6958ffc6 Christos Stavrakakis
        return type_, val
987 6958ffc6 Christos Stavrakakis
    except ValueError:
988 6958ffc6 Christos Stavrakakis
        msg = "Invalid %s format. Must be [id|name]:.+" % option
989 6958ffc6 Christos Stavrakakis
        raise ValueError(msg)