Statistics
| Branch: | Tag: | Revision:

root / ci / utils.py @ fe402df2

History | View | Annotate | Download (29.7 kB)

1 525f2979 Ilias Tsitsimpis
#!/usr/bin/env python
2 525f2979 Ilias Tsitsimpis
3 525f2979 Ilias Tsitsimpis
"""
4 525f2979 Ilias Tsitsimpis
Synnefo ci utils module
5 525f2979 Ilias Tsitsimpis
"""
6 525f2979 Ilias Tsitsimpis
7 525f2979 Ilias Tsitsimpis
import os
8 dae06cfc Ilias Tsitsimpis
import re
9 525f2979 Ilias Tsitsimpis
import sys
10 525f2979 Ilias Tsitsimpis
import time
11 525f2979 Ilias Tsitsimpis
import logging
12 525f2979 Ilias Tsitsimpis
import fabric.api as fabric
13 024faf05 Christos Stavrakakis
import subprocess
14 124c300e Christos Stavrakakis
import tempfile
15 525f2979 Ilias Tsitsimpis
from ConfigParser import ConfigParser, DuplicateSectionError
16 525f2979 Ilias Tsitsimpis
17 da593e0c Christos Stavrakakis
from kamaki.cli import config as kamaki_config
18 525f2979 Ilias Tsitsimpis
from kamaki.clients.astakos import AstakosClient
19 525f2979 Ilias Tsitsimpis
from kamaki.clients.cyclades import CycladesClient
20 525f2979 Ilias Tsitsimpis
from kamaki.clients.image import ImageClient
21 dae06cfc Ilias Tsitsimpis
from kamaki.clients.compute import ComputeClient
22 4b61ee63 Ilias Tsitsimpis
import filelocker
23 525f2979 Ilias Tsitsimpis
24 e2db4a57 Christos Stavrakakis
DEFAULT_CONFIG_FILE = "new_config"
25 60ddcdc3 Ilias Tsitsimpis
# UUID of owner of system images
26 60ddcdc3 Ilias Tsitsimpis
DEFAULT_SYSTEM_IMAGES_UUID = [
27 60ddcdc3 Ilias Tsitsimpis
    "25ecced9-bf53-4145-91ee-cf47377e9fb2",  # production (okeanos.grnet.gr)
28 60ddcdc3 Ilias Tsitsimpis
    "04cbe33f-29b7-4ef1-94fb-015929e5fc06",  # testing (okeanos.io)
29 60ddcdc3 Ilias Tsitsimpis
    ]
30 e2db4a57 Christos Stavrakakis
31 525f2979 Ilias Tsitsimpis
32 525f2979 Ilias Tsitsimpis
def _run(cmd, verbose):
33 525f2979 Ilias Tsitsimpis
    """Run fabric with verbose level"""
34 525f2979 Ilias Tsitsimpis
    if verbose:
35 525f2979 Ilias Tsitsimpis
        args = ('running',)
36 525f2979 Ilias Tsitsimpis
    else:
37 525f2979 Ilias Tsitsimpis
        args = ('running', 'stdout',)
38 33ad9a0d Ilias Tsitsimpis
    with fabric.hide(*args):  # Used * or ** magic. pylint: disable-msg=W0142
39 525f2979 Ilias Tsitsimpis
        return fabric.run(cmd)
40 525f2979 Ilias Tsitsimpis
41 525f2979 Ilias Tsitsimpis
42 4c818bb2 Ilias Tsitsimpis
def _put(local, remote):
43 4c818bb2 Ilias Tsitsimpis
    """Run fabric put command without output"""
44 4c818bb2 Ilias Tsitsimpis
    with fabric.quiet():
45 4c818bb2 Ilias Tsitsimpis
        fabric.put(local, remote)
46 4c818bb2 Ilias Tsitsimpis
47 4c818bb2 Ilias Tsitsimpis
48 525f2979 Ilias Tsitsimpis
def _red(msg):
49 525f2979 Ilias Tsitsimpis
    """Red color"""
50 525f2979 Ilias Tsitsimpis
    #return "\x1b[31m" + str(msg) + "\x1b[0m"
51 525f2979 Ilias Tsitsimpis
    return str(msg)
52 525f2979 Ilias Tsitsimpis
53 525f2979 Ilias Tsitsimpis
54 525f2979 Ilias Tsitsimpis
def _yellow(msg):
55 525f2979 Ilias Tsitsimpis
    """Yellow color"""
56 525f2979 Ilias Tsitsimpis
    #return "\x1b[33m" + str(msg) + "\x1b[0m"
57 525f2979 Ilias Tsitsimpis
    return str(msg)
58 525f2979 Ilias Tsitsimpis
59 525f2979 Ilias Tsitsimpis
60 525f2979 Ilias Tsitsimpis
def _green(msg):
61 525f2979 Ilias Tsitsimpis
    """Green color"""
62 525f2979 Ilias Tsitsimpis
    #return "\x1b[32m" + str(msg) + "\x1b[0m"
63 525f2979 Ilias Tsitsimpis
    return str(msg)
64 525f2979 Ilias Tsitsimpis
65 525f2979 Ilias Tsitsimpis
66 525f2979 Ilias Tsitsimpis
def _check_fabric(fun):
67 525f2979 Ilias Tsitsimpis
    """Check if fabric env has been set"""
68 947b6106 Christos Stavrakakis
    def wrapper(self, *args, **kwargs):
69 525f2979 Ilias Tsitsimpis
        """wrapper function"""
70 525f2979 Ilias Tsitsimpis
        if not self.fabric_installed:
71 525f2979 Ilias Tsitsimpis
            self.setup_fabric()
72 eaf0b161 Ilias Tsitsimpis
            self.fabric_installed = True
73 947b6106 Christos Stavrakakis
        return fun(self, *args, **kwargs)
74 525f2979 Ilias Tsitsimpis
    return wrapper
75 525f2979 Ilias Tsitsimpis
76 525f2979 Ilias Tsitsimpis
77 525f2979 Ilias Tsitsimpis
def _check_kamaki(fun):
78 525f2979 Ilias Tsitsimpis
    """Check if kamaki has been initialized"""
79 947b6106 Christos Stavrakakis
    def wrapper(self, *args, **kwargs):
80 525f2979 Ilias Tsitsimpis
        """wrapper function"""
81 525f2979 Ilias Tsitsimpis
        if not self.kamaki_installed:
82 525f2979 Ilias Tsitsimpis
            self.setup_kamaki()
83 eaf0b161 Ilias Tsitsimpis
            self.kamaki_installed = True
84 947b6106 Christos Stavrakakis
        return fun(self, *args, **kwargs)
85 525f2979 Ilias Tsitsimpis
    return wrapper
86 525f2979 Ilias Tsitsimpis
87 525f2979 Ilias Tsitsimpis
88 525f2979 Ilias Tsitsimpis
class _MyFormatter(logging.Formatter):
89 525f2979 Ilias Tsitsimpis
    """Logging Formatter"""
90 525f2979 Ilias Tsitsimpis
    def format(self, record):
91 525f2979 Ilias Tsitsimpis
        format_orig = self._fmt
92 525f2979 Ilias Tsitsimpis
        if record.levelno == logging.DEBUG:
93 525f2979 Ilias Tsitsimpis
            self._fmt = "  %(msg)s"
94 525f2979 Ilias Tsitsimpis
        elif record.levelno == logging.INFO:
95 525f2979 Ilias Tsitsimpis
            self._fmt = "%(msg)s"
96 525f2979 Ilias Tsitsimpis
        elif record.levelno == logging.WARNING:
97 525f2979 Ilias Tsitsimpis
            self._fmt = _yellow("[W] %(msg)s")
98 525f2979 Ilias Tsitsimpis
        elif record.levelno == logging.ERROR:
99 525f2979 Ilias Tsitsimpis
            self._fmt = _red("[E] %(msg)s")
100 525f2979 Ilias Tsitsimpis
        result = logging.Formatter.format(self, record)
101 525f2979 Ilias Tsitsimpis
        self._fmt = format_orig
102 525f2979 Ilias Tsitsimpis
        return result
103 525f2979 Ilias Tsitsimpis
104 525f2979 Ilias Tsitsimpis
105 d8363ea2 Ilias Tsitsimpis
# Too few public methods. pylint: disable-msg=R0903
106 d8363ea2 Ilias Tsitsimpis
class _InfoFilter(logging.Filter):
107 d8363ea2 Ilias Tsitsimpis
    """Logging Filter that allows DEBUG and INFO messages only"""
108 d8363ea2 Ilias Tsitsimpis
    def filter(self, rec):
109 d8363ea2 Ilias Tsitsimpis
        """The filter"""
110 d8363ea2 Ilias Tsitsimpis
        return rec.levelno in (logging.DEBUG, logging.INFO)
111 d8363ea2 Ilias Tsitsimpis
112 d8363ea2 Ilias Tsitsimpis
113 d8363ea2 Ilias Tsitsimpis
# Too many instance attributes. pylint: disable-msg=R0902
114 525f2979 Ilias Tsitsimpis
class SynnefoCI(object):
115 525f2979 Ilias Tsitsimpis
    """SynnefoCI python class"""
116 525f2979 Ilias Tsitsimpis
117 c441b6a7 Ilias Tsitsimpis
    def __init__(self, config_file=None, build_id=None, cloud=None):
118 525f2979 Ilias Tsitsimpis
        """ Initialize SynnefoCI python class
119 525f2979 Ilias Tsitsimpis

120 525f2979 Ilias Tsitsimpis
        Setup logger, local_dir, config and kamaki
121 525f2979 Ilias Tsitsimpis
        """
122 525f2979 Ilias Tsitsimpis
        # Setup logger
123 525f2979 Ilias Tsitsimpis
        self.logger = logging.getLogger('synnefo-ci')
124 525f2979 Ilias Tsitsimpis
        self.logger.setLevel(logging.DEBUG)
125 d8363ea2 Ilias Tsitsimpis
126 d8363ea2 Ilias Tsitsimpis
        handler1 = logging.StreamHandler(sys.stdout)
127 d8363ea2 Ilias Tsitsimpis
        handler1.setLevel(logging.DEBUG)
128 d8363ea2 Ilias Tsitsimpis
        handler1.addFilter(_InfoFilter())
129 d8363ea2 Ilias Tsitsimpis
        handler1.setFormatter(_MyFormatter())
130 d8363ea2 Ilias Tsitsimpis
        handler2 = logging.StreamHandler(sys.stderr)
131 d8363ea2 Ilias Tsitsimpis
        handler2.setLevel(logging.WARNING)
132 d8363ea2 Ilias Tsitsimpis
        handler2.setFormatter(_MyFormatter())
133 d8363ea2 Ilias Tsitsimpis
134 d8363ea2 Ilias Tsitsimpis
        self.logger.addHandler(handler1)
135 d8363ea2 Ilias Tsitsimpis
        self.logger.addHandler(handler2)
136 525f2979 Ilias Tsitsimpis
137 525f2979 Ilias Tsitsimpis
        # Get our local dir
138 525f2979 Ilias Tsitsimpis
        self.ci_dir = os.path.dirname(os.path.abspath(__file__))
139 525f2979 Ilias Tsitsimpis
        self.repo_dir = os.path.dirname(self.ci_dir)
140 525f2979 Ilias Tsitsimpis
141 525f2979 Ilias Tsitsimpis
        # Read config file
142 e2db4a57 Christos Stavrakakis
        if config_file is None:
143 e2db4a57 Christos Stavrakakis
            config_file = DEFAULT_CONFIG_FILE
144 e2db4a57 Christos Stavrakakis
        if not os.path.isabs(config_file):
145 e2db4a57 Christos Stavrakakis
            config_file = os.path.join(self.ci_dir, config_file)
146 525f2979 Ilias Tsitsimpis
        self.config = ConfigParser()
147 525f2979 Ilias Tsitsimpis
        self.config.optionxform = str
148 e2db4a57 Christos Stavrakakis
        self.config.read(config_file)
149 c441b6a7 Ilias Tsitsimpis
150 c441b6a7 Ilias Tsitsimpis
        # Read temporary_config file
151 4b61ee63 Ilias Tsitsimpis
        self.temp_config_file = \
152 4b61ee63 Ilias Tsitsimpis
            os.path.expanduser(self.config.get('Global', 'temporary_config'))
153 c441b6a7 Ilias Tsitsimpis
        self.temp_config = ConfigParser()
154 c441b6a7 Ilias Tsitsimpis
        self.temp_config.optionxform = str
155 4b61ee63 Ilias Tsitsimpis
        self.temp_config.read(self.temp_config_file)
156 ef695867 Ilias Tsitsimpis
        self.build_id = build_id
157 ef695867 Ilias Tsitsimpis
        self.logger.info("Will use \"%s\" as build id" % _green(self.build_id))
158 525f2979 Ilias Tsitsimpis
159 da593e0c Christos Stavrakakis
        # Set kamaki cloud
160 da593e0c Christos Stavrakakis
        if cloud is not None:
161 da593e0c Christos Stavrakakis
            self.kamaki_cloud = cloud
162 da593e0c Christos Stavrakakis
        elif self.config.has_option("Deployment", "kamaki_cloud"):
163 da593e0c Christos Stavrakakis
            kamaki_cloud = self.config.get("Deployment", "kamaki_cloud")
164 da593e0c Christos Stavrakakis
            if kamaki_cloud == "":
165 da593e0c Christos Stavrakakis
                self.kamaki_cloud = None
166 da593e0c Christos Stavrakakis
        else:
167 da593e0c Christos Stavrakakis
            self.kamaki_cloud = None
168 da593e0c Christos Stavrakakis
169 525f2979 Ilias Tsitsimpis
        # Initialize variables
170 525f2979 Ilias Tsitsimpis
        self.fabric_installed = False
171 525f2979 Ilias Tsitsimpis
        self.kamaki_installed = False
172 525f2979 Ilias Tsitsimpis
        self.cyclades_client = None
173 dae06cfc Ilias Tsitsimpis
        self.compute_client = None
174 525f2979 Ilias Tsitsimpis
        self.image_client = None
175 525f2979 Ilias Tsitsimpis
176 525f2979 Ilias Tsitsimpis
    def setup_kamaki(self):
177 525f2979 Ilias Tsitsimpis
        """Initialize kamaki
178 525f2979 Ilias Tsitsimpis

179 dae06cfc Ilias Tsitsimpis
        Setup cyclades_client, image_client and compute_client
180 525f2979 Ilias Tsitsimpis
        """
181 da593e0c Christos Stavrakakis
182 da593e0c Christos Stavrakakis
        config = kamaki_config.Config()
183 da593e0c Christos Stavrakakis
        if self.kamaki_cloud is None:
184 da593e0c Christos Stavrakakis
            self.kamaki_cloud = config.get_global("default_cloud")
185 da593e0c Christos Stavrakakis
186 da593e0c Christos Stavrakakis
        self.logger.info("Setup kamaki client, using cloud '%s'.." %
187 da593e0c Christos Stavrakakis
                         self.kamaki_cloud)
188 da593e0c Christos Stavrakakis
        auth_url = config.get_cloud(self.kamaki_cloud, "url")
189 525f2979 Ilias Tsitsimpis
        self.logger.debug("Authentication URL is %s" % _green(auth_url))
190 da593e0c Christos Stavrakakis
        token = config.get_cloud(self.kamaki_cloud, "token")
191 525f2979 Ilias Tsitsimpis
        #self.logger.debug("Token is %s" % _green(token))
192 525f2979 Ilias Tsitsimpis
193 525f2979 Ilias Tsitsimpis
        astakos_client = AstakosClient(auth_url, token)
194 525f2979 Ilias Tsitsimpis
195 525f2979 Ilias Tsitsimpis
        cyclades_url = \
196 525f2979 Ilias Tsitsimpis
            astakos_client.get_service_endpoints('compute')['publicURL']
197 525f2979 Ilias Tsitsimpis
        self.logger.debug("Cyclades API url is %s" % _green(cyclades_url))
198 525f2979 Ilias Tsitsimpis
        self.cyclades_client = CycladesClient(cyclades_url, token)
199 525f2979 Ilias Tsitsimpis
        self.cyclades_client.CONNECTION_RETRY_LIMIT = 2
200 525f2979 Ilias Tsitsimpis
201 525f2979 Ilias Tsitsimpis
        image_url = \
202 525f2979 Ilias Tsitsimpis
            astakos_client.get_service_endpoints('image')['publicURL']
203 525f2979 Ilias Tsitsimpis
        self.logger.debug("Images API url is %s" % _green(image_url))
204 525f2979 Ilias Tsitsimpis
        self.image_client = ImageClient(cyclades_url, token)
205 525f2979 Ilias Tsitsimpis
        self.image_client.CONNECTION_RETRY_LIMIT = 2
206 525f2979 Ilias Tsitsimpis
207 dae06cfc Ilias Tsitsimpis
        compute_url = \
208 dae06cfc Ilias Tsitsimpis
            astakos_client.get_service_endpoints('compute')['publicURL']
209 dae06cfc Ilias Tsitsimpis
        self.logger.debug("Compute API url is %s" % _green(compute_url))
210 dae06cfc Ilias Tsitsimpis
        self.compute_client = ComputeClient(compute_url, token)
211 dae06cfc Ilias Tsitsimpis
        self.compute_client.CONNECTION_RETRY_LIMIT = 2
212 dae06cfc Ilias Tsitsimpis
213 525f2979 Ilias Tsitsimpis
    def _wait_transition(self, server_id, current_status, new_status):
214 525f2979 Ilias Tsitsimpis
        """Wait for server to go from current_status to new_status"""
215 525f2979 Ilias Tsitsimpis
        self.logger.debug("Waiting for server to become %s" % new_status)
216 525f2979 Ilias Tsitsimpis
        timeout = self.config.getint('Global', 'build_timeout')
217 525f2979 Ilias Tsitsimpis
        sleep_time = 5
218 525f2979 Ilias Tsitsimpis
        while True:
219 525f2979 Ilias Tsitsimpis
            server = self.cyclades_client.get_server_details(server_id)
220 525f2979 Ilias Tsitsimpis
            if server['status'] == new_status:
221 525f2979 Ilias Tsitsimpis
                return server
222 525f2979 Ilias Tsitsimpis
            elif timeout < 0:
223 525f2979 Ilias Tsitsimpis
                self.logger.error(
224 525f2979 Ilias Tsitsimpis
                    "Waiting for server to become %s timed out" % new_status)
225 525f2979 Ilias Tsitsimpis
                self.destroy_server(False)
226 525f2979 Ilias Tsitsimpis
                sys.exit(-1)
227 525f2979 Ilias Tsitsimpis
            elif server['status'] == current_status:
228 525f2979 Ilias Tsitsimpis
                # Sleep for #n secs and continue
229 525f2979 Ilias Tsitsimpis
                timeout = timeout - sleep_time
230 525f2979 Ilias Tsitsimpis
                time.sleep(sleep_time)
231 525f2979 Ilias Tsitsimpis
            else:
232 525f2979 Ilias Tsitsimpis
                self.logger.error(
233 525f2979 Ilias Tsitsimpis
                    "Server failed with status %s" % server['status'])
234 525f2979 Ilias Tsitsimpis
                self.destroy_server(False)
235 525f2979 Ilias Tsitsimpis
                sys.exit(-1)
236 525f2979 Ilias Tsitsimpis
237 525f2979 Ilias Tsitsimpis
    @_check_kamaki
238 525f2979 Ilias Tsitsimpis
    def destroy_server(self, wait=True):
239 525f2979 Ilias Tsitsimpis
        """Destroy slave server"""
240 ef695867 Ilias Tsitsimpis
        server_id = int(self.read_temp_config('server_id'))
241 525f2979 Ilias Tsitsimpis
        self.logger.info("Destoying server with id %s " % server_id)
242 525f2979 Ilias Tsitsimpis
        self.cyclades_client.delete_server(server_id)
243 525f2979 Ilias Tsitsimpis
        if wait:
244 525f2979 Ilias Tsitsimpis
            self._wait_transition(server_id, "ACTIVE", "DELETED")
245 525f2979 Ilias Tsitsimpis
246 525f2979 Ilias Tsitsimpis
    @_check_kamaki
247 6f8b2632 Ilias Tsitsimpis
    def create_server(self, image=None, flavor=None, ssh_keys=None):
248 525f2979 Ilias Tsitsimpis
        """Create slave server"""
249 525f2979 Ilias Tsitsimpis
        self.logger.info("Create a new server..")
250 ef695867 Ilias Tsitsimpis
251 ef695867 Ilias Tsitsimpis
        # Find a build_id to use
252 ef695867 Ilias Tsitsimpis
        if self.build_id is None:
253 ef695867 Ilias Tsitsimpis
            # If build_id is given use this, else ..
254 ef695867 Ilias Tsitsimpis
            # Find a uniq build_id to use
255 ef695867 Ilias Tsitsimpis
            ids = self.temp_config.sections()
256 ef695867 Ilias Tsitsimpis
            if ids:
257 ef695867 Ilias Tsitsimpis
                max_id = int(max(self.temp_config.sections(), key=int))
258 ef695867 Ilias Tsitsimpis
                self.build_id = max_id + 1
259 ef695867 Ilias Tsitsimpis
            else:
260 ef695867 Ilias Tsitsimpis
                self.build_id = 1
261 ef695867 Ilias Tsitsimpis
        self.logger.debug("New build id \"%s\" was created"
262 ef695867 Ilias Tsitsimpis
                          % _green(self.build_id))
263 ef695867 Ilias Tsitsimpis
264 ef695867 Ilias Tsitsimpis
        # Find an image to use
265 6f8b2632 Ilias Tsitsimpis
        image_id = self._find_image(image)
266 ef695867 Ilias Tsitsimpis
        # Find a flavor to use
267 6f8b2632 Ilias Tsitsimpis
        flavor_id = self._find_flavor(flavor)
268 6f8b2632 Ilias Tsitsimpis
269 6f8b2632 Ilias Tsitsimpis
        # Create Server
270 69a64e60 Christos Stavrakakis
        server_name = self.config.get("Deployment", "server_name")
271 525f2979 Ilias Tsitsimpis
        server = self.cyclades_client.create_server(
272 69a64e60 Christos Stavrakakis
            "%s(BID: %s)" % (server_name, self.build_id),
273 947b6106 Christos Stavrakakis
            flavor_id,
274 947b6106 Christos Stavrakakis
            image_id)
275 525f2979 Ilias Tsitsimpis
        server_id = server['id']
276 ef695867 Ilias Tsitsimpis
        self.write_temp_config('server_id', server_id)
277 525f2979 Ilias Tsitsimpis
        self.logger.debug("Server got id %s" % _green(server_id))
278 525f2979 Ilias Tsitsimpis
        server_user = server['metadata']['users']
279 ef695867 Ilias Tsitsimpis
        self.write_temp_config('server_user', server_user)
280 525f2979 Ilias Tsitsimpis
        self.logger.debug("Server's admin user is %s" % _green(server_user))
281 525f2979 Ilias Tsitsimpis
        server_passwd = server['adminPass']
282 ef695867 Ilias Tsitsimpis
        self.write_temp_config('server_passwd', server_passwd)
283 525f2979 Ilias Tsitsimpis
284 525f2979 Ilias Tsitsimpis
        server = self._wait_transition(server_id, "BUILD", "ACTIVE")
285 525f2979 Ilias Tsitsimpis
        self._get_server_ip_and_port(server)
286 4c818bb2 Ilias Tsitsimpis
        self._copy_ssh_keys(ssh_keys)
287 525f2979 Ilias Tsitsimpis
288 eaf0b161 Ilias Tsitsimpis
        # Setup Firewall
289 525f2979 Ilias Tsitsimpis
        self.setup_fabric()
290 525f2979 Ilias Tsitsimpis
        self.logger.info("Setup firewall")
291 4c818bb2 Ilias Tsitsimpis
        accept_ssh_from = self.config.get('Global', 'accept_ssh_from')
292 0082c787 Christos Stavrakakis
        if accept_ssh_from != "":
293 0082c787 Christos Stavrakakis
            self.logger.debug("Block ssh except from %s" % accept_ssh_from)
294 0082c787 Christos Stavrakakis
            cmd = """
295 0082c787 Christos Stavrakakis
            local_ip=$(/sbin/ifconfig eth0 | grep 'inet addr:' | \
296 0082c787 Christos Stavrakakis
                cut -d':' -f2 | cut -d' ' -f1)
297 0082c787 Christos Stavrakakis
            iptables -A INPUT -s localhost -j ACCEPT
298 0082c787 Christos Stavrakakis
            iptables -A INPUT -s $local_ip -j ACCEPT
299 0082c787 Christos Stavrakakis
            iptables -A INPUT -s {0} -p tcp --dport 22 -j ACCEPT
300 0082c787 Christos Stavrakakis
            iptables -A INPUT -p tcp --dport 22 -j DROP
301 0082c787 Christos Stavrakakis
            """.format(accept_ssh_from)
302 0082c787 Christos Stavrakakis
            _run(cmd, False)
303 525f2979 Ilias Tsitsimpis
304 eaf0b161 Ilias Tsitsimpis
        # Setup apt, download packages
305 eaf0b161 Ilias Tsitsimpis
        self.logger.debug("Setup apt. Install x2goserver and firefox")
306 eaf0b161 Ilias Tsitsimpis
        cmd = """
307 eaf0b161 Ilias Tsitsimpis
        echo 'APT::Install-Suggests "false";' >> /etc/apt/apt.conf
308 eaf0b161 Ilias Tsitsimpis
        apt-get update
309 769db39e Ilias Tsitsimpis
        apt-get install curl --yes --force-yes
310 eaf0b161 Ilias Tsitsimpis
        echo -e "\n\n{0}" >> /etc/apt/sources.list
311 eaf0b161 Ilias Tsitsimpis
        # Synnefo repo's key
312 eaf0b161 Ilias Tsitsimpis
        curl https://dev.grnet.gr/files/apt-grnetdev.pub | apt-key add -
313 eaf0b161 Ilias Tsitsimpis
        # X2GO Key
314 eaf0b161 Ilias Tsitsimpis
        apt-key adv --recv-keys --keyserver keys.gnupg.net E1F958385BFE2B6E
315 769db39e Ilias Tsitsimpis
        apt-get install x2go-keyring --yes --force-yes
316 eaf0b161 Ilias Tsitsimpis
        apt-get update
317 769db39e Ilias Tsitsimpis
        apt-get install x2goserver x2goserver-xsession \
318 769db39e Ilias Tsitsimpis
                iceweasel --yes --force-yes
319 eaf0b161 Ilias Tsitsimpis
        """.format(self.config.get('Global', 'apt_repo'))
320 eaf0b161 Ilias Tsitsimpis
        _run(cmd, False)
321 eaf0b161 Ilias Tsitsimpis
322 6f8b2632 Ilias Tsitsimpis
    def _find_flavor(self, flavor=None):
323 6f8b2632 Ilias Tsitsimpis
        """Find a suitable flavor to use
324 6f8b2632 Ilias Tsitsimpis

325 6f8b2632 Ilias Tsitsimpis
        Search by name (reg expression) or by id
326 6f8b2632 Ilias Tsitsimpis
        """
327 6f8b2632 Ilias Tsitsimpis
        # Get a list of flavors from config file
328 6f8b2632 Ilias Tsitsimpis
        flavors = self.config.get('Deployment', 'flavors').split(",")
329 6f8b2632 Ilias Tsitsimpis
        if flavor is not None:
330 c314fcd2 Ilias Tsitsimpis
            # If we have a flavor_name to use, add it to our list
331 6f8b2632 Ilias Tsitsimpis
            flavors.insert(0, flavor)
332 6f8b2632 Ilias Tsitsimpis
333 6f8b2632 Ilias Tsitsimpis
        list_flavors = self.compute_client.list_flavors()
334 6f8b2632 Ilias Tsitsimpis
        for flv in flavors:
335 fe402df2 Christos Stavrakakis
            flv_type, flv_value = parse_typed_option(option="flavor", value=flv)
336 6f8b2632 Ilias Tsitsimpis
            if flv_type == "name":
337 6f8b2632 Ilias Tsitsimpis
                # Filter flavors by name
338 6f8b2632 Ilias Tsitsimpis
                self.logger.debug(
339 6f8b2632 Ilias Tsitsimpis
                    "Trying to find a flavor with name \"%s\"" % flv_value)
340 6f8b2632 Ilias Tsitsimpis
                list_flvs = \
341 6f8b2632 Ilias Tsitsimpis
                    [f for f in list_flavors
342 6f8b2632 Ilias Tsitsimpis
                     if re.search(flv_value, f['name'], flags=re.I) is not None]
343 6f8b2632 Ilias Tsitsimpis
            elif flv_type == "id":
344 6f8b2632 Ilias Tsitsimpis
                # Filter flavors by id
345 6f8b2632 Ilias Tsitsimpis
                self.logger.debug(
346 6f8b2632 Ilias Tsitsimpis
                    "Trying to find a flavor with id \"%s\"" % flv_value)
347 6f8b2632 Ilias Tsitsimpis
                list_flvs = \
348 6f8b2632 Ilias Tsitsimpis
                    [f for f in list_flavors
349 6f8b2632 Ilias Tsitsimpis
                     if f['id'].lower() == flv_value.lower()]
350 6f8b2632 Ilias Tsitsimpis
            else:
351 6f8b2632 Ilias Tsitsimpis
                self.logger.error("Unrecognized flavor type %s" % flv_type)
352 6f8b2632 Ilias Tsitsimpis
353 6f8b2632 Ilias Tsitsimpis
            # Check if we found one
354 6f8b2632 Ilias Tsitsimpis
            if list_flvs:
355 6f8b2632 Ilias Tsitsimpis
                self.logger.debug("Will use \"%s\" with id \"%s\""
356 6f8b2632 Ilias Tsitsimpis
                                  % (list_flvs[0]['name'], list_flvs[0]['id']))
357 6f8b2632 Ilias Tsitsimpis
                return list_flvs[0]['id']
358 c314fcd2 Ilias Tsitsimpis
359 c314fcd2 Ilias Tsitsimpis
        self.logger.error("No matching flavor found.. aborting")
360 c314fcd2 Ilias Tsitsimpis
        sys.exit(1)
361 dae06cfc Ilias Tsitsimpis
362 6f8b2632 Ilias Tsitsimpis
    def _find_image(self, image=None):
363 525f2979 Ilias Tsitsimpis
        """Find a suitable image to use
364 525f2979 Ilias Tsitsimpis

365 6f8b2632 Ilias Tsitsimpis
        In case of search by name, the image has to belong to one
366 6f8b2632 Ilias Tsitsimpis
        of the `DEFAULT_SYSTEM_IMAGES_UUID' users.
367 6f8b2632 Ilias Tsitsimpis
        In case of search by id it only has to exist.
368 525f2979 Ilias Tsitsimpis
        """
369 6f8b2632 Ilias Tsitsimpis
        # Get a list of images from config file
370 6f8b2632 Ilias Tsitsimpis
        images = self.config.get('Deployment', 'images').split(",")
371 6f8b2632 Ilias Tsitsimpis
        if image is not None:
372 6f8b2632 Ilias Tsitsimpis
            # If we have an image from command line, add it to our list
373 6f8b2632 Ilias Tsitsimpis
            images.insert(0, image)
374 6f8b2632 Ilias Tsitsimpis
375 6f8b2632 Ilias Tsitsimpis
        list_images = self.image_client.list_public(detail=True)['images']
376 6f8b2632 Ilias Tsitsimpis
        for img in images:
377 fe402df2 Christos Stavrakakis
            img_type, img_value = parse_typed_option(option="image", value=img)
378 6f8b2632 Ilias Tsitsimpis
            if img_type == "name":
379 6f8b2632 Ilias Tsitsimpis
                # Filter images by name
380 6f8b2632 Ilias Tsitsimpis
                self.logger.debug(
381 6f8b2632 Ilias Tsitsimpis
                    "Trying to find an image with name \"%s\"" % img_value)
382 6f8b2632 Ilias Tsitsimpis
                list_imgs = \
383 6f8b2632 Ilias Tsitsimpis
                    [i for i in list_images
384 6f8b2632 Ilias Tsitsimpis
                     if i['user_id'] in DEFAULT_SYSTEM_IMAGES_UUID and
385 6f8b2632 Ilias Tsitsimpis
                        re.search(img_value, i['name'], flags=re.I) is not None]
386 6f8b2632 Ilias Tsitsimpis
            elif img_type == "id":
387 6f8b2632 Ilias Tsitsimpis
                # Filter images by id
388 6f8b2632 Ilias Tsitsimpis
                self.logger.debug(
389 6f8b2632 Ilias Tsitsimpis
                    "Trying to find an image with id \"%s\"" % img_value)
390 6f8b2632 Ilias Tsitsimpis
                list_imgs = \
391 6f8b2632 Ilias Tsitsimpis
                    [i for i in list_images
392 6f8b2632 Ilias Tsitsimpis
                     if i['id'].lower() == img_value.lower()]
393 6f8b2632 Ilias Tsitsimpis
            else:
394 6f8b2632 Ilias Tsitsimpis
                self.logger.error("Unrecognized image type %s" % img_type)
395 6f8b2632 Ilias Tsitsimpis
                sys.exit(1)
396 6f8b2632 Ilias Tsitsimpis
397 6f8b2632 Ilias Tsitsimpis
            # Check if we found one
398 6f8b2632 Ilias Tsitsimpis
            if list_imgs:
399 6f8b2632 Ilias Tsitsimpis
                self.logger.debug("Will use \"%s\" with id \"%s\""
400 6f8b2632 Ilias Tsitsimpis
                                  % (list_imgs[0]['name'], list_imgs[0]['id']))
401 6f8b2632 Ilias Tsitsimpis
                return list_imgs[0]['id']
402 6f8b2632 Ilias Tsitsimpis
403 6f8b2632 Ilias Tsitsimpis
        # We didn't found one
404 6f8b2632 Ilias Tsitsimpis
        self.logger.error("No matching image found.. aborting")
405 6f8b2632 Ilias Tsitsimpis
        sys.exit(1)
406 525f2979 Ilias Tsitsimpis
407 525f2979 Ilias Tsitsimpis
    def _get_server_ip_and_port(self, server):
408 525f2979 Ilias Tsitsimpis
        """Compute server's IPv4 and ssh port number"""
409 525f2979 Ilias Tsitsimpis
        self.logger.info("Get server connection details..")
410 525f2979 Ilias Tsitsimpis
        server_ip = server['attachments'][0]['ipv4']
411 c4c22bb4 Christos Stavrakakis
        if (".okeanos.io" in self.cyclades_client.base_url or
412 c4c22bb4 Christos Stavrakakis
           ".demo.synnefo.org" in self.cyclades_client.base_url):
413 525f2979 Ilias Tsitsimpis
            tmp1 = int(server_ip.split(".")[2])
414 525f2979 Ilias Tsitsimpis
            tmp2 = int(server_ip.split(".")[3])
415 525f2979 Ilias Tsitsimpis
            server_ip = "gate.okeanos.io"
416 525f2979 Ilias Tsitsimpis
            server_port = 10000 + tmp1 * 256 + tmp2
417 525f2979 Ilias Tsitsimpis
        else:
418 525f2979 Ilias Tsitsimpis
            server_port = 22
419 ef695867 Ilias Tsitsimpis
        self.write_temp_config('server_ip', server_ip)
420 525f2979 Ilias Tsitsimpis
        self.logger.debug("Server's IPv4 is %s" % _green(server_ip))
421 ef695867 Ilias Tsitsimpis
        self.write_temp_config('server_port', server_port)
422 525f2979 Ilias Tsitsimpis
        self.logger.debug("Server's ssh port is %s" % _green(server_port))
423 064d2f1b Ilias Tsitsimpis
        self.logger.debug("Access server using \"ssh -X -p %s %s@%s\"" %
424 2fab294e Ilias Tsitsimpis
                          (server_port, server['metadata']['users'], server_ip))
425 525f2979 Ilias Tsitsimpis
426 5c3b5c9f Christos Stavrakakis
    @_check_fabric
427 4c818bb2 Ilias Tsitsimpis
    def _copy_ssh_keys(self, ssh_keys):
428 33ad9a0d Ilias Tsitsimpis
        """Upload/Install ssh keys to server"""
429 2fab294e Ilias Tsitsimpis
        self.logger.debug("Check for authentication keys to use")
430 4c818bb2 Ilias Tsitsimpis
        if ssh_keys is None:
431 4c818bb2 Ilias Tsitsimpis
            ssh_keys = self.config.get("Deployment", "ssh_keys")
432 4c818bb2 Ilias Tsitsimpis
433 2fab294e Ilias Tsitsimpis
        if ssh_keys != "":
434 8baa4ae7 Ilias Tsitsimpis
            ssh_keys = os.path.expanduser(ssh_keys)
435 2fab294e Ilias Tsitsimpis
            self.logger.debug("Will use %s authentication keys file" % ssh_keys)
436 5c3b5c9f Christos Stavrakakis
            keyfile = '/tmp/%s.pub' % fabric.env.user
437 5c3b5c9f Christos Stavrakakis
            _run('mkdir -p ~/.ssh && chmod 700 ~/.ssh', False)
438 2fab294e Ilias Tsitsimpis
            if ssh_keys.startswith("http://") or \
439 2fab294e Ilias Tsitsimpis
                    ssh_keys.startswith("https://") or \
440 2fab294e Ilias Tsitsimpis
                    ssh_keys.startswith("ftp://"):
441 2fab294e Ilias Tsitsimpis
                cmd = """
442 2fab294e Ilias Tsitsimpis
                apt-get update
443 769db39e Ilias Tsitsimpis
                apt-get install wget --yes --force-yes
444 2fab294e Ilias Tsitsimpis
                wget {0} -O {1} --no-check-certificate
445 2fab294e Ilias Tsitsimpis
                """.format(ssh_keys, keyfile)
446 2fab294e Ilias Tsitsimpis
                _run(cmd, False)
447 2fab294e Ilias Tsitsimpis
            elif os.path.exists(ssh_keys):
448 2fab294e Ilias Tsitsimpis
                _put(ssh_keys, keyfile)
449 2fab294e Ilias Tsitsimpis
            else:
450 2fab294e Ilias Tsitsimpis
                self.logger.debug("No ssh keys found")
451 8baa4ae7 Ilias Tsitsimpis
                return
452 5c3b5c9f Christos Stavrakakis
            _run('cat %s >> ~/.ssh/authorized_keys' % keyfile, False)
453 5c3b5c9f Christos Stavrakakis
            _run('rm %s' % keyfile, False)
454 5c3b5c9f Christos Stavrakakis
            self.logger.debug("Uploaded ssh authorized keys")
455 5c3b5c9f Christos Stavrakakis
        else:
456 5c3b5c9f Christos Stavrakakis
            self.logger.debug("No ssh keys found")
457 5c3b5c9f Christos Stavrakakis
458 ef695867 Ilias Tsitsimpis
    def write_temp_config(self, option, value):
459 525f2979 Ilias Tsitsimpis
        """Write changes back to config file"""
460 4b61ee63 Ilias Tsitsimpis
        # Acquire the lock to write to temp_config_file
461 4b61ee63 Ilias Tsitsimpis
        with filelocker.lock("%s.lock" % self.temp_config_file,
462 4b61ee63 Ilias Tsitsimpis
                             filelocker.LOCK_EX):
463 4b61ee63 Ilias Tsitsimpis
464 4b61ee63 Ilias Tsitsimpis
            # Read temp_config again to get any new entries
465 4b61ee63 Ilias Tsitsimpis
            self.temp_config.read(self.temp_config_file)
466 4b61ee63 Ilias Tsitsimpis
467 4b61ee63 Ilias Tsitsimpis
            # If build_id section doesn't exist create a new one
468 4b61ee63 Ilias Tsitsimpis
            try:
469 4b61ee63 Ilias Tsitsimpis
                self.temp_config.add_section(str(self.build_id))
470 4b61ee63 Ilias Tsitsimpis
                creation_time = \
471 4b61ee63 Ilias Tsitsimpis
                    time.strftime("%a, %d %b %Y %X", time.localtime())
472 4b61ee63 Ilias Tsitsimpis
                self.temp_config.set(str(self.build_id),
473 4b61ee63 Ilias Tsitsimpis
                                     "created", str(creation_time))
474 4b61ee63 Ilias Tsitsimpis
            except DuplicateSectionError:
475 4b61ee63 Ilias Tsitsimpis
                pass
476 4b61ee63 Ilias Tsitsimpis
            self.temp_config.set(str(self.build_id), option, str(value))
477 4b61ee63 Ilias Tsitsimpis
            curr_time = time.strftime("%a, %d %b %Y %X", time.localtime())
478 4b61ee63 Ilias Tsitsimpis
            self.temp_config.set(str(self.build_id), "modified", curr_time)
479 4b61ee63 Ilias Tsitsimpis
            with open(self.temp_config_file, 'wb') as tcf:
480 4b61ee63 Ilias Tsitsimpis
                self.temp_config.write(tcf)
481 525f2979 Ilias Tsitsimpis
482 ef695867 Ilias Tsitsimpis
    def read_temp_config(self, option):
483 ef695867 Ilias Tsitsimpis
        """Read from temporary_config file"""
484 ef695867 Ilias Tsitsimpis
        # If build_id is None use the latest one
485 ef695867 Ilias Tsitsimpis
        if self.build_id is None:
486 ef695867 Ilias Tsitsimpis
            ids = self.temp_config.sections()
487 ef695867 Ilias Tsitsimpis
            if ids:
488 ef695867 Ilias Tsitsimpis
                self.build_id = int(ids[-1])
489 ef695867 Ilias Tsitsimpis
            else:
490 ef695867 Ilias Tsitsimpis
                self.logger.error("No sections in temporary config file")
491 ef695867 Ilias Tsitsimpis
                sys.exit(1)
492 ef695867 Ilias Tsitsimpis
            self.logger.debug("Will use \"%s\" as build id"
493 ef695867 Ilias Tsitsimpis
                              % _green(self.build_id))
494 ef695867 Ilias Tsitsimpis
        # Read specified option
495 ef695867 Ilias Tsitsimpis
        return self.temp_config.get(str(self.build_id), option)
496 ef695867 Ilias Tsitsimpis
497 525f2979 Ilias Tsitsimpis
    def setup_fabric(self):
498 525f2979 Ilias Tsitsimpis
        """Setup fabric environment"""
499 525f2979 Ilias Tsitsimpis
        self.logger.info("Setup fabric parameters..")
500 ef695867 Ilias Tsitsimpis
        fabric.env.user = self.read_temp_config('server_user')
501 ef695867 Ilias Tsitsimpis
        fabric.env.host_string = self.read_temp_config('server_ip')
502 ef695867 Ilias Tsitsimpis
        fabric.env.port = int(self.read_temp_config('server_port'))
503 ef695867 Ilias Tsitsimpis
        fabric.env.password = self.read_temp_config('server_passwd')
504 525f2979 Ilias Tsitsimpis
        fabric.env.connection_attempts = 10
505 525f2979 Ilias Tsitsimpis
        fabric.env.shell = "/bin/bash -c"
506 525f2979 Ilias Tsitsimpis
        fabric.env.disable_known_hosts = True
507 525f2979 Ilias Tsitsimpis
        fabric.env.output_prefix = None
508 525f2979 Ilias Tsitsimpis
509 525f2979 Ilias Tsitsimpis
    def _check_hash_sum(self, localfile, remotefile):
510 525f2979 Ilias Tsitsimpis
        """Check hash sums of two files"""
511 525f2979 Ilias Tsitsimpis
        self.logger.debug("Check hash sum for local file %s" % localfile)
512 525f2979 Ilias Tsitsimpis
        hash1 = os.popen("sha256sum %s" % localfile).read().split(' ')[0]
513 525f2979 Ilias Tsitsimpis
        self.logger.debug("Local file has sha256 hash %s" % hash1)
514 525f2979 Ilias Tsitsimpis
        self.logger.debug("Check hash sum for remote file %s" % remotefile)
515 525f2979 Ilias Tsitsimpis
        hash2 = _run("sha256sum %s" % remotefile, False)
516 525f2979 Ilias Tsitsimpis
        hash2 = hash2.split(' ')[0]
517 525f2979 Ilias Tsitsimpis
        self.logger.debug("Remote file has sha256 hash %s" % hash2)
518 525f2979 Ilias Tsitsimpis
        if hash1 != hash2:
519 525f2979 Ilias Tsitsimpis
            self.logger.error("Hashes differ.. aborting")
520 525f2979 Ilias Tsitsimpis
            sys.exit(-1)
521 525f2979 Ilias Tsitsimpis
522 525f2979 Ilias Tsitsimpis
    @_check_fabric
523 1d9f2031 Ilias Tsitsimpis
    def clone_repo(self, local_repo=False):
524 525f2979 Ilias Tsitsimpis
        """Clone Synnefo repo from slave server"""
525 525f2979 Ilias Tsitsimpis
        self.logger.info("Configure repositories on remote server..")
526 eaf0b161 Ilias Tsitsimpis
        self.logger.debug("Install/Setup git")
527 525f2979 Ilias Tsitsimpis
        cmd = """
528 769db39e Ilias Tsitsimpis
        apt-get install git --yes --force-yes
529 eaf0b161 Ilias Tsitsimpis
        git config --global user.name {0}
530 eaf0b161 Ilias Tsitsimpis
        git config --global user.email {1}
531 eaf0b161 Ilias Tsitsimpis
        """.format(self.config.get('Global', 'git_config_name'),
532 525f2979 Ilias Tsitsimpis
                   self.config.get('Global', 'git_config_mail'))
533 525f2979 Ilias Tsitsimpis
        _run(cmd, False)
534 525f2979 Ilias Tsitsimpis
535 1d9f2031 Ilias Tsitsimpis
        # Find synnefo_repo and synnefo_branch to use
536 525f2979 Ilias Tsitsimpis
        synnefo_repo = self.config.get('Global', 'synnefo_repo')
537 024faf05 Christos Stavrakakis
        synnefo_branch = self.config.get("Global", "synnefo_branch")
538 024faf05 Christos Stavrakakis
        if synnefo_branch == "":
539 33ad9a0d Ilias Tsitsimpis
            synnefo_branch = \
540 33ad9a0d Ilias Tsitsimpis
                subprocess.Popen(
541 33ad9a0d Ilias Tsitsimpis
                    ["git", "rev-parse", "--abbrev-ref", "HEAD"],
542 024faf05 Christos Stavrakakis
                    stdout=subprocess.PIPE).communicate()[0].strip()
543 024faf05 Christos Stavrakakis
            if synnefo_branch == "HEAD":
544 024faf05 Christos Stavrakakis
                synnefo_branch = \
545 33ad9a0d Ilias Tsitsimpis
                    subprocess.Popen(
546 33ad9a0d Ilias Tsitsimpis
                        ["git", "rev-parse", "--short", "HEAD"],
547 024faf05 Christos Stavrakakis
                        stdout=subprocess.PIPE).communicate()[0].strip()
548 024faf05 Christos Stavrakakis
        self.logger.info("Will use branch %s" % synnefo_branch)
549 1d9f2031 Ilias Tsitsimpis
550 1d9f2031 Ilias Tsitsimpis
        if local_repo or synnefo_branch == "":
551 1d9f2031 Ilias Tsitsimpis
            # Use local_repo
552 1d9f2031 Ilias Tsitsimpis
            self.logger.debug("Push local repo to server")
553 1d9f2031 Ilias Tsitsimpis
            # Firstly create the remote repo
554 1d9f2031 Ilias Tsitsimpis
            _run("git init synnefo", False)
555 1d9f2031 Ilias Tsitsimpis
            # Then push our local repo over ssh
556 1d9f2031 Ilias Tsitsimpis
            # We have to pass some arguments to ssh command
557 1d9f2031 Ilias Tsitsimpis
            # namely to disable host checking.
558 1d9f2031 Ilias Tsitsimpis
            (temp_ssh_file_handle, temp_ssh_file) = tempfile.mkstemp()
559 1d9f2031 Ilias Tsitsimpis
            os.close(temp_ssh_file_handle)
560 4b61ee63 Ilias Tsitsimpis
            # XXX: git push doesn't read the password
561 1d9f2031 Ilias Tsitsimpis
            cmd = """
562 1d9f2031 Ilias Tsitsimpis
            echo 'exec ssh -o "StrictHostKeyChecking no" \
563 1d9f2031 Ilias Tsitsimpis
                           -o "UserKnownHostsFile /dev/null" \
564 1d9f2031 Ilias Tsitsimpis
                           -q "$@"' > {4}
565 1d9f2031 Ilias Tsitsimpis
            chmod u+x {4}
566 1d9f2031 Ilias Tsitsimpis
            export GIT_SSH="{4}"
567 1d9f2031 Ilias Tsitsimpis
            echo "{0}" | git push --mirror ssh://{1}@{2}:{3}/~/synnefo
568 1d9f2031 Ilias Tsitsimpis
            rm -f {4}
569 1d9f2031 Ilias Tsitsimpis
            """.format(fabric.env.password,
570 1d9f2031 Ilias Tsitsimpis
                       fabric.env.user,
571 1d9f2031 Ilias Tsitsimpis
                       fabric.env.host_string,
572 1d9f2031 Ilias Tsitsimpis
                       fabric.env.port,
573 1d9f2031 Ilias Tsitsimpis
                       temp_ssh_file)
574 1d9f2031 Ilias Tsitsimpis
            os.system(cmd)
575 1d9f2031 Ilias Tsitsimpis
        else:
576 1d9f2031 Ilias Tsitsimpis
            # Clone Synnefo from remote repo
577 1d9f2031 Ilias Tsitsimpis
            # Currently clonning synnefo can fail unexpectedly
578 1d9f2031 Ilias Tsitsimpis
            cloned = False
579 1d9f2031 Ilias Tsitsimpis
            for i in range(10):
580 1d9f2031 Ilias Tsitsimpis
                self.logger.debug("Clone synnefo from %s" % synnefo_repo)
581 1d9f2031 Ilias Tsitsimpis
                try:
582 1d9f2031 Ilias Tsitsimpis
                    _run("git clone %s synnefo" % synnefo_repo, False)
583 1d9f2031 Ilias Tsitsimpis
                    cloned = True
584 1d9f2031 Ilias Tsitsimpis
                    break
585 1d9f2031 Ilias Tsitsimpis
                except BaseException:
586 1d9f2031 Ilias Tsitsimpis
                    self.logger.warning(
587 1d9f2031 Ilias Tsitsimpis
                        "Clonning synnefo failed.. retrying %s" % i)
588 1d9f2031 Ilias Tsitsimpis
            if not cloned:
589 1d9f2031 Ilias Tsitsimpis
                self.logger.error("Can not clone Synnefo repo.")
590 1d9f2031 Ilias Tsitsimpis
                sys.exit(-1)
591 1d9f2031 Ilias Tsitsimpis
592 1d9f2031 Ilias Tsitsimpis
        # Checkout the desired synnefo_branch
593 064d2f1b Ilias Tsitsimpis
        self.logger.debug("Checkout \"%s\" branch/commit" % synnefo_branch)
594 33ad9a0d Ilias Tsitsimpis
        cmd = """
595 024faf05 Christos Stavrakakis
        cd synnefo
596 33ad9a0d Ilias Tsitsimpis
        for branch in `git branch -a | grep remotes | \
597 33ad9a0d Ilias Tsitsimpis
                       grep -v HEAD | grep -v master`; do
598 024faf05 Christos Stavrakakis
            git branch --track ${branch##*/} $branch
599 024faf05 Christos Stavrakakis
        done
600 024faf05 Christos Stavrakakis
        git checkout %s
601 024faf05 Christos Stavrakakis
        """ % (synnefo_branch)
602 024faf05 Christos Stavrakakis
        _run(cmd, False)
603 024faf05 Christos Stavrakakis
604 525f2979 Ilias Tsitsimpis
    @_check_fabric
605 525f2979 Ilias Tsitsimpis
    def build_synnefo(self):
606 525f2979 Ilias Tsitsimpis
        """Build Synnefo packages"""
607 525f2979 Ilias Tsitsimpis
        self.logger.info("Build Synnefo packages..")
608 525f2979 Ilias Tsitsimpis
        self.logger.debug("Install development packages")
609 525f2979 Ilias Tsitsimpis
        cmd = """
610 525f2979 Ilias Tsitsimpis
        apt-get update
611 525f2979 Ilias Tsitsimpis
        apt-get install zlib1g-dev dpkg-dev debhelper git-buildpackage \
612 769db39e Ilias Tsitsimpis
                python-dev python-all python-pip --yes --force-yes
613 525f2979 Ilias Tsitsimpis
        pip install devflow
614 525f2979 Ilias Tsitsimpis
        """
615 525f2979 Ilias Tsitsimpis
        _run(cmd, False)
616 525f2979 Ilias Tsitsimpis
617 133e5a5b Christos Stavrakakis
        if self.config.get('Global', 'patch_pydist') == "True":
618 525f2979 Ilias Tsitsimpis
            self.logger.debug("Patch pydist.py module")
619 525f2979 Ilias Tsitsimpis
            cmd = r"""
620 525f2979 Ilias Tsitsimpis
            sed -r -i 's/(\(\?P<name>\[A-Za-z\]\[A-Za-z0-9_\.)/\1\\\-/' \
621 525f2979 Ilias Tsitsimpis
                /usr/share/python/debpython/pydist.py
622 525f2979 Ilias Tsitsimpis
            """
623 525f2979 Ilias Tsitsimpis
            _run(cmd, False)
624 525f2979 Ilias Tsitsimpis

625 24ccf5d3 Ilias Tsitsimpis
        # Build synnefo packages
626 24ccf5d3 Ilias Tsitsimpis
        self.logger.debug("Build synnefo packages")
627 525f2979 Ilias Tsitsimpis
        cmd = """
628 24ccf5d3 Ilias Tsitsimpis
        devflow-autopkg snapshot -b ~/synnefo_build-area --no-sign
629 525f2979 Ilias Tsitsimpis
        """
630 24ccf5d3 Ilias Tsitsimpis
        with fabric.cd("synnefo"):
631 525f2979 Ilias Tsitsimpis
            _run(cmd, True)
632 525f2979 Ilias Tsitsimpis

633 24ccf5d3 Ilias Tsitsimpis
        # Install snf-deploy package
634 525f2979 Ilias Tsitsimpis
        self.logger.debug("Install snf-deploy package")
635 525f2979 Ilias Tsitsimpis
        cmd = """
636 525f2979 Ilias Tsitsimpis
        dpkg -i snf-deploy*.deb
637 769db39e Ilias Tsitsimpis
        apt-get -f install --yes --force-yes
638 525f2979 Ilias Tsitsimpis
        """
639 24ccf5d3 Ilias Tsitsimpis
        with fabric.cd("synnefo_build-area"):
640 525f2979 Ilias Tsitsimpis
            with fabric.settings(warn_only=True):
641 525f2979 Ilias Tsitsimpis
                _run(cmd, True)
642 525f2979 Ilias Tsitsimpis

643 24ccf5d3 Ilias Tsitsimpis
        # Setup synnefo packages for snf-deploy
644 525f2979 Ilias Tsitsimpis
        self.logger.debug("Copy synnefo debs to snf-deploy packages dir")
645 525f2979 Ilias Tsitsimpis
        cmd = """
646 525f2979 Ilias Tsitsimpis
        cp ~/synnefo_build-area/*.deb /var/lib/snf-deploy/packages/
647 525f2979 Ilias Tsitsimpis
        """
648 525f2979 Ilias Tsitsimpis
        _run(cmd, False)
649 525f2979 Ilias Tsitsimpis

650 124c300e Christos Stavrakakis
    @_check_fabric
651 124c300e Christos Stavrakakis
    def build_documentation(self):
652 33ad9a0d Ilias Tsitsimpis
        """Build Synnefo documentation"""
653 124c300e Christos Stavrakakis
        self.logger.info("Build Synnefo documentation..")
654 124c300e Christos Stavrakakis
        _run("pip install -U Sphinx", False)
655 124c300e Christos Stavrakakis
        with fabric.cd("synnefo"):
656 33ad9a0d Ilias Tsitsimpis
            _run("devflow-update-version; "
657 33ad9a0d Ilias Tsitsimpis
                 "./ci/make_docs.sh synnefo_documentation", False)
658 124c300e Christos Stavrakakis

659 124c300e Christos Stavrakakis
    def fetch_documentation(self, dest=None):
660 33ad9a0d Ilias Tsitsimpis
        """Fetch Synnefo documentation"""
661 33ad9a0d Ilias Tsitsimpis
        self.logger.info("Fetch Synnefo documentation..")
662 124c300e Christos Stavrakakis
        if dest is None:
663 124c300e Christos Stavrakakis
            dest = "synnefo_documentation"
664 124c300e Christos Stavrakakis
        dest = os.path.abspath(dest)
665 124c300e Christos Stavrakakis
        if not os.path.exists(dest):
666 124c300e Christos Stavrakakis
            os.makedirs(dest)
667 124c300e Christos Stavrakakis
        self.fetch_compressed("synnefo/synnefo_documentation", dest)
668 124c300e Christos Stavrakakis
        self.logger.info("Downloaded documentation to %s" %
669 124c300e Christos Stavrakakis
                         _green(dest))
670 124c300e Christos Stavrakakis

671 525f2979 Ilias Tsitsimpis
    @_check_fabric
672 46a07468 Christos Stavrakakis
    def deploy_synnefo(self, schema=None):
673 525f2979 Ilias Tsitsimpis
        """Deploy Synnefo using snf-deploy"""
674 525f2979 Ilias Tsitsimpis
        self.logger.info("Deploy Synnefo..")
675 46a07468 Christos Stavrakakis
        if schema is None:
676 46a07468 Christos Stavrakakis
            schema = self.config.get('Global', 'schema')
677 6f8b2632 Ilias Tsitsimpis
        self.logger.debug("Will use \"%s\" schema" % schema)
678 525f2979 Ilias Tsitsimpis

679 46a07468 Christos Stavrakakis
        schema_dir = os.path.join(self.ci_dir, "schemas/%s" % schema)
680 46a07468 Christos Stavrakakis
        if not (os.path.exists(schema_dir) and os.path.isdir(schema_dir)):
681 46a07468 Christos Stavrakakis
            raise ValueError("Unknown schema: %s" % schema)
682 46a07468 Christos Stavrakakis

683 525f2979 Ilias Tsitsimpis
        self.logger.debug("Upload schema files to server")
684 4c818bb2 Ilias Tsitsimpis
        _put(os.path.join(schema_dir, "*"), "/etc/snf-deploy/")
685 525f2979 Ilias Tsitsimpis

686 525f2979 Ilias Tsitsimpis
        self.logger.debug("Change password in nodes.conf file")
687 525f2979 Ilias Tsitsimpis
        cmd = """
688 525f2979 Ilias Tsitsimpis
        sed -i 's/^password =.*/password = {0}/' /etc/snf-deploy/nodes.conf
689 525f2979 Ilias Tsitsimpis
        """.format(fabric.env.password)
690 525f2979 Ilias Tsitsimpis
        _run(cmd, False)
691 525f2979 Ilias Tsitsimpis

692 525f2979 Ilias Tsitsimpis
        self.logger.debug("Run snf-deploy")
693 525f2979 Ilias Tsitsimpis
        cmd = """
694 39dd21c2 Christos Stavrakakis
        snf-deploy --disable-colors --autoconf all
695 525f2979 Ilias Tsitsimpis
        """
696 525f2979 Ilias Tsitsimpis
        _run(cmd, True)
697 525f2979 Ilias Tsitsimpis

698 525f2979 Ilias Tsitsimpis
    @_check_fabric
699 525f2979 Ilias Tsitsimpis
    def unit_test(self):
700 525f2979 Ilias Tsitsimpis
        """Run Synnefo unit test suite"""
701 525f2979 Ilias Tsitsimpis
        self.logger.info("Run Synnefo unit test suite")
702 525f2979 Ilias Tsitsimpis
        component = self.config.get('Unit Tests', 'component')
703 525f2979 Ilias Tsitsimpis

704 525f2979 Ilias Tsitsimpis
        self.logger.debug("Install needed packages")
705 525f2979 Ilias Tsitsimpis
        cmd = """
706 525f2979 Ilias Tsitsimpis
        pip install mock
707 525f2979 Ilias Tsitsimpis
        pip install factory_boy
708 525f2979 Ilias Tsitsimpis
        """
709 525f2979 Ilias Tsitsimpis
        _run(cmd, False)
710 525f2979 Ilias Tsitsimpis

711 2255812d Ilias Tsitsimpis
        self.logger.debug("Upload tests.sh file")
712 2255812d Ilias Tsitsimpis
        unit_tests_file = os.path.join(self.ci_dir, "tests.sh")
713 4c818bb2 Ilias Tsitsimpis
        _put(unit_tests_file, ".")
714 525f2979 Ilias Tsitsimpis

715 525f2979 Ilias Tsitsimpis
        self.logger.debug("Run unit tests")
716 525f2979 Ilias Tsitsimpis
        cmd = """
717 2255812d Ilias Tsitsimpis
        bash tests.sh {0}
718 525f2979 Ilias Tsitsimpis
        """.format(component)
719 525f2979 Ilias Tsitsimpis
        _run(cmd, True)
720 525f2979 Ilias Tsitsimpis

721 525f2979 Ilias Tsitsimpis
    @_check_fabric
722 525f2979 Ilias Tsitsimpis
    def run_burnin(self):
723 525f2979 Ilias Tsitsimpis
        """Run burnin functional test suite"""
724 525f2979 Ilias Tsitsimpis
        self.logger.info("Run Burnin functional test suite")
725 525f2979 Ilias Tsitsimpis
        cmd = """
726 525f2979 Ilias Tsitsimpis
        auth_url=$(grep -e '^url =' .kamakirc | cut -d' ' -f3)
727 525f2979 Ilias Tsitsimpis
        token=$(grep -e '^token =' .kamakirc | cut -d' ' -f3)
728 525f2979 Ilias Tsitsimpis
        images_user=$(kamaki image list -l | grep owner | \
729 525f2979 Ilias Tsitsimpis
                      cut -d':' -f2 | tr -d ' ')
730 525f2979 Ilias Tsitsimpis
        snf-burnin --auth-url=$auth_url --token=$token \
731 525f2979 Ilias Tsitsimpis
            --force-flavor=2 --image-id=all \
732 525f2979 Ilias Tsitsimpis
            --system-images-user=$images_user \
733 525f2979 Ilias Tsitsimpis
            {0}
734 525f2979 Ilias Tsitsimpis
        log_folder=$(ls -1d /var/log/burnin/* | tail -n1)
735 525f2979 Ilias Tsitsimpis
        for i in $(ls $log_folder/*/details*); do
736 525f2979 Ilias Tsitsimpis
            echo -e "\\n\\n"
737 525f2979 Ilias Tsitsimpis
            echo -e "***** $i\\n"
738 525f2979 Ilias Tsitsimpis
            cat $i
739 525f2979 Ilias Tsitsimpis
        done
740 525f2979 Ilias Tsitsimpis
        """.format(self.config.get('Burnin', 'cmd_options'))
741 525f2979 Ilias Tsitsimpis
        _run(cmd, True)
742 525f2979 Ilias Tsitsimpis

743 525f2979 Ilias Tsitsimpis
    @_check_fabric
744 124c300e Christos Stavrakakis
    def fetch_compressed(self, src, dest=None):
745 33ad9a0d Ilias Tsitsimpis
        """Create a tarball and fetch it locally"""
746 124c300e Christos Stavrakakis
        self.logger.debug("Creating tarball of %s" % src)
747 124c300e Christos Stavrakakis
        basename = os.path.basename(src)
748 124c300e Christos Stavrakakis
        tar_file = basename + ".tgz"
749 124c300e Christos Stavrakakis
        cmd = "tar czf %s %s" % (tar_file, src)
750 525f2979 Ilias Tsitsimpis
        _run(cmd, False)
751 124c300e Christos Stavrakakis
        if not os.path.exists(dest):
752 124c300e Christos Stavrakakis
            os.makedirs(dest)
753 525f2979 Ilias Tsitsimpis

754 124c300e Christos Stavrakakis
        tmp_dir = tempfile.mkdtemp()
755 124c300e Christos Stavrakakis
        fabric.get(tar_file, tmp_dir)
756 525f2979 Ilias Tsitsimpis

757 124c300e Christos Stavrakakis
        dest_file = os.path.join(tmp_dir, tar_file)
758 124c300e Christos Stavrakakis
        self._check_hash_sum(dest_file, tar_file)
759 124c300e Christos Stavrakakis
        self.logger.debug("Untar packages file %s" % dest_file)
760 ee2419c5 Christos Stavrakakis
        cmd = """
761 ee2419c5 Christos Stavrakakis
        cd %s
762 124c300e Christos Stavrakakis
        tar xzf %s
763 124c300e Christos Stavrakakis
        cp -r %s/* %s
764 124c300e Christos Stavrakakis
        rm -r %s
765 124c300e Christos Stavrakakis
        """ % (tmp_dir, tar_file, src, dest, tmp_dir)
766 ee2419c5 Christos Stavrakakis
        os.system(cmd)
767 124c300e Christos Stavrakakis
        self.logger.info("Downloaded %s to %s" %
768 124c300e Christos Stavrakakis
                         (src, _green(dest)))
769 124c300e Christos Stavrakakis

770 124c300e Christos Stavrakakis
    @_check_fabric
771 124c300e Christos Stavrakakis
    def fetch_packages(self, dest=None):
772 33ad9a0d Ilias Tsitsimpis
        """Fetch Synnefo packages"""
773 124c300e Christos Stavrakakis
        if dest is None:
774 124c300e Christos Stavrakakis
            dest = self.config.get('Global', 'pkgs_dir')
775 8baa4ae7 Ilias Tsitsimpis
        dest = os.path.abspath(os.path.expanduser(dest))
776 124c300e Christos Stavrakakis
        if not os.path.exists(dest):
777 124c300e Christos Stavrakakis
            os.makedirs(dest)
778 124c300e Christos Stavrakakis
        self.fetch_compressed("synnefo_build-area", dest)
779 85bcc8da Christos Stavrakakis
        self.logger.info("Downloaded debian packages to %s" %
780 124c300e Christos Stavrakakis
                         _green(dest))
781 fe402df2 Christos Stavrakakis

782 fe402df2 Christos Stavrakakis

783 fe402df2 Christos Stavrakakis
def parse_typed_option(option, value):
784 fe402df2 Christos Stavrakakis
    try:
785 fe402df2 Christos Stavrakakis
        [type_, val] = value.strip().split(':')
786 fe402df2 Christos Stavrakakis
        if type_ not in ["id", "name"]:
787 fe402df2 Christos Stavrakakis
            raise ValueError
788 fe402df2 Christos Stavrakakis
        return type_, val
789 fe402df2 Christos Stavrakakis
    except ValueError:
790 fe402df2 Christos Stavrakakis
        msg = "Invalid %s format. Must be [id|name]:.+" % option
791 fe402df2 Christos Stavrakakis
        raise ValueError(msg)