Statistics
| Branch: | Tag: | Revision:

root / ci / utils.py @ 985442f5

History | View | Annotate | Download (30.1 kB)

1 c29ac11d Ilias Tsitsimpis
#!/usr/bin/env python
2 c29ac11d Ilias Tsitsimpis
3 c29ac11d Ilias Tsitsimpis
"""
4 c29ac11d Ilias Tsitsimpis
Synnefo ci utils module
5 c29ac11d Ilias Tsitsimpis
"""
6 c29ac11d Ilias Tsitsimpis
7 c29ac11d Ilias Tsitsimpis
import os
8 ad21ffa9 Ilias Tsitsimpis
import re
9 c29ac11d Ilias Tsitsimpis
import sys
10 c29ac11d Ilias Tsitsimpis
import time
11 c29ac11d Ilias Tsitsimpis
import logging
12 c29ac11d Ilias Tsitsimpis
import fabric.api as fabric
13 b69b55ca Christos Stavrakakis
import subprocess
14 88e6558b Christos Stavrakakis
import tempfile
15 c29ac11d Ilias Tsitsimpis
from ConfigParser import ConfigParser, DuplicateSectionError
16 c29ac11d Ilias Tsitsimpis
17 c1b1d444 Christos Stavrakakis
from kamaki.cli import config as kamaki_config
18 c29ac11d Ilias Tsitsimpis
from kamaki.clients.astakos import AstakosClient
19 c29ac11d Ilias Tsitsimpis
from kamaki.clients.cyclades import CycladesClient
20 c29ac11d Ilias Tsitsimpis
from kamaki.clients.image import ImageClient
21 ad21ffa9 Ilias Tsitsimpis
from kamaki.clients.compute import ComputeClient
22 62fcf0e5 Ilias Tsitsimpis
import filelocker
23 c29ac11d Ilias Tsitsimpis
24 6ca8f81a Christos Stavrakakis
DEFAULT_CONFIG_FILE = "new_config"
25 62f3f54f Ilias Tsitsimpis
# UUID of owner of system images
26 62f3f54f Ilias Tsitsimpis
DEFAULT_SYSTEM_IMAGES_UUID = [
27 62f3f54f Ilias Tsitsimpis
    "25ecced9-bf53-4145-91ee-cf47377e9fb2",  # production (okeanos.grnet.gr)
28 62f3f54f Ilias Tsitsimpis
    "04cbe33f-29b7-4ef1-94fb-015929e5fc06",  # testing (okeanos.io)
29 905bb10b Christos Stavrakakis
]
30 6ca8f81a Christos Stavrakakis
31 c29ac11d Ilias Tsitsimpis
32 c29ac11d Ilias Tsitsimpis
def _run(cmd, verbose):
33 c29ac11d Ilias Tsitsimpis
    """Run fabric with verbose level"""
34 c29ac11d Ilias Tsitsimpis
    if verbose:
35 c29ac11d Ilias Tsitsimpis
        args = ('running',)
36 c29ac11d Ilias Tsitsimpis
    else:
37 c29ac11d Ilias Tsitsimpis
        args = ('running', 'stdout',)
38 e2a0abb8 Ilias Tsitsimpis
    with fabric.hide(*args):  # Used * or ** magic. pylint: disable-msg=W0142
39 c29ac11d Ilias Tsitsimpis
        return fabric.run(cmd)
40 c29ac11d Ilias Tsitsimpis
41 c29ac11d Ilias Tsitsimpis
42 6868804a Ilias Tsitsimpis
def _put(local, remote):
43 6868804a Ilias Tsitsimpis
    """Run fabric put command without output"""
44 6868804a Ilias Tsitsimpis
    with fabric.quiet():
45 6868804a Ilias Tsitsimpis
        fabric.put(local, remote)
46 6868804a Ilias Tsitsimpis
47 6868804a Ilias Tsitsimpis
48 c29ac11d Ilias Tsitsimpis
def _red(msg):
49 c29ac11d Ilias Tsitsimpis
    """Red color"""
50 c29ac11d Ilias Tsitsimpis
    #return "\x1b[31m" + str(msg) + "\x1b[0m"
51 c29ac11d Ilias Tsitsimpis
    return str(msg)
52 c29ac11d Ilias Tsitsimpis
53 c29ac11d Ilias Tsitsimpis
54 c29ac11d Ilias Tsitsimpis
def _yellow(msg):
55 c29ac11d Ilias Tsitsimpis
    """Yellow color"""
56 c29ac11d Ilias Tsitsimpis
    #return "\x1b[33m" + str(msg) + "\x1b[0m"
57 c29ac11d Ilias Tsitsimpis
    return str(msg)
58 c29ac11d Ilias Tsitsimpis
59 c29ac11d Ilias Tsitsimpis
60 c29ac11d Ilias Tsitsimpis
def _green(msg):
61 c29ac11d Ilias Tsitsimpis
    """Green color"""
62 c29ac11d Ilias Tsitsimpis
    #return "\x1b[32m" + str(msg) + "\x1b[0m"
63 c29ac11d Ilias Tsitsimpis
    return str(msg)
64 c29ac11d Ilias Tsitsimpis
65 c29ac11d Ilias Tsitsimpis
66 c29ac11d Ilias Tsitsimpis
def _check_fabric(fun):
67 c29ac11d Ilias Tsitsimpis
    """Check if fabric env has been set"""
68 43a22402 Christos Stavrakakis
    def wrapper(self, *args, **kwargs):
69 c29ac11d Ilias Tsitsimpis
        """wrapper function"""
70 c29ac11d Ilias Tsitsimpis
        if not self.fabric_installed:
71 c29ac11d Ilias Tsitsimpis
            self.setup_fabric()
72 6c3cc77e Ilias Tsitsimpis
            self.fabric_installed = True
73 43a22402 Christos Stavrakakis
        return fun(self, *args, **kwargs)
74 c29ac11d Ilias Tsitsimpis
    return wrapper
75 c29ac11d Ilias Tsitsimpis
76 c29ac11d Ilias Tsitsimpis
77 c29ac11d Ilias Tsitsimpis
def _check_kamaki(fun):
78 c29ac11d Ilias Tsitsimpis
    """Check if kamaki has been initialized"""
79 43a22402 Christos Stavrakakis
    def wrapper(self, *args, **kwargs):
80 c29ac11d Ilias Tsitsimpis
        """wrapper function"""
81 c29ac11d Ilias Tsitsimpis
        if not self.kamaki_installed:
82 c29ac11d Ilias Tsitsimpis
            self.setup_kamaki()
83 6c3cc77e Ilias Tsitsimpis
            self.kamaki_installed = True
84 43a22402 Christos Stavrakakis
        return fun(self, *args, **kwargs)
85 c29ac11d Ilias Tsitsimpis
    return wrapper
86 c29ac11d Ilias Tsitsimpis
87 c29ac11d Ilias Tsitsimpis
88 c29ac11d Ilias Tsitsimpis
class _MyFormatter(logging.Formatter):
89 c29ac11d Ilias Tsitsimpis
    """Logging Formatter"""
90 c29ac11d Ilias Tsitsimpis
    def format(self, record):
91 c29ac11d Ilias Tsitsimpis
        format_orig = self._fmt
92 c29ac11d Ilias Tsitsimpis
        if record.levelno == logging.DEBUG:
93 c29ac11d Ilias Tsitsimpis
            self._fmt = "  %(msg)s"
94 c29ac11d Ilias Tsitsimpis
        elif record.levelno == logging.INFO:
95 c29ac11d Ilias Tsitsimpis
            self._fmt = "%(msg)s"
96 c29ac11d Ilias Tsitsimpis
        elif record.levelno == logging.WARNING:
97 c29ac11d Ilias Tsitsimpis
            self._fmt = _yellow("[W] %(msg)s")
98 c29ac11d Ilias Tsitsimpis
        elif record.levelno == logging.ERROR:
99 c29ac11d Ilias Tsitsimpis
            self._fmt = _red("[E] %(msg)s")
100 c29ac11d Ilias Tsitsimpis
        result = logging.Formatter.format(self, record)
101 c29ac11d Ilias Tsitsimpis
        self._fmt = format_orig
102 c29ac11d Ilias Tsitsimpis
        return result
103 c29ac11d Ilias Tsitsimpis
104 c29ac11d Ilias Tsitsimpis
105 56ee1bbc Ilias Tsitsimpis
# Too few public methods. pylint: disable-msg=R0903
106 56ee1bbc Ilias Tsitsimpis
class _InfoFilter(logging.Filter):
107 56ee1bbc Ilias Tsitsimpis
    """Logging Filter that allows DEBUG and INFO messages only"""
108 56ee1bbc Ilias Tsitsimpis
    def filter(self, rec):
109 56ee1bbc Ilias Tsitsimpis
        """The filter"""
110 56ee1bbc Ilias Tsitsimpis
        return rec.levelno in (logging.DEBUG, logging.INFO)
111 56ee1bbc Ilias Tsitsimpis
112 56ee1bbc Ilias Tsitsimpis
113 56ee1bbc Ilias Tsitsimpis
# Too many instance attributes. pylint: disable-msg=R0902
114 c29ac11d Ilias Tsitsimpis
class SynnefoCI(object):
115 c29ac11d Ilias Tsitsimpis
    """SynnefoCI python class"""
116 c29ac11d Ilias Tsitsimpis
117 79144a72 Ilias Tsitsimpis
    def __init__(self, config_file=None, build_id=None, cloud=None):
118 c29ac11d Ilias Tsitsimpis
        """ Initialize SynnefoCI python class
119 c29ac11d Ilias Tsitsimpis

120 c29ac11d Ilias Tsitsimpis
        Setup logger, local_dir, config and kamaki
121 c29ac11d Ilias Tsitsimpis
        """
122 c29ac11d Ilias Tsitsimpis
        # Setup logger
123 c29ac11d Ilias Tsitsimpis
        self.logger = logging.getLogger('synnefo-ci')
124 c29ac11d Ilias Tsitsimpis
        self.logger.setLevel(logging.DEBUG)
125 56ee1bbc Ilias Tsitsimpis
126 56ee1bbc Ilias Tsitsimpis
        handler1 = logging.StreamHandler(sys.stdout)
127 56ee1bbc Ilias Tsitsimpis
        handler1.setLevel(logging.DEBUG)
128 56ee1bbc Ilias Tsitsimpis
        handler1.addFilter(_InfoFilter())
129 56ee1bbc Ilias Tsitsimpis
        handler1.setFormatter(_MyFormatter())
130 56ee1bbc Ilias Tsitsimpis
        handler2 = logging.StreamHandler(sys.stderr)
131 56ee1bbc Ilias Tsitsimpis
        handler2.setLevel(logging.WARNING)
132 56ee1bbc Ilias Tsitsimpis
        handler2.setFormatter(_MyFormatter())
133 56ee1bbc Ilias Tsitsimpis
134 56ee1bbc Ilias Tsitsimpis
        self.logger.addHandler(handler1)
135 56ee1bbc Ilias Tsitsimpis
        self.logger.addHandler(handler2)
136 c29ac11d Ilias Tsitsimpis
137 c29ac11d Ilias Tsitsimpis
        # Get our local dir
138 c29ac11d Ilias Tsitsimpis
        self.ci_dir = os.path.dirname(os.path.abspath(__file__))
139 c29ac11d Ilias Tsitsimpis
        self.repo_dir = os.path.dirname(self.ci_dir)
140 c29ac11d Ilias Tsitsimpis
141 c29ac11d Ilias Tsitsimpis
        # Read config file
142 6ca8f81a Christos Stavrakakis
        if config_file is None:
143 6ca8f81a Christos Stavrakakis
            config_file = DEFAULT_CONFIG_FILE
144 6ca8f81a Christos Stavrakakis
        if not os.path.isabs(config_file):
145 6ca8f81a Christos Stavrakakis
            config_file = os.path.join(self.ci_dir, config_file)
146 c29ac11d Ilias Tsitsimpis
        self.config = ConfigParser()
147 c29ac11d Ilias Tsitsimpis
        self.config.optionxform = str
148 6ca8f81a Christos Stavrakakis
        self.config.read(config_file)
149 79144a72 Ilias Tsitsimpis
150 79144a72 Ilias Tsitsimpis
        # Read temporary_config file
151 62fcf0e5 Ilias Tsitsimpis
        self.temp_config_file = \
152 62fcf0e5 Ilias Tsitsimpis
            os.path.expanduser(self.config.get('Global', 'temporary_config'))
153 79144a72 Ilias Tsitsimpis
        self.temp_config = ConfigParser()
154 79144a72 Ilias Tsitsimpis
        self.temp_config.optionxform = str
155 62fcf0e5 Ilias Tsitsimpis
        self.temp_config.read(self.temp_config_file)
156 b7496c88 Ilias Tsitsimpis
        self.build_id = build_id
157 b7496c88 Ilias Tsitsimpis
        self.logger.info("Will use \"%s\" as build id" % _green(self.build_id))
158 c29ac11d Ilias Tsitsimpis
159 c1b1d444 Christos Stavrakakis
        # Set kamaki cloud
160 c1b1d444 Christos Stavrakakis
        if cloud is not None:
161 c1b1d444 Christos Stavrakakis
            self.kamaki_cloud = cloud
162 c1b1d444 Christos Stavrakakis
        elif self.config.has_option("Deployment", "kamaki_cloud"):
163 c1b1d444 Christos Stavrakakis
            kamaki_cloud = self.config.get("Deployment", "kamaki_cloud")
164 c1b1d444 Christos Stavrakakis
            if kamaki_cloud == "":
165 c1b1d444 Christos Stavrakakis
                self.kamaki_cloud = None
166 c1b1d444 Christos Stavrakakis
        else:
167 c1b1d444 Christos Stavrakakis
            self.kamaki_cloud = None
168 c1b1d444 Christos Stavrakakis
169 c29ac11d Ilias Tsitsimpis
        # Initialize variables
170 c29ac11d Ilias Tsitsimpis
        self.fabric_installed = False
171 c29ac11d Ilias Tsitsimpis
        self.kamaki_installed = False
172 c29ac11d Ilias Tsitsimpis
        self.cyclades_client = None
173 ad21ffa9 Ilias Tsitsimpis
        self.compute_client = None
174 c29ac11d Ilias Tsitsimpis
        self.image_client = None
175 c29ac11d Ilias Tsitsimpis
176 c29ac11d Ilias Tsitsimpis
    def setup_kamaki(self):
177 c29ac11d Ilias Tsitsimpis
        """Initialize kamaki
178 c29ac11d Ilias Tsitsimpis

179 ad21ffa9 Ilias Tsitsimpis
        Setup cyclades_client, image_client and compute_client
180 c29ac11d Ilias Tsitsimpis
        """
181 c1b1d444 Christos Stavrakakis
182 c1b1d444 Christos Stavrakakis
        config = kamaki_config.Config()
183 c1b1d444 Christos Stavrakakis
        if self.kamaki_cloud is None:
184 c1b1d444 Christos Stavrakakis
            self.kamaki_cloud = config.get_global("default_cloud")
185 c1b1d444 Christos Stavrakakis
186 c1b1d444 Christos Stavrakakis
        self.logger.info("Setup kamaki client, using cloud '%s'.." %
187 c1b1d444 Christos Stavrakakis
                         self.kamaki_cloud)
188 c1b1d444 Christos Stavrakakis
        auth_url = config.get_cloud(self.kamaki_cloud, "url")
189 c29ac11d Ilias Tsitsimpis
        self.logger.debug("Authentication URL is %s" % _green(auth_url))
190 c1b1d444 Christos Stavrakakis
        token = config.get_cloud(self.kamaki_cloud, "token")
191 c29ac11d Ilias Tsitsimpis
        #self.logger.debug("Token is %s" % _green(token))
192 c29ac11d Ilias Tsitsimpis
193 905bb10b Christos Stavrakakis
        self.astakos_client = AstakosClient(auth_url, token)
194 c29ac11d Ilias Tsitsimpis
195 c29ac11d Ilias Tsitsimpis
        cyclades_url = \
196 905bb10b Christos Stavrakakis
            self.astakos_client.get_service_endpoints('compute')['publicURL']
197 c29ac11d Ilias Tsitsimpis
        self.logger.debug("Cyclades API url is %s" % _green(cyclades_url))
198 c29ac11d Ilias Tsitsimpis
        self.cyclades_client = CycladesClient(cyclades_url, token)
199 c29ac11d Ilias Tsitsimpis
        self.cyclades_client.CONNECTION_RETRY_LIMIT = 2
200 c29ac11d Ilias Tsitsimpis
201 c29ac11d Ilias Tsitsimpis
        image_url = \
202 905bb10b Christos Stavrakakis
            self.astakos_client.get_service_endpoints('image')['publicURL']
203 c29ac11d Ilias Tsitsimpis
        self.logger.debug("Images API url is %s" % _green(image_url))
204 c29ac11d Ilias Tsitsimpis
        self.image_client = ImageClient(cyclades_url, token)
205 c29ac11d Ilias Tsitsimpis
        self.image_client.CONNECTION_RETRY_LIMIT = 2
206 c29ac11d Ilias Tsitsimpis
207 ad21ffa9 Ilias Tsitsimpis
        compute_url = \
208 905bb10b Christos Stavrakakis
            self.astakos_client.get_service_endpoints('compute')['publicURL']
209 ad21ffa9 Ilias Tsitsimpis
        self.logger.debug("Compute API url is %s" % _green(compute_url))
210 ad21ffa9 Ilias Tsitsimpis
        self.compute_client = ComputeClient(compute_url, token)
211 ad21ffa9 Ilias Tsitsimpis
        self.compute_client.CONNECTION_RETRY_LIMIT = 2
212 ad21ffa9 Ilias Tsitsimpis
213 c29ac11d Ilias Tsitsimpis
    def _wait_transition(self, server_id, current_status, new_status):
214 c29ac11d Ilias Tsitsimpis
        """Wait for server to go from current_status to new_status"""
215 c29ac11d Ilias Tsitsimpis
        self.logger.debug("Waiting for server to become %s" % new_status)
216 c29ac11d Ilias Tsitsimpis
        timeout = self.config.getint('Global', 'build_timeout')
217 c29ac11d Ilias Tsitsimpis
        sleep_time = 5
218 c29ac11d Ilias Tsitsimpis
        while True:
219 c29ac11d Ilias Tsitsimpis
            server = self.cyclades_client.get_server_details(server_id)
220 c29ac11d Ilias Tsitsimpis
            if server['status'] == new_status:
221 c29ac11d Ilias Tsitsimpis
                return server
222 c29ac11d Ilias Tsitsimpis
            elif timeout < 0:
223 c29ac11d Ilias Tsitsimpis
                self.logger.error(
224 c29ac11d Ilias Tsitsimpis
                    "Waiting for server to become %s timed out" % new_status)
225 c29ac11d Ilias Tsitsimpis
                self.destroy_server(False)
226 c29ac11d Ilias Tsitsimpis
                sys.exit(-1)
227 c29ac11d Ilias Tsitsimpis
            elif server['status'] == current_status:
228 c29ac11d Ilias Tsitsimpis
                # Sleep for #n secs and continue
229 c29ac11d Ilias Tsitsimpis
                timeout = timeout - sleep_time
230 c29ac11d Ilias Tsitsimpis
                time.sleep(sleep_time)
231 c29ac11d Ilias Tsitsimpis
            else:
232 c29ac11d Ilias Tsitsimpis
                self.logger.error(
233 c29ac11d Ilias Tsitsimpis
                    "Server failed with status %s" % server['status'])
234 c29ac11d Ilias Tsitsimpis
                self.destroy_server(False)
235 c29ac11d Ilias Tsitsimpis
                sys.exit(-1)
236 c29ac11d Ilias Tsitsimpis
237 c29ac11d Ilias Tsitsimpis
    @_check_kamaki
238 c29ac11d Ilias Tsitsimpis
    def destroy_server(self, wait=True):
239 c29ac11d Ilias Tsitsimpis
        """Destroy slave server"""
240 b7496c88 Ilias Tsitsimpis
        server_id = int(self.read_temp_config('server_id'))
241 c29ac11d Ilias Tsitsimpis
        self.logger.info("Destoying server with id %s " % server_id)
242 c29ac11d Ilias Tsitsimpis
        self.cyclades_client.delete_server(server_id)
243 c29ac11d Ilias Tsitsimpis
        if wait:
244 c29ac11d Ilias Tsitsimpis
            self._wait_transition(server_id, "ACTIVE", "DELETED")
245 c29ac11d Ilias Tsitsimpis
246 c29ac11d Ilias Tsitsimpis
    @_check_kamaki
247 358a19bc Ilias Tsitsimpis
    def create_server(self, image=None, flavor=None, ssh_keys=None):
248 c29ac11d Ilias Tsitsimpis
        """Create slave server"""
249 c29ac11d Ilias Tsitsimpis
        self.logger.info("Create a new server..")
250 b7496c88 Ilias Tsitsimpis
251 b7496c88 Ilias Tsitsimpis
        # Find a build_id to use
252 b7496c88 Ilias Tsitsimpis
        if self.build_id is None:
253 b7496c88 Ilias Tsitsimpis
            # If build_id is given use this, else ..
254 b7496c88 Ilias Tsitsimpis
            # Find a uniq build_id to use
255 b7496c88 Ilias Tsitsimpis
            ids = self.temp_config.sections()
256 b7496c88 Ilias Tsitsimpis
            if ids:
257 b7496c88 Ilias Tsitsimpis
                max_id = int(max(self.temp_config.sections(), key=int))
258 b7496c88 Ilias Tsitsimpis
                self.build_id = max_id + 1
259 b7496c88 Ilias Tsitsimpis
            else:
260 b7496c88 Ilias Tsitsimpis
                self.build_id = 1
261 b7496c88 Ilias Tsitsimpis
        self.logger.debug("New build id \"%s\" was created"
262 b7496c88 Ilias Tsitsimpis
                          % _green(self.build_id))
263 b7496c88 Ilias Tsitsimpis
264 b7496c88 Ilias Tsitsimpis
        # Find an image to use
265 358a19bc Ilias Tsitsimpis
        image_id = self._find_image(image)
266 b7496c88 Ilias Tsitsimpis
        # Find a flavor to use
267 358a19bc Ilias Tsitsimpis
        flavor_id = self._find_flavor(flavor)
268 358a19bc Ilias Tsitsimpis
269 358a19bc Ilias Tsitsimpis
        # Create Server
270 07c13754 Christos Stavrakakis
        server_name = self.config.get("Deployment", "server_name")
271 c29ac11d Ilias Tsitsimpis
        server = self.cyclades_client.create_server(
272 07c13754 Christos Stavrakakis
            "%s(BID: %s)" % (server_name, self.build_id),
273 43a22402 Christos Stavrakakis
            flavor_id,
274 43a22402 Christos Stavrakakis
            image_id)
275 c29ac11d Ilias Tsitsimpis
        server_id = server['id']
276 b7496c88 Ilias Tsitsimpis
        self.write_temp_config('server_id', server_id)
277 c29ac11d Ilias Tsitsimpis
        self.logger.debug("Server got id %s" % _green(server_id))
278 c29ac11d Ilias Tsitsimpis
        server_user = server['metadata']['users']
279 b7496c88 Ilias Tsitsimpis
        self.write_temp_config('server_user', server_user)
280 c29ac11d Ilias Tsitsimpis
        self.logger.debug("Server's admin user is %s" % _green(server_user))
281 c29ac11d Ilias Tsitsimpis
        server_passwd = server['adminPass']
282 b7496c88 Ilias Tsitsimpis
        self.write_temp_config('server_passwd', server_passwd)
283 c29ac11d Ilias Tsitsimpis
284 c29ac11d Ilias Tsitsimpis
        server = self._wait_transition(server_id, "BUILD", "ACTIVE")
285 c29ac11d Ilias Tsitsimpis
        self._get_server_ip_and_port(server)
286 6868804a Ilias Tsitsimpis
        self._copy_ssh_keys(ssh_keys)
287 c29ac11d Ilias Tsitsimpis
288 6c3cc77e Ilias Tsitsimpis
        # Setup Firewall
289 c29ac11d Ilias Tsitsimpis
        self.setup_fabric()
290 c29ac11d Ilias Tsitsimpis
        self.logger.info("Setup firewall")
291 6868804a Ilias Tsitsimpis
        accept_ssh_from = self.config.get('Global', 'accept_ssh_from')
292 1c75c4dd Christos Stavrakakis
        if accept_ssh_from != "":
293 1c75c4dd Christos Stavrakakis
            self.logger.debug("Block ssh except from %s" % accept_ssh_from)
294 1c75c4dd Christos Stavrakakis
            cmd = """
295 1c75c4dd Christos Stavrakakis
            local_ip=$(/sbin/ifconfig eth0 | grep 'inet addr:' | \
296 1c75c4dd Christos Stavrakakis
                cut -d':' -f2 | cut -d' ' -f1)
297 1c75c4dd Christos Stavrakakis
            iptables -A INPUT -s localhost -j ACCEPT
298 1c75c4dd Christos Stavrakakis
            iptables -A INPUT -s $local_ip -j ACCEPT
299 1c75c4dd Christos Stavrakakis
            iptables -A INPUT -s {0} -p tcp --dport 22 -j ACCEPT
300 1c75c4dd Christos Stavrakakis
            iptables -A INPUT -p tcp --dport 22 -j DROP
301 1c75c4dd Christos Stavrakakis
            """.format(accept_ssh_from)
302 1c75c4dd Christos Stavrakakis
            _run(cmd, False)
303 c29ac11d Ilias Tsitsimpis
304 6c3cc77e Ilias Tsitsimpis
        # Setup apt, download packages
305 6c3cc77e Ilias Tsitsimpis
        self.logger.debug("Setup apt. Install x2goserver and firefox")
306 6c3cc77e Ilias Tsitsimpis
        cmd = """
307 6c3cc77e Ilias Tsitsimpis
        echo 'APT::Install-Suggests "false";' >> /etc/apt/apt.conf
308 6c3cc77e Ilias Tsitsimpis
        apt-get update
309 71f2649f Ilias Tsitsimpis
        apt-get install curl --yes --force-yes
310 6c3cc77e Ilias Tsitsimpis
        echo -e "\n\n{0}" >> /etc/apt/sources.list
311 6c3cc77e Ilias Tsitsimpis
        # Synnefo repo's key
312 6c3cc77e Ilias Tsitsimpis
        curl https://dev.grnet.gr/files/apt-grnetdev.pub | apt-key add -
313 6c3cc77e Ilias Tsitsimpis
        # X2GO Key
314 6c3cc77e Ilias Tsitsimpis
        apt-key adv --recv-keys --keyserver keys.gnupg.net E1F958385BFE2B6E
315 71f2649f Ilias Tsitsimpis
        apt-get install x2go-keyring --yes --force-yes
316 6c3cc77e Ilias Tsitsimpis
        apt-get update
317 71f2649f Ilias Tsitsimpis
        apt-get install x2goserver x2goserver-xsession \
318 71f2649f Ilias Tsitsimpis
                iceweasel --yes --force-yes
319 6c3cc77e Ilias Tsitsimpis
        """.format(self.config.get('Global', 'apt_repo'))
320 6c3cc77e Ilias Tsitsimpis
        _run(cmd, False)
321 6c3cc77e Ilias Tsitsimpis
322 358a19bc Ilias Tsitsimpis
    def _find_flavor(self, flavor=None):
323 358a19bc Ilias Tsitsimpis
        """Find a suitable flavor to use
324 358a19bc Ilias Tsitsimpis

325 358a19bc Ilias Tsitsimpis
        Search by name (reg expression) or by id
326 358a19bc Ilias Tsitsimpis
        """
327 358a19bc Ilias Tsitsimpis
        # Get a list of flavors from config file
328 358a19bc Ilias Tsitsimpis
        flavors = self.config.get('Deployment', 'flavors').split(",")
329 358a19bc Ilias Tsitsimpis
        if flavor is not None:
330 a8bfeffc Ilias Tsitsimpis
            # If we have a flavor_name to use, add it to our list
331 358a19bc Ilias Tsitsimpis
            flavors.insert(0, flavor)
332 358a19bc Ilias Tsitsimpis
333 358a19bc Ilias Tsitsimpis
        list_flavors = self.compute_client.list_flavors()
334 358a19bc Ilias Tsitsimpis
        for flv in flavors:
335 905bb10b Christos Stavrakakis
            flv_type, flv_value = parse_typed_option(option="flavor",
336 905bb10b Christos Stavrakakis
                                                     value=flv)
337 358a19bc Ilias Tsitsimpis
            if flv_type == "name":
338 358a19bc Ilias Tsitsimpis
                # Filter flavors by name
339 358a19bc Ilias Tsitsimpis
                self.logger.debug(
340 358a19bc Ilias Tsitsimpis
                    "Trying to find a flavor with name \"%s\"" % flv_value)
341 358a19bc Ilias Tsitsimpis
                list_flvs = \
342 358a19bc Ilias Tsitsimpis
                    [f for f in list_flavors
343 905bb10b Christos Stavrakakis
                     if re.search(flv_value, f['name'], flags=re.I)
344 905bb10b Christos Stavrakakis
                     is not None]
345 358a19bc Ilias Tsitsimpis
            elif flv_type == "id":
346 358a19bc Ilias Tsitsimpis
                # Filter flavors by id
347 358a19bc Ilias Tsitsimpis
                self.logger.debug(
348 358a19bc Ilias Tsitsimpis
                    "Trying to find a flavor with id \"%s\"" % flv_value)
349 358a19bc Ilias Tsitsimpis
                list_flvs = \
350 358a19bc Ilias Tsitsimpis
                    [f for f in list_flavors
351 358a19bc Ilias Tsitsimpis
                     if f['id'].lower() == flv_value.lower()]
352 358a19bc Ilias Tsitsimpis
            else:
353 358a19bc Ilias Tsitsimpis
                self.logger.error("Unrecognized flavor type %s" % flv_type)
354 358a19bc Ilias Tsitsimpis
355 358a19bc Ilias Tsitsimpis
            # Check if we found one
356 358a19bc Ilias Tsitsimpis
            if list_flvs:
357 358a19bc Ilias Tsitsimpis
                self.logger.debug("Will use \"%s\" with id \"%s\""
358 358a19bc Ilias Tsitsimpis
                                  % (list_flvs[0]['name'], list_flvs[0]['id']))
359 358a19bc Ilias Tsitsimpis
                return list_flvs[0]['id']
360 a8bfeffc Ilias Tsitsimpis
361 a8bfeffc Ilias Tsitsimpis
        self.logger.error("No matching flavor found.. aborting")
362 a8bfeffc Ilias Tsitsimpis
        sys.exit(1)
363 ad21ffa9 Ilias Tsitsimpis
364 358a19bc Ilias Tsitsimpis
    def _find_image(self, image=None):
365 c29ac11d Ilias Tsitsimpis
        """Find a suitable image to use
366 c29ac11d Ilias Tsitsimpis

367 358a19bc Ilias Tsitsimpis
        In case of search by name, the image has to belong to one
368 358a19bc Ilias Tsitsimpis
        of the `DEFAULT_SYSTEM_IMAGES_UUID' users.
369 358a19bc Ilias Tsitsimpis
        In case of search by id it only has to exist.
370 c29ac11d Ilias Tsitsimpis
        """
371 358a19bc Ilias Tsitsimpis
        # Get a list of images from config file
372 358a19bc Ilias Tsitsimpis
        images = self.config.get('Deployment', 'images').split(",")
373 358a19bc Ilias Tsitsimpis
        if image is not None:
374 358a19bc Ilias Tsitsimpis
            # If we have an image from command line, add it to our list
375 358a19bc Ilias Tsitsimpis
            images.insert(0, image)
376 358a19bc Ilias Tsitsimpis
377 905bb10b Christos Stavrakakis
        auth = self.astakos_client.authenticate()
378 905bb10b Christos Stavrakakis
        user_uuid = auth["access"]["token"]["tenant"]["id"]
379 358a19bc Ilias Tsitsimpis
        list_images = self.image_client.list_public(detail=True)['images']
380 358a19bc Ilias Tsitsimpis
        for img in images:
381 6958ffc6 Christos Stavrakakis
            img_type, img_value = parse_typed_option(option="image", value=img)
382 358a19bc Ilias Tsitsimpis
            if img_type == "name":
383 358a19bc Ilias Tsitsimpis
                # Filter images by name
384 358a19bc Ilias Tsitsimpis
                self.logger.debug(
385 358a19bc Ilias Tsitsimpis
                    "Trying to find an image with name \"%s\"" % img_value)
386 905bb10b Christos Stavrakakis
                accepted_uuids = DEFAULT_SYSTEM_IMAGES_UUID + [user_uuid]
387 358a19bc Ilias Tsitsimpis
                list_imgs = \
388 905bb10b Christos Stavrakakis
                    [i for i in list_images if i['user_id'] in accepted_uuids
389 905bb10b Christos Stavrakakis
                     and
390 905bb10b Christos Stavrakakis
                     re.search(img_value, i['name'], flags=re.I) is not None]
391 358a19bc Ilias Tsitsimpis
            elif img_type == "id":
392 358a19bc Ilias Tsitsimpis
                # Filter images by id
393 358a19bc Ilias Tsitsimpis
                self.logger.debug(
394 358a19bc Ilias Tsitsimpis
                    "Trying to find an image with id \"%s\"" % img_value)
395 358a19bc Ilias Tsitsimpis
                list_imgs = \
396 358a19bc Ilias Tsitsimpis
                    [i for i in list_images
397 358a19bc Ilias Tsitsimpis
                     if i['id'].lower() == img_value.lower()]
398 358a19bc Ilias Tsitsimpis
            else:
399 358a19bc Ilias Tsitsimpis
                self.logger.error("Unrecognized image type %s" % img_type)
400 358a19bc Ilias Tsitsimpis
                sys.exit(1)
401 358a19bc Ilias Tsitsimpis
402 358a19bc Ilias Tsitsimpis
            # Check if we found one
403 358a19bc Ilias Tsitsimpis
            if list_imgs:
404 358a19bc Ilias Tsitsimpis
                self.logger.debug("Will use \"%s\" with id \"%s\""
405 358a19bc Ilias Tsitsimpis
                                  % (list_imgs[0]['name'], list_imgs[0]['id']))
406 358a19bc Ilias Tsitsimpis
                return list_imgs[0]['id']
407 358a19bc Ilias Tsitsimpis
408 358a19bc Ilias Tsitsimpis
        # We didn't found one
409 358a19bc Ilias Tsitsimpis
        self.logger.error("No matching image found.. aborting")
410 358a19bc Ilias Tsitsimpis
        sys.exit(1)
411 c29ac11d Ilias Tsitsimpis
412 c29ac11d Ilias Tsitsimpis
    def _get_server_ip_and_port(self, server):
413 c29ac11d Ilias Tsitsimpis
        """Compute server's IPv4 and ssh port number"""
414 c29ac11d Ilias Tsitsimpis
        self.logger.info("Get server connection details..")
415 c29ac11d Ilias Tsitsimpis
        server_ip = server['attachments'][0]['ipv4']
416 705f70a9 Christos Stavrakakis
        if (".okeanos.io" in self.cyclades_client.base_url or
417 705f70a9 Christos Stavrakakis
           ".demo.synnefo.org" in self.cyclades_client.base_url):
418 c29ac11d Ilias Tsitsimpis
            tmp1 = int(server_ip.split(".")[2])
419 c29ac11d Ilias Tsitsimpis
            tmp2 = int(server_ip.split(".")[3])
420 c29ac11d Ilias Tsitsimpis
            server_ip = "gate.okeanos.io"
421 c29ac11d Ilias Tsitsimpis
            server_port = 10000 + tmp1 * 256 + tmp2
422 c29ac11d Ilias Tsitsimpis
        else:
423 c29ac11d Ilias Tsitsimpis
            server_port = 22
424 b7496c88 Ilias Tsitsimpis
        self.write_temp_config('server_ip', server_ip)
425 c29ac11d Ilias Tsitsimpis
        self.logger.debug("Server's IPv4 is %s" % _green(server_ip))
426 b7496c88 Ilias Tsitsimpis
        self.write_temp_config('server_port', server_port)
427 c29ac11d Ilias Tsitsimpis
        self.logger.debug("Server's ssh port is %s" % _green(server_port))
428 07710fc3 Ilias Tsitsimpis
        self.logger.debug("Access server using \"ssh -X -p %s %s@%s\"" %
429 905bb10b Christos Stavrakakis
                          (server_port, server['metadata']['users'],
430 905bb10b Christos Stavrakakis
                           server_ip))
431 c29ac11d Ilias Tsitsimpis
432 e480f0ce Christos Stavrakakis
    @_check_fabric
433 6868804a Ilias Tsitsimpis
    def _copy_ssh_keys(self, ssh_keys):
434 e2a0abb8 Ilias Tsitsimpis
        """Upload/Install ssh keys to server"""
435 4b743650 Ilias Tsitsimpis
        self.logger.debug("Check for authentication keys to use")
436 6868804a Ilias Tsitsimpis
        if ssh_keys is None:
437 6868804a Ilias Tsitsimpis
            ssh_keys = self.config.get("Deployment", "ssh_keys")
438 6868804a Ilias Tsitsimpis
439 4b743650 Ilias Tsitsimpis
        if ssh_keys != "":
440 6058ae72 Ilias Tsitsimpis
            ssh_keys = os.path.expanduser(ssh_keys)
441 905bb10b Christos Stavrakakis
            self.logger.debug("Will use %s authentication keys file" %
442 905bb10b Christos Stavrakakis
                              ssh_keys)
443 e480f0ce Christos Stavrakakis
            keyfile = '/tmp/%s.pub' % fabric.env.user
444 e480f0ce Christos Stavrakakis
            _run('mkdir -p ~/.ssh && chmod 700 ~/.ssh', False)
445 4b743650 Ilias Tsitsimpis
            if ssh_keys.startswith("http://") or \
446 4b743650 Ilias Tsitsimpis
                    ssh_keys.startswith("https://") or \
447 4b743650 Ilias Tsitsimpis
                    ssh_keys.startswith("ftp://"):
448 4b743650 Ilias Tsitsimpis
                cmd = """
449 4b743650 Ilias Tsitsimpis
                apt-get update
450 71f2649f Ilias Tsitsimpis
                apt-get install wget --yes --force-yes
451 4b743650 Ilias Tsitsimpis
                wget {0} -O {1} --no-check-certificate
452 4b743650 Ilias Tsitsimpis
                """.format(ssh_keys, keyfile)
453 4b743650 Ilias Tsitsimpis
                _run(cmd, False)
454 4b743650 Ilias Tsitsimpis
            elif os.path.exists(ssh_keys):
455 4b743650 Ilias Tsitsimpis
                _put(ssh_keys, keyfile)
456 4b743650 Ilias Tsitsimpis
            else:
457 4b743650 Ilias Tsitsimpis
                self.logger.debug("No ssh keys found")
458 6058ae72 Ilias Tsitsimpis
                return
459 e480f0ce Christos Stavrakakis
            _run('cat %s >> ~/.ssh/authorized_keys' % keyfile, False)
460 e480f0ce Christos Stavrakakis
            _run('rm %s' % keyfile, False)
461 e480f0ce Christos Stavrakakis
            self.logger.debug("Uploaded ssh authorized keys")
462 e480f0ce Christos Stavrakakis
        else:
463 e480f0ce Christos Stavrakakis
            self.logger.debug("No ssh keys found")
464 e480f0ce Christos Stavrakakis
465 b7496c88 Ilias Tsitsimpis
    def write_temp_config(self, option, value):
466 c29ac11d Ilias Tsitsimpis
        """Write changes back to config file"""
467 62fcf0e5 Ilias Tsitsimpis
        # Acquire the lock to write to temp_config_file
468 62fcf0e5 Ilias Tsitsimpis
        with filelocker.lock("%s.lock" % self.temp_config_file,
469 62fcf0e5 Ilias Tsitsimpis
                             filelocker.LOCK_EX):
470 62fcf0e5 Ilias Tsitsimpis
471 62fcf0e5 Ilias Tsitsimpis
            # Read temp_config again to get any new entries
472 62fcf0e5 Ilias Tsitsimpis
            self.temp_config.read(self.temp_config_file)
473 62fcf0e5 Ilias Tsitsimpis
474 62fcf0e5 Ilias Tsitsimpis
            # If build_id section doesn't exist create a new one
475 62fcf0e5 Ilias Tsitsimpis
            try:
476 62fcf0e5 Ilias Tsitsimpis
                self.temp_config.add_section(str(self.build_id))
477 62fcf0e5 Ilias Tsitsimpis
                creation_time = \
478 62fcf0e5 Ilias Tsitsimpis
                    time.strftime("%a, %d %b %Y %X", time.localtime())
479 62fcf0e5 Ilias Tsitsimpis
                self.temp_config.set(str(self.build_id),
480 62fcf0e5 Ilias Tsitsimpis
                                     "created", str(creation_time))
481 62fcf0e5 Ilias Tsitsimpis
            except DuplicateSectionError:
482 62fcf0e5 Ilias Tsitsimpis
                pass
483 62fcf0e5 Ilias Tsitsimpis
            self.temp_config.set(str(self.build_id), option, str(value))
484 62fcf0e5 Ilias Tsitsimpis
            curr_time = time.strftime("%a, %d %b %Y %X", time.localtime())
485 62fcf0e5 Ilias Tsitsimpis
            self.temp_config.set(str(self.build_id), "modified", curr_time)
486 62fcf0e5 Ilias Tsitsimpis
            with open(self.temp_config_file, 'wb') as tcf:
487 62fcf0e5 Ilias Tsitsimpis
                self.temp_config.write(tcf)
488 c29ac11d Ilias Tsitsimpis
489 b7496c88 Ilias Tsitsimpis
    def read_temp_config(self, option):
490 b7496c88 Ilias Tsitsimpis
        """Read from temporary_config file"""
491 b7496c88 Ilias Tsitsimpis
        # If build_id is None use the latest one
492 b7496c88 Ilias Tsitsimpis
        if self.build_id is None:
493 b7496c88 Ilias Tsitsimpis
            ids = self.temp_config.sections()
494 b7496c88 Ilias Tsitsimpis
            if ids:
495 b7496c88 Ilias Tsitsimpis
                self.build_id = int(ids[-1])
496 b7496c88 Ilias Tsitsimpis
            else:
497 b7496c88 Ilias Tsitsimpis
                self.logger.error("No sections in temporary config file")
498 b7496c88 Ilias Tsitsimpis
                sys.exit(1)
499 b7496c88 Ilias Tsitsimpis
            self.logger.debug("Will use \"%s\" as build id"
500 b7496c88 Ilias Tsitsimpis
                              % _green(self.build_id))
501 b7496c88 Ilias Tsitsimpis
        # Read specified option
502 b7496c88 Ilias Tsitsimpis
        return self.temp_config.get(str(self.build_id), option)
503 b7496c88 Ilias Tsitsimpis
504 c29ac11d Ilias Tsitsimpis
    def setup_fabric(self):
505 c29ac11d Ilias Tsitsimpis
        """Setup fabric environment"""
506 c29ac11d Ilias Tsitsimpis
        self.logger.info("Setup fabric parameters..")
507 b7496c88 Ilias Tsitsimpis
        fabric.env.user = self.read_temp_config('server_user')
508 b7496c88 Ilias Tsitsimpis
        fabric.env.host_string = self.read_temp_config('server_ip')
509 b7496c88 Ilias Tsitsimpis
        fabric.env.port = int(self.read_temp_config('server_port'))
510 b7496c88 Ilias Tsitsimpis
        fabric.env.password = self.read_temp_config('server_passwd')
511 c29ac11d Ilias Tsitsimpis
        fabric.env.connection_attempts = 10
512 c29ac11d Ilias Tsitsimpis
        fabric.env.shell = "/bin/bash -c"
513 c29ac11d Ilias Tsitsimpis
        fabric.env.disable_known_hosts = True
514 c29ac11d Ilias Tsitsimpis
        fabric.env.output_prefix = None
515 c29ac11d Ilias Tsitsimpis
516 c29ac11d Ilias Tsitsimpis
    def _check_hash_sum(self, localfile, remotefile):
517 c29ac11d Ilias Tsitsimpis
        """Check hash sums of two files"""
518 c29ac11d Ilias Tsitsimpis
        self.logger.debug("Check hash sum for local file %s" % localfile)
519 c29ac11d Ilias Tsitsimpis
        hash1 = os.popen("sha256sum %s" % localfile).read().split(' ')[0]
520 c29ac11d Ilias Tsitsimpis
        self.logger.debug("Local file has sha256 hash %s" % hash1)
521 c29ac11d Ilias Tsitsimpis
        self.logger.debug("Check hash sum for remote file %s" % remotefile)
522 c29ac11d Ilias Tsitsimpis
        hash2 = _run("sha256sum %s" % remotefile, False)
523 c29ac11d Ilias Tsitsimpis
        hash2 = hash2.split(' ')[0]
524 c29ac11d Ilias Tsitsimpis
        self.logger.debug("Remote file has sha256 hash %s" % hash2)
525 c29ac11d Ilias Tsitsimpis
        if hash1 != hash2:
526 c29ac11d Ilias Tsitsimpis
            self.logger.error("Hashes differ.. aborting")
527 c29ac11d Ilias Tsitsimpis
            sys.exit(-1)
528 c29ac11d Ilias Tsitsimpis
529 c29ac11d Ilias Tsitsimpis
    @_check_fabric
530 5323404f Ilias Tsitsimpis
    def clone_repo(self, local_repo=False):
531 c29ac11d Ilias Tsitsimpis
        """Clone Synnefo repo from slave server"""
532 c29ac11d Ilias Tsitsimpis
        self.logger.info("Configure repositories on remote server..")
533 6c3cc77e Ilias Tsitsimpis
        self.logger.debug("Install/Setup git")
534 c29ac11d Ilias Tsitsimpis
        cmd = """
535 71f2649f Ilias Tsitsimpis
        apt-get install git --yes --force-yes
536 6c3cc77e Ilias Tsitsimpis
        git config --global user.name {0}
537 6c3cc77e Ilias Tsitsimpis
        git config --global user.email {1}
538 6c3cc77e Ilias Tsitsimpis
        """.format(self.config.get('Global', 'git_config_name'),
539 c29ac11d Ilias Tsitsimpis
                   self.config.get('Global', 'git_config_mail'))
540 c29ac11d Ilias Tsitsimpis
        _run(cmd, False)
541 c29ac11d Ilias Tsitsimpis
542 5323404f Ilias Tsitsimpis
        # Find synnefo_repo and synnefo_branch to use
543 c29ac11d Ilias Tsitsimpis
        synnefo_repo = self.config.get('Global', 'synnefo_repo')
544 b69b55ca Christos Stavrakakis
        synnefo_branch = self.config.get("Global", "synnefo_branch")
545 b69b55ca Christos Stavrakakis
        if synnefo_branch == "":
546 e2a0abb8 Ilias Tsitsimpis
            synnefo_branch = \
547 e2a0abb8 Ilias Tsitsimpis
                subprocess.Popen(
548 e2a0abb8 Ilias Tsitsimpis
                    ["git", "rev-parse", "--abbrev-ref", "HEAD"],
549 b69b55ca Christos Stavrakakis
                    stdout=subprocess.PIPE).communicate()[0].strip()
550 b69b55ca Christos Stavrakakis
            if synnefo_branch == "HEAD":
551 b69b55ca Christos Stavrakakis
                synnefo_branch = \
552 e2a0abb8 Ilias Tsitsimpis
                    subprocess.Popen(
553 e2a0abb8 Ilias Tsitsimpis
                        ["git", "rev-parse", "--short", "HEAD"],
554 b69b55ca Christos Stavrakakis
                        stdout=subprocess.PIPE).communicate()[0].strip()
555 b69b55ca Christos Stavrakakis
        self.logger.info("Will use branch %s" % synnefo_branch)
556 5323404f Ilias Tsitsimpis
557 5323404f Ilias Tsitsimpis
        if local_repo or synnefo_branch == "":
558 5323404f Ilias Tsitsimpis
            # Use local_repo
559 5323404f Ilias Tsitsimpis
            self.logger.debug("Push local repo to server")
560 5323404f Ilias Tsitsimpis
            # Firstly create the remote repo
561 5323404f Ilias Tsitsimpis
            _run("git init synnefo", False)
562 5323404f Ilias Tsitsimpis
            # Then push our local repo over ssh
563 5323404f Ilias Tsitsimpis
            # We have to pass some arguments to ssh command
564 5323404f Ilias Tsitsimpis
            # namely to disable host checking.
565 5323404f Ilias Tsitsimpis
            (temp_ssh_file_handle, temp_ssh_file) = tempfile.mkstemp()
566 5323404f Ilias Tsitsimpis
            os.close(temp_ssh_file_handle)
567 62fcf0e5 Ilias Tsitsimpis
            # XXX: git push doesn't read the password
568 5323404f Ilias Tsitsimpis
            cmd = """
569 5323404f Ilias Tsitsimpis
            echo 'exec ssh -o "StrictHostKeyChecking no" \
570 5323404f Ilias Tsitsimpis
                           -o "UserKnownHostsFile /dev/null" \
571 5323404f Ilias Tsitsimpis
                           -q "$@"' > {4}
572 5323404f Ilias Tsitsimpis
            chmod u+x {4}
573 5323404f Ilias Tsitsimpis
            export GIT_SSH="{4}"
574 5323404f Ilias Tsitsimpis
            echo "{0}" | git push --mirror ssh://{1}@{2}:{3}/~/synnefo
575 5323404f Ilias Tsitsimpis
            rm -f {4}
576 5323404f Ilias Tsitsimpis
            """.format(fabric.env.password,
577 5323404f Ilias Tsitsimpis
                       fabric.env.user,
578 5323404f Ilias Tsitsimpis
                       fabric.env.host_string,
579 5323404f Ilias Tsitsimpis
                       fabric.env.port,
580 5323404f Ilias Tsitsimpis
                       temp_ssh_file)
581 5323404f Ilias Tsitsimpis
            os.system(cmd)
582 5323404f Ilias Tsitsimpis
        else:
583 5323404f Ilias Tsitsimpis
            # Clone Synnefo from remote repo
584 5323404f Ilias Tsitsimpis
            # Currently clonning synnefo can fail unexpectedly
585 5323404f Ilias Tsitsimpis
            cloned = False
586 5323404f Ilias Tsitsimpis
            for i in range(10):
587 5323404f Ilias Tsitsimpis
                self.logger.debug("Clone synnefo from %s" % synnefo_repo)
588 5323404f Ilias Tsitsimpis
                try:
589 5323404f Ilias Tsitsimpis
                    _run("git clone %s synnefo" % synnefo_repo, False)
590 5323404f Ilias Tsitsimpis
                    cloned = True
591 5323404f Ilias Tsitsimpis
                    break
592 5323404f Ilias Tsitsimpis
                except BaseException:
593 5323404f Ilias Tsitsimpis
                    self.logger.warning(
594 5323404f Ilias Tsitsimpis
                        "Clonning synnefo failed.. retrying %s" % i)
595 5323404f Ilias Tsitsimpis
            if not cloned:
596 5323404f Ilias Tsitsimpis
                self.logger.error("Can not clone Synnefo repo.")
597 5323404f Ilias Tsitsimpis
                sys.exit(-1)
598 5323404f Ilias Tsitsimpis
599 5323404f Ilias Tsitsimpis
        # Checkout the desired synnefo_branch
600 07710fc3 Ilias Tsitsimpis
        self.logger.debug("Checkout \"%s\" branch/commit" % synnefo_branch)
601 e2a0abb8 Ilias Tsitsimpis
        cmd = """
602 b69b55ca Christos Stavrakakis
        cd synnefo
603 e2a0abb8 Ilias Tsitsimpis
        for branch in `git branch -a | grep remotes | \
604 e2a0abb8 Ilias Tsitsimpis
                       grep -v HEAD | grep -v master`; do
605 b69b55ca Christos Stavrakakis
            git branch --track ${branch##*/} $branch
606 b69b55ca Christos Stavrakakis
        done
607 b69b55ca Christos Stavrakakis
        git checkout %s
608 b69b55ca Christos Stavrakakis
        """ % (synnefo_branch)
609 b69b55ca Christos Stavrakakis
        _run(cmd, False)
610 b69b55ca Christos Stavrakakis
611 c29ac11d Ilias Tsitsimpis
    @_check_fabric
612 c29ac11d Ilias Tsitsimpis
    def build_synnefo(self):
613 c29ac11d Ilias Tsitsimpis
        """Build Synnefo packages"""
614 c29ac11d Ilias Tsitsimpis
        self.logger.info("Build Synnefo packages..")
615 c29ac11d Ilias Tsitsimpis
        self.logger.debug("Install development packages")
616 c29ac11d Ilias Tsitsimpis
        cmd = """
617 c29ac11d Ilias Tsitsimpis
        apt-get update
618 c29ac11d Ilias Tsitsimpis
        apt-get install zlib1g-dev dpkg-dev debhelper git-buildpackage \
619 71f2649f Ilias Tsitsimpis
                python-dev python-all python-pip --yes --force-yes
620 c29ac11d Ilias Tsitsimpis
        pip install devflow
621 c29ac11d Ilias Tsitsimpis
        """
622 c29ac11d Ilias Tsitsimpis
        _run(cmd, False)
623 c29ac11d Ilias Tsitsimpis
624 e908a902 Christos Stavrakakis
        if self.config.get('Global', 'patch_pydist') == "True":
625 c29ac11d Ilias Tsitsimpis
            self.logger.debug("Patch pydist.py module")
626 c29ac11d Ilias Tsitsimpis
            cmd = r"""
627 c29ac11d Ilias Tsitsimpis
            sed -r -i 's/(\(\?P<name>\[A-Za-z\]\[A-Za-z0-9_\.)/\1\\\-/' \
628 c29ac11d Ilias Tsitsimpis
                /usr/share/python/debpython/pydist.py
629 c29ac11d Ilias Tsitsimpis
            """
630 c29ac11d Ilias Tsitsimpis
            _run(cmd, False)
631 c29ac11d Ilias Tsitsimpis

632 1c230fd8 Ilias Tsitsimpis
        # Build synnefo packages
633 1c230fd8 Ilias Tsitsimpis
        self.logger.debug("Build synnefo packages")
634 c29ac11d Ilias Tsitsimpis
        cmd = """
635 1c230fd8 Ilias Tsitsimpis
        devflow-autopkg snapshot -b ~/synnefo_build-area --no-sign
636 c29ac11d Ilias Tsitsimpis
        """
637 1c230fd8 Ilias Tsitsimpis
        with fabric.cd("synnefo"):
638 c29ac11d Ilias Tsitsimpis
            _run(cmd, True)
639 c29ac11d Ilias Tsitsimpis

640 1c230fd8 Ilias Tsitsimpis
        # Install snf-deploy package
641 c29ac11d Ilias Tsitsimpis
        self.logger.debug("Install snf-deploy package")
642 c29ac11d Ilias Tsitsimpis
        cmd = """
643 c29ac11d Ilias Tsitsimpis
        dpkg -i snf-deploy*.deb
644 71f2649f Ilias Tsitsimpis
        apt-get -f install --yes --force-yes
645 c29ac11d Ilias Tsitsimpis
        """
646 1c230fd8 Ilias Tsitsimpis
        with fabric.cd("synnefo_build-area"):
647 c29ac11d Ilias Tsitsimpis
            with fabric.settings(warn_only=True):
648 c29ac11d Ilias Tsitsimpis
                _run(cmd, True)
649 c29ac11d Ilias Tsitsimpis

650 1c230fd8 Ilias Tsitsimpis
        # Setup synnefo packages for snf-deploy
651 c29ac11d Ilias Tsitsimpis
        self.logger.debug("Copy synnefo debs to snf-deploy packages dir")
652 c29ac11d Ilias Tsitsimpis
        cmd = """
653 c29ac11d Ilias Tsitsimpis
        cp ~/synnefo_build-area/*.deb /var/lib/snf-deploy/packages/
654 c29ac11d Ilias Tsitsimpis
        """
655 c29ac11d Ilias Tsitsimpis
        _run(cmd, False)
656 c29ac11d Ilias Tsitsimpis

657 88e6558b Christos Stavrakakis
    @_check_fabric
658 88e6558b Christos Stavrakakis
    def build_documentation(self):
659 e2a0abb8 Ilias Tsitsimpis
        """Build Synnefo documentation"""
660 88e6558b Christos Stavrakakis
        self.logger.info("Build Synnefo documentation..")
661 88e6558b Christos Stavrakakis
        _run("pip install -U Sphinx", False)
662 88e6558b Christos Stavrakakis
        with fabric.cd("synnefo"):
663 e2a0abb8 Ilias Tsitsimpis
            _run("devflow-update-version; "
664 e2a0abb8 Ilias Tsitsimpis
                 "./ci/make_docs.sh synnefo_documentation", False)
665 88e6558b Christos Stavrakakis

666 88e6558b Christos Stavrakakis
    def fetch_documentation(self, dest=None):
667 e2a0abb8 Ilias Tsitsimpis
        """Fetch Synnefo documentation"""
668 e2a0abb8 Ilias Tsitsimpis
        self.logger.info("Fetch Synnefo documentation..")
669 88e6558b Christos Stavrakakis
        if dest is None:
670 88e6558b Christos Stavrakakis
            dest = "synnefo_documentation"
671 88e6558b Christos Stavrakakis
        dest = os.path.abspath(dest)
672 88e6558b Christos Stavrakakis
        if not os.path.exists(dest):
673 88e6558b Christos Stavrakakis
            os.makedirs(dest)
674 88e6558b Christos Stavrakakis
        self.fetch_compressed("synnefo/synnefo_documentation", dest)
675 88e6558b Christos Stavrakakis
        self.logger.info("Downloaded documentation to %s" %
676 88e6558b Christos Stavrakakis
                         _green(dest))
677 88e6558b Christos Stavrakakis

678 c29ac11d Ilias Tsitsimpis
    @_check_fabric
679 ec8bc030 Christos Stavrakakis
    def deploy_synnefo(self, schema=None):
680 c29ac11d Ilias Tsitsimpis
        """Deploy Synnefo using snf-deploy"""
681 c29ac11d Ilias Tsitsimpis
        self.logger.info("Deploy Synnefo..")
682 ec8bc030 Christos Stavrakakis
        if schema is None:
683 ec8bc030 Christos Stavrakakis
            schema = self.config.get('Global', 'schema')
684 358a19bc Ilias Tsitsimpis
        self.logger.debug("Will use \"%s\" schema" % schema)
685 c29ac11d Ilias Tsitsimpis

686 ec8bc030 Christos Stavrakakis
        schema_dir = os.path.join(self.ci_dir, "schemas/%s" % schema)
687 ec8bc030 Christos Stavrakakis
        if not (os.path.exists(schema_dir) and os.path.isdir(schema_dir)):
688 ec8bc030 Christos Stavrakakis
            raise ValueError("Unknown schema: %s" % schema)
689 ec8bc030 Christos Stavrakakis

690 c29ac11d Ilias Tsitsimpis
        self.logger.debug("Upload schema files to server")
691 6868804a Ilias Tsitsimpis
        _put(os.path.join(schema_dir, "*"), "/etc/snf-deploy/")
692 c29ac11d Ilias Tsitsimpis

693 c29ac11d Ilias Tsitsimpis
        self.logger.debug("Change password in nodes.conf file")
694 c29ac11d Ilias Tsitsimpis
        cmd = """
695 c29ac11d Ilias Tsitsimpis
        sed -i 's/^password =.*/password = {0}/' /etc/snf-deploy/nodes.conf
696 c29ac11d Ilias Tsitsimpis
        """.format(fabric.env.password)
697 c29ac11d Ilias Tsitsimpis
        _run(cmd, False)
698 c29ac11d Ilias Tsitsimpis

699 c29ac11d Ilias Tsitsimpis
        self.logger.debug("Run snf-deploy")
700 c29ac11d Ilias Tsitsimpis
        cmd = """
701 985442f5 Ilias Tsitsimpis
        snf-deploy keygen --force
702 f82ce32a Christos Stavrakakis
        snf-deploy --disable-colors --autoconf all
703 c29ac11d Ilias Tsitsimpis
        """
704 c29ac11d Ilias Tsitsimpis
        _run(cmd, True)
705 c29ac11d Ilias Tsitsimpis

706 c29ac11d Ilias Tsitsimpis
    @_check_fabric
707 c29ac11d Ilias Tsitsimpis
    def unit_test(self):
708 c29ac11d Ilias Tsitsimpis
        """Run Synnefo unit test suite"""
709 c29ac11d Ilias Tsitsimpis
        self.logger.info("Run Synnefo unit test suite")
710 c29ac11d Ilias Tsitsimpis
        component = self.config.get('Unit Tests', 'component')
711 c29ac11d Ilias Tsitsimpis

712 c29ac11d Ilias Tsitsimpis
        self.logger.debug("Install needed packages")
713 c29ac11d Ilias Tsitsimpis
        cmd = """
714 c29ac11d Ilias Tsitsimpis
        pip install mock
715 c29ac11d Ilias Tsitsimpis
        pip install factory_boy
716 c29ac11d Ilias Tsitsimpis
        """
717 c29ac11d Ilias Tsitsimpis
        _run(cmd, False)
718 c29ac11d Ilias Tsitsimpis

719 861b437e Ilias Tsitsimpis
        self.logger.debug("Upload tests.sh file")
720 861b437e Ilias Tsitsimpis
        unit_tests_file = os.path.join(self.ci_dir, "tests.sh")
721 6868804a Ilias Tsitsimpis
        _put(unit_tests_file, ".")
722 c29ac11d Ilias Tsitsimpis

723 c29ac11d Ilias Tsitsimpis
        self.logger.debug("Run unit tests")
724 c29ac11d Ilias Tsitsimpis
        cmd = """
725 861b437e Ilias Tsitsimpis
        bash tests.sh {0}
726 c29ac11d Ilias Tsitsimpis
        """.format(component)
727 c29ac11d Ilias Tsitsimpis
        _run(cmd, True)
728 c29ac11d Ilias Tsitsimpis

729 c29ac11d Ilias Tsitsimpis
    @_check_fabric
730 c29ac11d Ilias Tsitsimpis
    def run_burnin(self):
731 c29ac11d Ilias Tsitsimpis
        """Run burnin functional test suite"""
732 c29ac11d Ilias Tsitsimpis
        self.logger.info("Run Burnin functional test suite")
733 c29ac11d Ilias Tsitsimpis
        cmd = """
734 c29ac11d Ilias Tsitsimpis
        auth_url=$(grep -e '^url =' .kamakirc | cut -d' ' -f3)
735 c29ac11d Ilias Tsitsimpis
        token=$(grep -e '^token =' .kamakirc | cut -d' ' -f3)
736 c29ac11d Ilias Tsitsimpis
        images_user=$(kamaki image list -l | grep owner | \
737 c29ac11d Ilias Tsitsimpis
                      cut -d':' -f2 | tr -d ' ')
738 c29ac11d Ilias Tsitsimpis
        snf-burnin --auth-url=$auth_url --token=$token \
739 c29ac11d Ilias Tsitsimpis
            --force-flavor=2 --image-id=all \
740 c29ac11d Ilias Tsitsimpis
            --system-images-user=$images_user \
741 c29ac11d Ilias Tsitsimpis
            {0}
742 281c7634 Ilias Tsitsimpis
        BurninExitStatus=$?
743 c29ac11d Ilias Tsitsimpis
        log_folder=$(ls -1d /var/log/burnin/* | tail -n1)
744 c29ac11d Ilias Tsitsimpis
        for i in $(ls $log_folder/*/details*); do
745 c29ac11d Ilias Tsitsimpis
            echo -e "\\n\\n"
746 c29ac11d Ilias Tsitsimpis
            echo -e "***** $i\\n"
747 c29ac11d Ilias Tsitsimpis
            cat $i
748 c29ac11d Ilias Tsitsimpis
        done
749 281c7634 Ilias Tsitsimpis
        exit $BurninExitStatus
750 c29ac11d Ilias Tsitsimpis
        """.format(self.config.get('Burnin', 'cmd_options'))
751 c29ac11d Ilias Tsitsimpis
        _run(cmd, True)
752 c29ac11d Ilias Tsitsimpis

753 c29ac11d Ilias Tsitsimpis
    @_check_fabric
754 88e6558b Christos Stavrakakis
    def fetch_compressed(self, src, dest=None):
755 e2a0abb8 Ilias Tsitsimpis
        """Create a tarball and fetch it locally"""
756 88e6558b Christos Stavrakakis
        self.logger.debug("Creating tarball of %s" % src)
757 88e6558b Christos Stavrakakis
        basename = os.path.basename(src)
758 88e6558b Christos Stavrakakis
        tar_file = basename + ".tgz"
759 88e6558b Christos Stavrakakis
        cmd = "tar czf %s %s" % (tar_file, src)
760 c29ac11d Ilias Tsitsimpis
        _run(cmd, False)
761 88e6558b Christos Stavrakakis
        if not os.path.exists(dest):
762 88e6558b Christos Stavrakakis
            os.makedirs(dest)
763 c29ac11d Ilias Tsitsimpis

764 88e6558b Christos Stavrakakis
        tmp_dir = tempfile.mkdtemp()
765 88e6558b Christos Stavrakakis
        fabric.get(tar_file, tmp_dir)
766 c29ac11d Ilias Tsitsimpis

767 88e6558b Christos Stavrakakis
        dest_file = os.path.join(tmp_dir, tar_file)
768 88e6558b Christos Stavrakakis
        self._check_hash_sum(dest_file, tar_file)
769 88e6558b Christos Stavrakakis
        self.logger.debug("Untar packages file %s" % dest_file)
770 e2dc37b2 Christos Stavrakakis
        cmd = """
771 e2dc37b2 Christos Stavrakakis
        cd %s
772 88e6558b Christos Stavrakakis
        tar xzf %s
773 88e6558b Christos Stavrakakis
        cp -r %s/* %s
774 88e6558b Christos Stavrakakis
        rm -r %s
775 88e6558b Christos Stavrakakis
        """ % (tmp_dir, tar_file, src, dest, tmp_dir)
776 e2dc37b2 Christos Stavrakakis
        os.system(cmd)
777 88e6558b Christos Stavrakakis
        self.logger.info("Downloaded %s to %s" %
778 88e6558b Christos Stavrakakis
                         (src, _green(dest)))
779 88e6558b Christos Stavrakakis

780 88e6558b Christos Stavrakakis
    @_check_fabric
781 88e6558b Christos Stavrakakis
    def fetch_packages(self, dest=None):
782 e2a0abb8 Ilias Tsitsimpis
        """Fetch Synnefo packages"""
783 88e6558b Christos Stavrakakis
        if dest is None:
784 88e6558b Christos Stavrakakis
            dest = self.config.get('Global', 'pkgs_dir')
785 6058ae72 Ilias Tsitsimpis
        dest = os.path.abspath(os.path.expanduser(dest))
786 88e6558b Christos Stavrakakis
        if not os.path.exists(dest):
787 88e6558b Christos Stavrakakis
            os.makedirs(dest)
788 88e6558b Christos Stavrakakis
        self.fetch_compressed("synnefo_build-area", dest)
789 464e58e9 Christos Stavrakakis
        self.logger.info("Downloaded debian packages to %s" %
790 88e6558b Christos Stavrakakis
                         _green(dest))
791 6958ffc6 Christos Stavrakakis

792 6958ffc6 Christos Stavrakakis

793 6958ffc6 Christos Stavrakakis
def parse_typed_option(option, value):
794 6958ffc6 Christos Stavrakakis
    try:
795 6958ffc6 Christos Stavrakakis
        [type_, val] = value.strip().split(':')
796 6958ffc6 Christos Stavrakakis
        if type_ not in ["id", "name"]:
797 6958ffc6 Christos Stavrakakis
            raise ValueError
798 6958ffc6 Christos Stavrakakis
        return type_, val
799 6958ffc6 Christos Stavrakakis
    except ValueError:
800 6958ffc6 Christos Stavrakakis
        msg = "Invalid %s format. Must be [id|name]:.+" % option
801 6958ffc6 Christos Stavrakakis
        raise ValueError(msg)