Statistics
| Branch: | Tag: | Revision:

root / ci / utils.py @ ac1e7de3

History | View | Annotate | Download (30 kB)

1 525f2979 Ilias Tsitsimpis
#!/usr/bin/env python
2 525f2979 Ilias Tsitsimpis
3 525f2979 Ilias Tsitsimpis
"""
4 525f2979 Ilias Tsitsimpis
Synnefo ci utils module
5 525f2979 Ilias Tsitsimpis
"""
6 525f2979 Ilias Tsitsimpis
7 525f2979 Ilias Tsitsimpis
import os
8 dae06cfc Ilias Tsitsimpis
import re
9 525f2979 Ilias Tsitsimpis
import sys
10 525f2979 Ilias Tsitsimpis
import time
11 525f2979 Ilias Tsitsimpis
import logging
12 525f2979 Ilias Tsitsimpis
import fabric.api as fabric
13 024faf05 Christos Stavrakakis
import subprocess
14 124c300e Christos Stavrakakis
import tempfile
15 525f2979 Ilias Tsitsimpis
from ConfigParser import ConfigParser, DuplicateSectionError
16 525f2979 Ilias Tsitsimpis
17 da593e0c Christos Stavrakakis
from kamaki.cli import config as kamaki_config
18 525f2979 Ilias Tsitsimpis
from kamaki.clients.astakos import AstakosClient
19 525f2979 Ilias Tsitsimpis
from kamaki.clients.cyclades import CycladesClient
20 525f2979 Ilias Tsitsimpis
from kamaki.clients.image import ImageClient
21 dae06cfc Ilias Tsitsimpis
from kamaki.clients.compute import ComputeClient
22 4b61ee63 Ilias Tsitsimpis
import filelocker
23 525f2979 Ilias Tsitsimpis
24 e2db4a57 Christos Stavrakakis
DEFAULT_CONFIG_FILE = "new_config"
25 60ddcdc3 Ilias Tsitsimpis
# UUID of owner of system images
26 60ddcdc3 Ilias Tsitsimpis
DEFAULT_SYSTEM_IMAGES_UUID = [
27 60ddcdc3 Ilias Tsitsimpis
    "25ecced9-bf53-4145-91ee-cf47377e9fb2",  # production (okeanos.grnet.gr)
28 60ddcdc3 Ilias Tsitsimpis
    "04cbe33f-29b7-4ef1-94fb-015929e5fc06",  # testing (okeanos.io)
29 ac1e7de3 Christos Stavrakakis
]
30 e2db4a57 Christos Stavrakakis
31 525f2979 Ilias Tsitsimpis
32 525f2979 Ilias Tsitsimpis
def _run(cmd, verbose):
33 525f2979 Ilias Tsitsimpis
    """Run fabric with verbose level"""
34 525f2979 Ilias Tsitsimpis
    if verbose:
35 525f2979 Ilias Tsitsimpis
        args = ('running',)
36 525f2979 Ilias Tsitsimpis
    else:
37 525f2979 Ilias Tsitsimpis
        args = ('running', 'stdout',)
38 33ad9a0d Ilias Tsitsimpis
    with fabric.hide(*args):  # Used * or ** magic. pylint: disable-msg=W0142
39 525f2979 Ilias Tsitsimpis
        return fabric.run(cmd)
40 525f2979 Ilias Tsitsimpis
41 525f2979 Ilias Tsitsimpis
42 4c818bb2 Ilias Tsitsimpis
def _put(local, remote):
43 4c818bb2 Ilias Tsitsimpis
    """Run fabric put command without output"""
44 4c818bb2 Ilias Tsitsimpis
    with fabric.quiet():
45 4c818bb2 Ilias Tsitsimpis
        fabric.put(local, remote)
46 4c818bb2 Ilias Tsitsimpis
47 4c818bb2 Ilias Tsitsimpis
48 525f2979 Ilias Tsitsimpis
def _red(msg):
49 525f2979 Ilias Tsitsimpis
    """Red color"""
50 525f2979 Ilias Tsitsimpis
    #return "\x1b[31m" + str(msg) + "\x1b[0m"
51 525f2979 Ilias Tsitsimpis
    return str(msg)
52 525f2979 Ilias Tsitsimpis
53 525f2979 Ilias Tsitsimpis
54 525f2979 Ilias Tsitsimpis
def _yellow(msg):
55 525f2979 Ilias Tsitsimpis
    """Yellow color"""
56 525f2979 Ilias Tsitsimpis
    #return "\x1b[33m" + str(msg) + "\x1b[0m"
57 525f2979 Ilias Tsitsimpis
    return str(msg)
58 525f2979 Ilias Tsitsimpis
59 525f2979 Ilias Tsitsimpis
60 525f2979 Ilias Tsitsimpis
def _green(msg):
61 525f2979 Ilias Tsitsimpis
    """Green color"""
62 525f2979 Ilias Tsitsimpis
    #return "\x1b[32m" + str(msg) + "\x1b[0m"
63 525f2979 Ilias Tsitsimpis
    return str(msg)
64 525f2979 Ilias Tsitsimpis
65 525f2979 Ilias Tsitsimpis
66 525f2979 Ilias Tsitsimpis
def _check_fabric(fun):
67 525f2979 Ilias Tsitsimpis
    """Check if fabric env has been set"""
68 947b6106 Christos Stavrakakis
    def wrapper(self, *args, **kwargs):
69 525f2979 Ilias Tsitsimpis
        """wrapper function"""
70 525f2979 Ilias Tsitsimpis
        if not self.fabric_installed:
71 525f2979 Ilias Tsitsimpis
            self.setup_fabric()
72 eaf0b161 Ilias Tsitsimpis
            self.fabric_installed = True
73 947b6106 Christos Stavrakakis
        return fun(self, *args, **kwargs)
74 525f2979 Ilias Tsitsimpis
    return wrapper
75 525f2979 Ilias Tsitsimpis
76 525f2979 Ilias Tsitsimpis
77 525f2979 Ilias Tsitsimpis
def _check_kamaki(fun):
78 525f2979 Ilias Tsitsimpis
    """Check if kamaki has been initialized"""
79 947b6106 Christos Stavrakakis
    def wrapper(self, *args, **kwargs):
80 525f2979 Ilias Tsitsimpis
        """wrapper function"""
81 525f2979 Ilias Tsitsimpis
        if not self.kamaki_installed:
82 525f2979 Ilias Tsitsimpis
            self.setup_kamaki()
83 eaf0b161 Ilias Tsitsimpis
            self.kamaki_installed = True
84 947b6106 Christos Stavrakakis
        return fun(self, *args, **kwargs)
85 525f2979 Ilias Tsitsimpis
    return wrapper
86 525f2979 Ilias Tsitsimpis
87 525f2979 Ilias Tsitsimpis
88 525f2979 Ilias Tsitsimpis
class _MyFormatter(logging.Formatter):
89 525f2979 Ilias Tsitsimpis
    """Logging Formatter"""
90 525f2979 Ilias Tsitsimpis
    def format(self, record):
91 525f2979 Ilias Tsitsimpis
        format_orig = self._fmt
92 525f2979 Ilias Tsitsimpis
        if record.levelno == logging.DEBUG:
93 525f2979 Ilias Tsitsimpis
            self._fmt = "  %(msg)s"
94 525f2979 Ilias Tsitsimpis
        elif record.levelno == logging.INFO:
95 525f2979 Ilias Tsitsimpis
            self._fmt = "%(msg)s"
96 525f2979 Ilias Tsitsimpis
        elif record.levelno == logging.WARNING:
97 525f2979 Ilias Tsitsimpis
            self._fmt = _yellow("[W] %(msg)s")
98 525f2979 Ilias Tsitsimpis
        elif record.levelno == logging.ERROR:
99 525f2979 Ilias Tsitsimpis
            self._fmt = _red("[E] %(msg)s")
100 525f2979 Ilias Tsitsimpis
        result = logging.Formatter.format(self, record)
101 525f2979 Ilias Tsitsimpis
        self._fmt = format_orig
102 525f2979 Ilias Tsitsimpis
        return result
103 525f2979 Ilias Tsitsimpis
104 525f2979 Ilias Tsitsimpis
105 d8363ea2 Ilias Tsitsimpis
# Too few public methods. pylint: disable-msg=R0903
106 d8363ea2 Ilias Tsitsimpis
class _InfoFilter(logging.Filter):
107 d8363ea2 Ilias Tsitsimpis
    """Logging Filter that allows DEBUG and INFO messages only"""
108 d8363ea2 Ilias Tsitsimpis
    def filter(self, rec):
109 d8363ea2 Ilias Tsitsimpis
        """The filter"""
110 d8363ea2 Ilias Tsitsimpis
        return rec.levelno in (logging.DEBUG, logging.INFO)
111 d8363ea2 Ilias Tsitsimpis
112 d8363ea2 Ilias Tsitsimpis
113 d8363ea2 Ilias Tsitsimpis
# Too many instance attributes. pylint: disable-msg=R0902
114 525f2979 Ilias Tsitsimpis
class SynnefoCI(object):
115 525f2979 Ilias Tsitsimpis
    """SynnefoCI python class"""
116 525f2979 Ilias Tsitsimpis
117 c441b6a7 Ilias Tsitsimpis
    def __init__(self, config_file=None, build_id=None, cloud=None):
118 525f2979 Ilias Tsitsimpis
        """ Initialize SynnefoCI python class
119 525f2979 Ilias Tsitsimpis

120 525f2979 Ilias Tsitsimpis
        Setup logger, local_dir, config and kamaki
121 525f2979 Ilias Tsitsimpis
        """
122 525f2979 Ilias Tsitsimpis
        # Setup logger
123 525f2979 Ilias Tsitsimpis
        self.logger = logging.getLogger('synnefo-ci')
124 525f2979 Ilias Tsitsimpis
        self.logger.setLevel(logging.DEBUG)
125 d8363ea2 Ilias Tsitsimpis
126 d8363ea2 Ilias Tsitsimpis
        handler1 = logging.StreamHandler(sys.stdout)
127 d8363ea2 Ilias Tsitsimpis
        handler1.setLevel(logging.DEBUG)
128 d8363ea2 Ilias Tsitsimpis
        handler1.addFilter(_InfoFilter())
129 d8363ea2 Ilias Tsitsimpis
        handler1.setFormatter(_MyFormatter())
130 d8363ea2 Ilias Tsitsimpis
        handler2 = logging.StreamHandler(sys.stderr)
131 d8363ea2 Ilias Tsitsimpis
        handler2.setLevel(logging.WARNING)
132 d8363ea2 Ilias Tsitsimpis
        handler2.setFormatter(_MyFormatter())
133 d8363ea2 Ilias Tsitsimpis
134 d8363ea2 Ilias Tsitsimpis
        self.logger.addHandler(handler1)
135 d8363ea2 Ilias Tsitsimpis
        self.logger.addHandler(handler2)
136 525f2979 Ilias Tsitsimpis
137 525f2979 Ilias Tsitsimpis
        # Get our local dir
138 525f2979 Ilias Tsitsimpis
        self.ci_dir = os.path.dirname(os.path.abspath(__file__))
139 525f2979 Ilias Tsitsimpis
        self.repo_dir = os.path.dirname(self.ci_dir)
140 525f2979 Ilias Tsitsimpis
141 525f2979 Ilias Tsitsimpis
        # Read config file
142 e2db4a57 Christos Stavrakakis
        if config_file is None:
143 e2db4a57 Christos Stavrakakis
            config_file = DEFAULT_CONFIG_FILE
144 e2db4a57 Christos Stavrakakis
        if not os.path.isabs(config_file):
145 e2db4a57 Christos Stavrakakis
            config_file = os.path.join(self.ci_dir, config_file)
146 525f2979 Ilias Tsitsimpis
        self.config = ConfigParser()
147 525f2979 Ilias Tsitsimpis
        self.config.optionxform = str
148 e2db4a57 Christos Stavrakakis
        self.config.read(config_file)
149 c441b6a7 Ilias Tsitsimpis
150 c441b6a7 Ilias Tsitsimpis
        # Read temporary_config file
151 4b61ee63 Ilias Tsitsimpis
        self.temp_config_file = \
152 4b61ee63 Ilias Tsitsimpis
            os.path.expanduser(self.config.get('Global', 'temporary_config'))
153 c441b6a7 Ilias Tsitsimpis
        self.temp_config = ConfigParser()
154 c441b6a7 Ilias Tsitsimpis
        self.temp_config.optionxform = str
155 4b61ee63 Ilias Tsitsimpis
        self.temp_config.read(self.temp_config_file)
156 ef695867 Ilias Tsitsimpis
        self.build_id = build_id
157 ef695867 Ilias Tsitsimpis
        self.logger.info("Will use \"%s\" as build id" % _green(self.build_id))
158 525f2979 Ilias Tsitsimpis
159 da593e0c Christos Stavrakakis
        # Set kamaki cloud
160 da593e0c Christos Stavrakakis
        if cloud is not None:
161 da593e0c Christos Stavrakakis
            self.kamaki_cloud = cloud
162 da593e0c Christos Stavrakakis
        elif self.config.has_option("Deployment", "kamaki_cloud"):
163 da593e0c Christos Stavrakakis
            kamaki_cloud = self.config.get("Deployment", "kamaki_cloud")
164 da593e0c Christos Stavrakakis
            if kamaki_cloud == "":
165 da593e0c Christos Stavrakakis
                self.kamaki_cloud = None
166 da593e0c Christos Stavrakakis
        else:
167 da593e0c Christos Stavrakakis
            self.kamaki_cloud = None
168 da593e0c Christos Stavrakakis
169 525f2979 Ilias Tsitsimpis
        # Initialize variables
170 525f2979 Ilias Tsitsimpis
        self.fabric_installed = False
171 525f2979 Ilias Tsitsimpis
        self.kamaki_installed = False
172 525f2979 Ilias Tsitsimpis
        self.cyclades_client = None
173 dae06cfc Ilias Tsitsimpis
        self.compute_client = None
174 525f2979 Ilias Tsitsimpis
        self.image_client = None
175 525f2979 Ilias Tsitsimpis
176 525f2979 Ilias Tsitsimpis
    def setup_kamaki(self):
177 525f2979 Ilias Tsitsimpis
        """Initialize kamaki
178 525f2979 Ilias Tsitsimpis

179 dae06cfc Ilias Tsitsimpis
        Setup cyclades_client, image_client and compute_client
180 525f2979 Ilias Tsitsimpis
        """
181 da593e0c Christos Stavrakakis
182 da593e0c Christos Stavrakakis
        config = kamaki_config.Config()
183 da593e0c Christos Stavrakakis
        if self.kamaki_cloud is None:
184 da593e0c Christos Stavrakakis
            self.kamaki_cloud = config.get_global("default_cloud")
185 da593e0c Christos Stavrakakis
186 da593e0c Christos Stavrakakis
        self.logger.info("Setup kamaki client, using cloud '%s'.." %
187 da593e0c Christos Stavrakakis
                         self.kamaki_cloud)
188 da593e0c Christos Stavrakakis
        auth_url = config.get_cloud(self.kamaki_cloud, "url")
189 525f2979 Ilias Tsitsimpis
        self.logger.debug("Authentication URL is %s" % _green(auth_url))
190 da593e0c Christos Stavrakakis
        token = config.get_cloud(self.kamaki_cloud, "token")
191 525f2979 Ilias Tsitsimpis
        #self.logger.debug("Token is %s" % _green(token))
192 525f2979 Ilias Tsitsimpis
193 ac1e7de3 Christos Stavrakakis
        self.astakos_client = AstakosClient(auth_url, token)
194 525f2979 Ilias Tsitsimpis
195 525f2979 Ilias Tsitsimpis
        cyclades_url = \
196 ac1e7de3 Christos Stavrakakis
            self.astakos_client.get_service_endpoints('compute')['publicURL']
197 525f2979 Ilias Tsitsimpis
        self.logger.debug("Cyclades API url is %s" % _green(cyclades_url))
198 525f2979 Ilias Tsitsimpis
        self.cyclades_client = CycladesClient(cyclades_url, token)
199 525f2979 Ilias Tsitsimpis
        self.cyclades_client.CONNECTION_RETRY_LIMIT = 2
200 525f2979 Ilias Tsitsimpis
201 525f2979 Ilias Tsitsimpis
        image_url = \
202 ac1e7de3 Christos Stavrakakis
            self.astakos_client.get_service_endpoints('image')['publicURL']
203 525f2979 Ilias Tsitsimpis
        self.logger.debug("Images API url is %s" % _green(image_url))
204 525f2979 Ilias Tsitsimpis
        self.image_client = ImageClient(cyclades_url, token)
205 525f2979 Ilias Tsitsimpis
        self.image_client.CONNECTION_RETRY_LIMIT = 2
206 525f2979 Ilias Tsitsimpis
207 dae06cfc Ilias Tsitsimpis
        compute_url = \
208 ac1e7de3 Christos Stavrakakis
            self.astakos_client.get_service_endpoints('compute')['publicURL']
209 dae06cfc Ilias Tsitsimpis
        self.logger.debug("Compute API url is %s" % _green(compute_url))
210 dae06cfc Ilias Tsitsimpis
        self.compute_client = ComputeClient(compute_url, token)
211 dae06cfc Ilias Tsitsimpis
        self.compute_client.CONNECTION_RETRY_LIMIT = 2
212 dae06cfc Ilias Tsitsimpis
213 525f2979 Ilias Tsitsimpis
    def _wait_transition(self, server_id, current_status, new_status):
214 525f2979 Ilias Tsitsimpis
        """Wait for server to go from current_status to new_status"""
215 525f2979 Ilias Tsitsimpis
        self.logger.debug("Waiting for server to become %s" % new_status)
216 525f2979 Ilias Tsitsimpis
        timeout = self.config.getint('Global', 'build_timeout')
217 525f2979 Ilias Tsitsimpis
        sleep_time = 5
218 525f2979 Ilias Tsitsimpis
        while True:
219 525f2979 Ilias Tsitsimpis
            server = self.cyclades_client.get_server_details(server_id)
220 525f2979 Ilias Tsitsimpis
            if server['status'] == new_status:
221 525f2979 Ilias Tsitsimpis
                return server
222 525f2979 Ilias Tsitsimpis
            elif timeout < 0:
223 525f2979 Ilias Tsitsimpis
                self.logger.error(
224 525f2979 Ilias Tsitsimpis
                    "Waiting for server to become %s timed out" % new_status)
225 525f2979 Ilias Tsitsimpis
                self.destroy_server(False)
226 525f2979 Ilias Tsitsimpis
                sys.exit(-1)
227 525f2979 Ilias Tsitsimpis
            elif server['status'] == current_status:
228 525f2979 Ilias Tsitsimpis
                # Sleep for #n secs and continue
229 525f2979 Ilias Tsitsimpis
                timeout = timeout - sleep_time
230 525f2979 Ilias Tsitsimpis
                time.sleep(sleep_time)
231 525f2979 Ilias Tsitsimpis
            else:
232 525f2979 Ilias Tsitsimpis
                self.logger.error(
233 525f2979 Ilias Tsitsimpis
                    "Server failed with status %s" % server['status'])
234 525f2979 Ilias Tsitsimpis
                self.destroy_server(False)
235 525f2979 Ilias Tsitsimpis
                sys.exit(-1)
236 525f2979 Ilias Tsitsimpis
237 525f2979 Ilias Tsitsimpis
    @_check_kamaki
238 525f2979 Ilias Tsitsimpis
    def destroy_server(self, wait=True):
239 525f2979 Ilias Tsitsimpis
        """Destroy slave server"""
240 ef695867 Ilias Tsitsimpis
        server_id = int(self.read_temp_config('server_id'))
241 525f2979 Ilias Tsitsimpis
        self.logger.info("Destoying server with id %s " % server_id)
242 525f2979 Ilias Tsitsimpis
        self.cyclades_client.delete_server(server_id)
243 525f2979 Ilias Tsitsimpis
        if wait:
244 525f2979 Ilias Tsitsimpis
            self._wait_transition(server_id, "ACTIVE", "DELETED")
245 525f2979 Ilias Tsitsimpis
246 525f2979 Ilias Tsitsimpis
    @_check_kamaki
247 6f8b2632 Ilias Tsitsimpis
    def create_server(self, image=None, flavor=None, ssh_keys=None):
248 525f2979 Ilias Tsitsimpis
        """Create slave server"""
249 525f2979 Ilias Tsitsimpis
        self.logger.info("Create a new server..")
250 ef695867 Ilias Tsitsimpis
251 ef695867 Ilias Tsitsimpis
        # Find a build_id to use
252 ef695867 Ilias Tsitsimpis
        if self.build_id is None:
253 ef695867 Ilias Tsitsimpis
            # If build_id is given use this, else ..
254 ef695867 Ilias Tsitsimpis
            # Find a uniq build_id to use
255 ef695867 Ilias Tsitsimpis
            ids = self.temp_config.sections()
256 ef695867 Ilias Tsitsimpis
            if ids:
257 ef695867 Ilias Tsitsimpis
                max_id = int(max(self.temp_config.sections(), key=int))
258 ef695867 Ilias Tsitsimpis
                self.build_id = max_id + 1
259 ef695867 Ilias Tsitsimpis
            else:
260 ef695867 Ilias Tsitsimpis
                self.build_id = 1
261 ef695867 Ilias Tsitsimpis
        self.logger.debug("New build id \"%s\" was created"
262 ef695867 Ilias Tsitsimpis
                          % _green(self.build_id))
263 ef695867 Ilias Tsitsimpis
264 ef695867 Ilias Tsitsimpis
        # Find an image to use
265 6f8b2632 Ilias Tsitsimpis
        image_id = self._find_image(image)
266 ef695867 Ilias Tsitsimpis
        # Find a flavor to use
267 6f8b2632 Ilias Tsitsimpis
        flavor_id = self._find_flavor(flavor)
268 6f8b2632 Ilias Tsitsimpis
269 6f8b2632 Ilias Tsitsimpis
        # Create Server
270 69a64e60 Christos Stavrakakis
        server_name = self.config.get("Deployment", "server_name")
271 525f2979 Ilias Tsitsimpis
        server = self.cyclades_client.create_server(
272 69a64e60 Christos Stavrakakis
            "%s(BID: %s)" % (server_name, self.build_id),
273 947b6106 Christos Stavrakakis
            flavor_id,
274 947b6106 Christos Stavrakakis
            image_id)
275 525f2979 Ilias Tsitsimpis
        server_id = server['id']
276 ef695867 Ilias Tsitsimpis
        self.write_temp_config('server_id', server_id)
277 525f2979 Ilias Tsitsimpis
        self.logger.debug("Server got id %s" % _green(server_id))
278 525f2979 Ilias Tsitsimpis
        server_user = server['metadata']['users']
279 ef695867 Ilias Tsitsimpis
        self.write_temp_config('server_user', server_user)
280 525f2979 Ilias Tsitsimpis
        self.logger.debug("Server's admin user is %s" % _green(server_user))
281 525f2979 Ilias Tsitsimpis
        server_passwd = server['adminPass']
282 ef695867 Ilias Tsitsimpis
        self.write_temp_config('server_passwd', server_passwd)
283 525f2979 Ilias Tsitsimpis
284 525f2979 Ilias Tsitsimpis
        server = self._wait_transition(server_id, "BUILD", "ACTIVE")
285 525f2979 Ilias Tsitsimpis
        self._get_server_ip_and_port(server)
286 4c818bb2 Ilias Tsitsimpis
        self._copy_ssh_keys(ssh_keys)
287 525f2979 Ilias Tsitsimpis
288 eaf0b161 Ilias Tsitsimpis
        # Setup Firewall
289 525f2979 Ilias Tsitsimpis
        self.setup_fabric()
290 525f2979 Ilias Tsitsimpis
        self.logger.info("Setup firewall")
291 4c818bb2 Ilias Tsitsimpis
        accept_ssh_from = self.config.get('Global', 'accept_ssh_from')
292 0082c787 Christos Stavrakakis
        if accept_ssh_from != "":
293 0082c787 Christos Stavrakakis
            self.logger.debug("Block ssh except from %s" % accept_ssh_from)
294 0082c787 Christos Stavrakakis
            cmd = """
295 0082c787 Christos Stavrakakis
            local_ip=$(/sbin/ifconfig eth0 | grep 'inet addr:' | \
296 0082c787 Christos Stavrakakis
                cut -d':' -f2 | cut -d' ' -f1)
297 0082c787 Christos Stavrakakis
            iptables -A INPUT -s localhost -j ACCEPT
298 0082c787 Christos Stavrakakis
            iptables -A INPUT -s $local_ip -j ACCEPT
299 0082c787 Christos Stavrakakis
            iptables -A INPUT -s {0} -p tcp --dport 22 -j ACCEPT
300 0082c787 Christos Stavrakakis
            iptables -A INPUT -p tcp --dport 22 -j DROP
301 0082c787 Christos Stavrakakis
            """.format(accept_ssh_from)
302 0082c787 Christos Stavrakakis
            _run(cmd, False)
303 525f2979 Ilias Tsitsimpis
304 eaf0b161 Ilias Tsitsimpis
        # Setup apt, download packages
305 eaf0b161 Ilias Tsitsimpis
        self.logger.debug("Setup apt. Install x2goserver and firefox")
306 eaf0b161 Ilias Tsitsimpis
        cmd = """
307 eaf0b161 Ilias Tsitsimpis
        echo 'APT::Install-Suggests "false";' >> /etc/apt/apt.conf
308 eaf0b161 Ilias Tsitsimpis
        apt-get update
309 769db39e Ilias Tsitsimpis
        apt-get install curl --yes --force-yes
310 eaf0b161 Ilias Tsitsimpis
        echo -e "\n\n{0}" >> /etc/apt/sources.list
311 eaf0b161 Ilias Tsitsimpis
        # Synnefo repo's key
312 eaf0b161 Ilias Tsitsimpis
        curl https://dev.grnet.gr/files/apt-grnetdev.pub | apt-key add -
313 eaf0b161 Ilias Tsitsimpis
        # X2GO Key
314 eaf0b161 Ilias Tsitsimpis
        apt-key adv --recv-keys --keyserver keys.gnupg.net E1F958385BFE2B6E
315 769db39e Ilias Tsitsimpis
        apt-get install x2go-keyring --yes --force-yes
316 eaf0b161 Ilias Tsitsimpis
        apt-get update
317 769db39e Ilias Tsitsimpis
        apt-get install x2goserver x2goserver-xsession \
318 769db39e Ilias Tsitsimpis
                iceweasel --yes --force-yes
319 eaf0b161 Ilias Tsitsimpis
        """.format(self.config.get('Global', 'apt_repo'))
320 eaf0b161 Ilias Tsitsimpis
        _run(cmd, False)
321 eaf0b161 Ilias Tsitsimpis
322 6f8b2632 Ilias Tsitsimpis
    def _find_flavor(self, flavor=None):
323 6f8b2632 Ilias Tsitsimpis
        """Find a suitable flavor to use
324 6f8b2632 Ilias Tsitsimpis

325 6f8b2632 Ilias Tsitsimpis
        Search by name (reg expression) or by id
326 6f8b2632 Ilias Tsitsimpis
        """
327 6f8b2632 Ilias Tsitsimpis
        # Get a list of flavors from config file
328 6f8b2632 Ilias Tsitsimpis
        flavors = self.config.get('Deployment', 'flavors').split(",")
329 6f8b2632 Ilias Tsitsimpis
        if flavor is not None:
330 c314fcd2 Ilias Tsitsimpis
            # If we have a flavor_name to use, add it to our list
331 6f8b2632 Ilias Tsitsimpis
            flavors.insert(0, flavor)
332 6f8b2632 Ilias Tsitsimpis
333 6f8b2632 Ilias Tsitsimpis
        list_flavors = self.compute_client.list_flavors()
334 6f8b2632 Ilias Tsitsimpis
        for flv in flavors:
335 ac1e7de3 Christos Stavrakakis
            flv_type, flv_value = parse_typed_option(option="flavor",
336 ac1e7de3 Christos Stavrakakis
                                                     value=flv)
337 6f8b2632 Ilias Tsitsimpis
            if flv_type == "name":
338 6f8b2632 Ilias Tsitsimpis
                # Filter flavors by name
339 6f8b2632 Ilias Tsitsimpis
                self.logger.debug(
340 6f8b2632 Ilias Tsitsimpis
                    "Trying to find a flavor with name \"%s\"" % flv_value)
341 6f8b2632 Ilias Tsitsimpis
                list_flvs = \
342 6f8b2632 Ilias Tsitsimpis
                    [f for f in list_flavors
343 ac1e7de3 Christos Stavrakakis
                     if re.search(flv_value, f['name'], flags=re.I)
344 ac1e7de3 Christos Stavrakakis
                     is not None]
345 6f8b2632 Ilias Tsitsimpis
            elif flv_type == "id":
346 6f8b2632 Ilias Tsitsimpis
                # Filter flavors by id
347 6f8b2632 Ilias Tsitsimpis
                self.logger.debug(
348 6f8b2632 Ilias Tsitsimpis
                    "Trying to find a flavor with id \"%s\"" % flv_value)
349 6f8b2632 Ilias Tsitsimpis
                list_flvs = \
350 6f8b2632 Ilias Tsitsimpis
                    [f for f in list_flavors
351 6f8b2632 Ilias Tsitsimpis
                     if f['id'].lower() == flv_value.lower()]
352 6f8b2632 Ilias Tsitsimpis
            else:
353 6f8b2632 Ilias Tsitsimpis
                self.logger.error("Unrecognized flavor type %s" % flv_type)
354 6f8b2632 Ilias Tsitsimpis
355 6f8b2632 Ilias Tsitsimpis
            # Check if we found one
356 6f8b2632 Ilias Tsitsimpis
            if list_flvs:
357 6f8b2632 Ilias Tsitsimpis
                self.logger.debug("Will use \"%s\" with id \"%s\""
358 6f8b2632 Ilias Tsitsimpis
                                  % (list_flvs[0]['name'], list_flvs[0]['id']))
359 6f8b2632 Ilias Tsitsimpis
                return list_flvs[0]['id']
360 c314fcd2 Ilias Tsitsimpis
361 c314fcd2 Ilias Tsitsimpis
        self.logger.error("No matching flavor found.. aborting")
362 c314fcd2 Ilias Tsitsimpis
        sys.exit(1)
363 dae06cfc Ilias Tsitsimpis
364 6f8b2632 Ilias Tsitsimpis
    def _find_image(self, image=None):
365 525f2979 Ilias Tsitsimpis
        """Find a suitable image to use
366 525f2979 Ilias Tsitsimpis

367 6f8b2632 Ilias Tsitsimpis
        In case of search by name, the image has to belong to one
368 6f8b2632 Ilias Tsitsimpis
        of the `DEFAULT_SYSTEM_IMAGES_UUID' users.
369 6f8b2632 Ilias Tsitsimpis
        In case of search by id it only has to exist.
370 525f2979 Ilias Tsitsimpis
        """
371 6f8b2632 Ilias Tsitsimpis
        # Get a list of images from config file
372 6f8b2632 Ilias Tsitsimpis
        images = self.config.get('Deployment', 'images').split(",")
373 6f8b2632 Ilias Tsitsimpis
        if image is not None:
374 6f8b2632 Ilias Tsitsimpis
            # If we have an image from command line, add it to our list
375 6f8b2632 Ilias Tsitsimpis
            images.insert(0, image)
376 6f8b2632 Ilias Tsitsimpis
377 ac1e7de3 Christos Stavrakakis
        auth = self.astakos_client.authenticate()
378 ac1e7de3 Christos Stavrakakis
        user_uuid = auth["access"]["token"]["tenant"]["id"]
379 6f8b2632 Ilias Tsitsimpis
        list_images = self.image_client.list_public(detail=True)['images']
380 6f8b2632 Ilias Tsitsimpis
        for img in images:
381 fe402df2 Christos Stavrakakis
            img_type, img_value = parse_typed_option(option="image", value=img)
382 6f8b2632 Ilias Tsitsimpis
            if img_type == "name":
383 6f8b2632 Ilias Tsitsimpis
                # Filter images by name
384 6f8b2632 Ilias Tsitsimpis
                self.logger.debug(
385 6f8b2632 Ilias Tsitsimpis
                    "Trying to find an image with name \"%s\"" % img_value)
386 ac1e7de3 Christos Stavrakakis
                accepted_uuids = DEFAULT_SYSTEM_IMAGES_UUID + [user_uuid]
387 6f8b2632 Ilias Tsitsimpis
                list_imgs = \
388 ac1e7de3 Christos Stavrakakis
                    [i for i in list_images if i['user_id'] in accepted_uuids
389 ac1e7de3 Christos Stavrakakis
                     and
390 ac1e7de3 Christos Stavrakakis
                     re.search(img_value, i['name'], flags=re.I) is not None]
391 6f8b2632 Ilias Tsitsimpis
            elif img_type == "id":
392 6f8b2632 Ilias Tsitsimpis
                # Filter images by id
393 6f8b2632 Ilias Tsitsimpis
                self.logger.debug(
394 6f8b2632 Ilias Tsitsimpis
                    "Trying to find an image with id \"%s\"" % img_value)
395 6f8b2632 Ilias Tsitsimpis
                list_imgs = \
396 6f8b2632 Ilias Tsitsimpis
                    [i for i in list_images
397 6f8b2632 Ilias Tsitsimpis
                     if i['id'].lower() == img_value.lower()]
398 6f8b2632 Ilias Tsitsimpis
            else:
399 6f8b2632 Ilias Tsitsimpis
                self.logger.error("Unrecognized image type %s" % img_type)
400 6f8b2632 Ilias Tsitsimpis
                sys.exit(1)
401 6f8b2632 Ilias Tsitsimpis
402 6f8b2632 Ilias Tsitsimpis
            # Check if we found one
403 6f8b2632 Ilias Tsitsimpis
            if list_imgs:
404 6f8b2632 Ilias Tsitsimpis
                self.logger.debug("Will use \"%s\" with id \"%s\""
405 6f8b2632 Ilias Tsitsimpis
                                  % (list_imgs[0]['name'], list_imgs[0]['id']))
406 6f8b2632 Ilias Tsitsimpis
                return list_imgs[0]['id']
407 6f8b2632 Ilias Tsitsimpis
408 6f8b2632 Ilias Tsitsimpis
        # We didn't found one
409 6f8b2632 Ilias Tsitsimpis
        self.logger.error("No matching image found.. aborting")
410 6f8b2632 Ilias Tsitsimpis
        sys.exit(1)
411 525f2979 Ilias Tsitsimpis
412 525f2979 Ilias Tsitsimpis
    def _get_server_ip_and_port(self, server):
413 525f2979 Ilias Tsitsimpis
        """Compute server's IPv4 and ssh port number"""
414 525f2979 Ilias Tsitsimpis
        self.logger.info("Get server connection details..")
415 525f2979 Ilias Tsitsimpis
        server_ip = server['attachments'][0]['ipv4']
416 c4c22bb4 Christos Stavrakakis
        if (".okeanos.io" in self.cyclades_client.base_url or
417 c4c22bb4 Christos Stavrakakis
           ".demo.synnefo.org" in self.cyclades_client.base_url):
418 525f2979 Ilias Tsitsimpis
            tmp1 = int(server_ip.split(".")[2])
419 525f2979 Ilias Tsitsimpis
            tmp2 = int(server_ip.split(".")[3])
420 525f2979 Ilias Tsitsimpis
            server_ip = "gate.okeanos.io"
421 525f2979 Ilias Tsitsimpis
            server_port = 10000 + tmp1 * 256 + tmp2
422 525f2979 Ilias Tsitsimpis
        else:
423 525f2979 Ilias Tsitsimpis
            server_port = 22
424 ef695867 Ilias Tsitsimpis
        self.write_temp_config('server_ip', server_ip)
425 525f2979 Ilias Tsitsimpis
        self.logger.debug("Server's IPv4 is %s" % _green(server_ip))
426 ef695867 Ilias Tsitsimpis
        self.write_temp_config('server_port', server_port)
427 525f2979 Ilias Tsitsimpis
        self.logger.debug("Server's ssh port is %s" % _green(server_port))
428 064d2f1b Ilias Tsitsimpis
        self.logger.debug("Access server using \"ssh -X -p %s %s@%s\"" %
429 ac1e7de3 Christos Stavrakakis
                          (server_port, server['metadata']['users'],
430 ac1e7de3 Christos Stavrakakis
                           server_ip))
431 525f2979 Ilias Tsitsimpis
432 5c3b5c9f Christos Stavrakakis
    @_check_fabric
433 4c818bb2 Ilias Tsitsimpis
    def _copy_ssh_keys(self, ssh_keys):
434 33ad9a0d Ilias Tsitsimpis
        """Upload/Install ssh keys to server"""
435 2fab294e Ilias Tsitsimpis
        self.logger.debug("Check for authentication keys to use")
436 4c818bb2 Ilias Tsitsimpis
        if ssh_keys is None:
437 4c818bb2 Ilias Tsitsimpis
            ssh_keys = self.config.get("Deployment", "ssh_keys")
438 4c818bb2 Ilias Tsitsimpis
439 2fab294e Ilias Tsitsimpis
        if ssh_keys != "":
440 8baa4ae7 Ilias Tsitsimpis
            ssh_keys = os.path.expanduser(ssh_keys)
441 ac1e7de3 Christos Stavrakakis
            self.logger.debug("Will use %s authentication keys file" %
442 ac1e7de3 Christos Stavrakakis
                              ssh_keys)
443 5c3b5c9f Christos Stavrakakis
            keyfile = '/tmp/%s.pub' % fabric.env.user
444 5c3b5c9f Christos Stavrakakis
            _run('mkdir -p ~/.ssh && chmod 700 ~/.ssh', False)
445 2fab294e Ilias Tsitsimpis
            if ssh_keys.startswith("http://") or \
446 2fab294e Ilias Tsitsimpis
                    ssh_keys.startswith("https://") or \
447 2fab294e Ilias Tsitsimpis
                    ssh_keys.startswith("ftp://"):
448 2fab294e Ilias Tsitsimpis
                cmd = """
449 2fab294e Ilias Tsitsimpis
                apt-get update
450 769db39e Ilias Tsitsimpis
                apt-get install wget --yes --force-yes
451 2fab294e Ilias Tsitsimpis
                wget {0} -O {1} --no-check-certificate
452 2fab294e Ilias Tsitsimpis
                """.format(ssh_keys, keyfile)
453 2fab294e Ilias Tsitsimpis
                _run(cmd, False)
454 2fab294e Ilias Tsitsimpis
            elif os.path.exists(ssh_keys):
455 2fab294e Ilias Tsitsimpis
                _put(ssh_keys, keyfile)
456 2fab294e Ilias Tsitsimpis
            else:
457 2fab294e Ilias Tsitsimpis
                self.logger.debug("No ssh keys found")
458 8baa4ae7 Ilias Tsitsimpis
                return
459 5c3b5c9f Christos Stavrakakis
            _run('cat %s >> ~/.ssh/authorized_keys' % keyfile, False)
460 5c3b5c9f Christos Stavrakakis
            _run('rm %s' % keyfile, False)
461 5c3b5c9f Christos Stavrakakis
            self.logger.debug("Uploaded ssh authorized keys")
462 5c3b5c9f Christos Stavrakakis
        else:
463 5c3b5c9f Christos Stavrakakis
            self.logger.debug("No ssh keys found")
464 5c3b5c9f Christos Stavrakakis
465 ef695867 Ilias Tsitsimpis
    def write_temp_config(self, option, value):
466 525f2979 Ilias Tsitsimpis
        """Write changes back to config file"""
467 4b61ee63 Ilias Tsitsimpis
        # Acquire the lock to write to temp_config_file
468 4b61ee63 Ilias Tsitsimpis
        with filelocker.lock("%s.lock" % self.temp_config_file,
469 4b61ee63 Ilias Tsitsimpis
                             filelocker.LOCK_EX):
470 4b61ee63 Ilias Tsitsimpis
471 4b61ee63 Ilias Tsitsimpis
            # Read temp_config again to get any new entries
472 4b61ee63 Ilias Tsitsimpis
            self.temp_config.read(self.temp_config_file)
473 4b61ee63 Ilias Tsitsimpis
474 4b61ee63 Ilias Tsitsimpis
            # If build_id section doesn't exist create a new one
475 4b61ee63 Ilias Tsitsimpis
            try:
476 4b61ee63 Ilias Tsitsimpis
                self.temp_config.add_section(str(self.build_id))
477 4b61ee63 Ilias Tsitsimpis
                creation_time = \
478 4b61ee63 Ilias Tsitsimpis
                    time.strftime("%a, %d %b %Y %X", time.localtime())
479 4b61ee63 Ilias Tsitsimpis
                self.temp_config.set(str(self.build_id),
480 4b61ee63 Ilias Tsitsimpis
                                     "created", str(creation_time))
481 4b61ee63 Ilias Tsitsimpis
            except DuplicateSectionError:
482 4b61ee63 Ilias Tsitsimpis
                pass
483 4b61ee63 Ilias Tsitsimpis
            self.temp_config.set(str(self.build_id), option, str(value))
484 4b61ee63 Ilias Tsitsimpis
            curr_time = time.strftime("%a, %d %b %Y %X", time.localtime())
485 4b61ee63 Ilias Tsitsimpis
            self.temp_config.set(str(self.build_id), "modified", curr_time)
486 4b61ee63 Ilias Tsitsimpis
            with open(self.temp_config_file, 'wb') as tcf:
487 4b61ee63 Ilias Tsitsimpis
                self.temp_config.write(tcf)
488 525f2979 Ilias Tsitsimpis
489 ef695867 Ilias Tsitsimpis
    def read_temp_config(self, option):
490 ef695867 Ilias Tsitsimpis
        """Read from temporary_config file"""
491 ef695867 Ilias Tsitsimpis
        # If build_id is None use the latest one
492 ef695867 Ilias Tsitsimpis
        if self.build_id is None:
493 ef695867 Ilias Tsitsimpis
            ids = self.temp_config.sections()
494 ef695867 Ilias Tsitsimpis
            if ids:
495 ef695867 Ilias Tsitsimpis
                self.build_id = int(ids[-1])
496 ef695867 Ilias Tsitsimpis
            else:
497 ef695867 Ilias Tsitsimpis
                self.logger.error("No sections in temporary config file")
498 ef695867 Ilias Tsitsimpis
                sys.exit(1)
499 ef695867 Ilias Tsitsimpis
            self.logger.debug("Will use \"%s\" as build id"
500 ef695867 Ilias Tsitsimpis
                              % _green(self.build_id))
501 ef695867 Ilias Tsitsimpis
        # Read specified option
502 ef695867 Ilias Tsitsimpis
        return self.temp_config.get(str(self.build_id), option)
503 ef695867 Ilias Tsitsimpis
504 525f2979 Ilias Tsitsimpis
    def setup_fabric(self):
505 525f2979 Ilias Tsitsimpis
        """Setup fabric environment"""
506 525f2979 Ilias Tsitsimpis
        self.logger.info("Setup fabric parameters..")
507 ef695867 Ilias Tsitsimpis
        fabric.env.user = self.read_temp_config('server_user')
508 ef695867 Ilias Tsitsimpis
        fabric.env.host_string = self.read_temp_config('server_ip')
509 ef695867 Ilias Tsitsimpis
        fabric.env.port = int(self.read_temp_config('server_port'))
510 ef695867 Ilias Tsitsimpis
        fabric.env.password = self.read_temp_config('server_passwd')
511 525f2979 Ilias Tsitsimpis
        fabric.env.connection_attempts = 10
512 525f2979 Ilias Tsitsimpis
        fabric.env.shell = "/bin/bash -c"
513 525f2979 Ilias Tsitsimpis
        fabric.env.disable_known_hosts = True
514 525f2979 Ilias Tsitsimpis
        fabric.env.output_prefix = None
515 525f2979 Ilias Tsitsimpis
516 525f2979 Ilias Tsitsimpis
    def _check_hash_sum(self, localfile, remotefile):
517 525f2979 Ilias Tsitsimpis
        """Check hash sums of two files"""
518 525f2979 Ilias Tsitsimpis
        self.logger.debug("Check hash sum for local file %s" % localfile)
519 525f2979 Ilias Tsitsimpis
        hash1 = os.popen("sha256sum %s" % localfile).read().split(' ')[0]
520 525f2979 Ilias Tsitsimpis
        self.logger.debug("Local file has sha256 hash %s" % hash1)
521 525f2979 Ilias Tsitsimpis
        self.logger.debug("Check hash sum for remote file %s" % remotefile)
522 525f2979 Ilias Tsitsimpis
        hash2 = _run("sha256sum %s" % remotefile, False)
523 525f2979 Ilias Tsitsimpis
        hash2 = hash2.split(' ')[0]
524 525f2979 Ilias Tsitsimpis
        self.logger.debug("Remote file has sha256 hash %s" % hash2)
525 525f2979 Ilias Tsitsimpis
        if hash1 != hash2:
526 525f2979 Ilias Tsitsimpis
            self.logger.error("Hashes differ.. aborting")
527 525f2979 Ilias Tsitsimpis
            sys.exit(-1)
528 525f2979 Ilias Tsitsimpis
529 525f2979 Ilias Tsitsimpis
    @_check_fabric
530 1d9f2031 Ilias Tsitsimpis
    def clone_repo(self, local_repo=False):
531 525f2979 Ilias Tsitsimpis
        """Clone Synnefo repo from slave server"""
532 525f2979 Ilias Tsitsimpis
        self.logger.info("Configure repositories on remote server..")
533 eaf0b161 Ilias Tsitsimpis
        self.logger.debug("Install/Setup git")
534 525f2979 Ilias Tsitsimpis
        cmd = """
535 769db39e Ilias Tsitsimpis
        apt-get install git --yes --force-yes
536 eaf0b161 Ilias Tsitsimpis
        git config --global user.name {0}
537 eaf0b161 Ilias Tsitsimpis
        git config --global user.email {1}
538 eaf0b161 Ilias Tsitsimpis
        """.format(self.config.get('Global', 'git_config_name'),
539 525f2979 Ilias Tsitsimpis
                   self.config.get('Global', 'git_config_mail'))
540 525f2979 Ilias Tsitsimpis
        _run(cmd, False)
541 525f2979 Ilias Tsitsimpis
542 1d9f2031 Ilias Tsitsimpis
        # Find synnefo_repo and synnefo_branch to use
543 525f2979 Ilias Tsitsimpis
        synnefo_repo = self.config.get('Global', 'synnefo_repo')
544 024faf05 Christos Stavrakakis
        synnefo_branch = self.config.get("Global", "synnefo_branch")
545 024faf05 Christos Stavrakakis
        if synnefo_branch == "":
546 33ad9a0d Ilias Tsitsimpis
            synnefo_branch = \
547 33ad9a0d Ilias Tsitsimpis
                subprocess.Popen(
548 33ad9a0d Ilias Tsitsimpis
                    ["git", "rev-parse", "--abbrev-ref", "HEAD"],
549 024faf05 Christos Stavrakakis
                    stdout=subprocess.PIPE).communicate()[0].strip()
550 024faf05 Christos Stavrakakis
            if synnefo_branch == "HEAD":
551 024faf05 Christos Stavrakakis
                synnefo_branch = \
552 33ad9a0d Ilias Tsitsimpis
                    subprocess.Popen(
553 33ad9a0d Ilias Tsitsimpis
                        ["git", "rev-parse", "--short", "HEAD"],
554 024faf05 Christos Stavrakakis
                        stdout=subprocess.PIPE).communicate()[0].strip()
555 024faf05 Christos Stavrakakis
        self.logger.info("Will use branch %s" % synnefo_branch)
556 1d9f2031 Ilias Tsitsimpis
557 1d9f2031 Ilias Tsitsimpis
        if local_repo or synnefo_branch == "":
558 1d9f2031 Ilias Tsitsimpis
            # Use local_repo
559 1d9f2031 Ilias Tsitsimpis
            self.logger.debug("Push local repo to server")
560 1d9f2031 Ilias Tsitsimpis
            # Firstly create the remote repo
561 1d9f2031 Ilias Tsitsimpis
            _run("git init synnefo", False)
562 1d9f2031 Ilias Tsitsimpis
            # Then push our local repo over ssh
563 1d9f2031 Ilias Tsitsimpis
            # We have to pass some arguments to ssh command
564 1d9f2031 Ilias Tsitsimpis
            # namely to disable host checking.
565 1d9f2031 Ilias Tsitsimpis
            (temp_ssh_file_handle, temp_ssh_file) = tempfile.mkstemp()
566 1d9f2031 Ilias Tsitsimpis
            os.close(temp_ssh_file_handle)
567 4b61ee63 Ilias Tsitsimpis
            # XXX: git push doesn't read the password
568 1d9f2031 Ilias Tsitsimpis
            cmd = """
569 1d9f2031 Ilias Tsitsimpis
            echo 'exec ssh -o "StrictHostKeyChecking no" \
570 1d9f2031 Ilias Tsitsimpis
                           -o "UserKnownHostsFile /dev/null" \
571 1d9f2031 Ilias Tsitsimpis
                           -q "$@"' > {4}
572 1d9f2031 Ilias Tsitsimpis
            chmod u+x {4}
573 1d9f2031 Ilias Tsitsimpis
            export GIT_SSH="{4}"
574 1d9f2031 Ilias Tsitsimpis
            echo "{0}" | git push --mirror ssh://{1}@{2}:{3}/~/synnefo
575 1d9f2031 Ilias Tsitsimpis
            rm -f {4}
576 1d9f2031 Ilias Tsitsimpis
            """.format(fabric.env.password,
577 1d9f2031 Ilias Tsitsimpis
                       fabric.env.user,
578 1d9f2031 Ilias Tsitsimpis
                       fabric.env.host_string,
579 1d9f2031 Ilias Tsitsimpis
                       fabric.env.port,
580 1d9f2031 Ilias Tsitsimpis
                       temp_ssh_file)
581 1d9f2031 Ilias Tsitsimpis
            os.system(cmd)
582 1d9f2031 Ilias Tsitsimpis
        else:
583 1d9f2031 Ilias Tsitsimpis
            # Clone Synnefo from remote repo
584 1d9f2031 Ilias Tsitsimpis
            # Currently clonning synnefo can fail unexpectedly
585 1d9f2031 Ilias Tsitsimpis
            cloned = False
586 1d9f2031 Ilias Tsitsimpis
            for i in range(10):
587 1d9f2031 Ilias Tsitsimpis
                self.logger.debug("Clone synnefo from %s" % synnefo_repo)
588 1d9f2031 Ilias Tsitsimpis
                try:
589 1d9f2031 Ilias Tsitsimpis
                    _run("git clone %s synnefo" % synnefo_repo, False)
590 1d9f2031 Ilias Tsitsimpis
                    cloned = True
591 1d9f2031 Ilias Tsitsimpis
                    break
592 1d9f2031 Ilias Tsitsimpis
                except BaseException:
593 1d9f2031 Ilias Tsitsimpis
                    self.logger.warning(
594 1d9f2031 Ilias Tsitsimpis
                        "Clonning synnefo failed.. retrying %s" % i)
595 1d9f2031 Ilias Tsitsimpis
            if not cloned:
596 1d9f2031 Ilias Tsitsimpis
                self.logger.error("Can not clone Synnefo repo.")
597 1d9f2031 Ilias Tsitsimpis
                sys.exit(-1)
598 1d9f2031 Ilias Tsitsimpis
599 1d9f2031 Ilias Tsitsimpis
        # Checkout the desired synnefo_branch
600 064d2f1b Ilias Tsitsimpis
        self.logger.debug("Checkout \"%s\" branch/commit" % synnefo_branch)
601 33ad9a0d Ilias Tsitsimpis
        cmd = """
602 024faf05 Christos Stavrakakis
        cd synnefo
603 33ad9a0d Ilias Tsitsimpis
        for branch in `git branch -a | grep remotes | \
604 33ad9a0d Ilias Tsitsimpis
                       grep -v HEAD | grep -v master`; do
605 024faf05 Christos Stavrakakis
            git branch --track ${branch##*/} $branch
606 024faf05 Christos Stavrakakis
        done
607 024faf05 Christos Stavrakakis
        git checkout %s
608 024faf05 Christos Stavrakakis
        """ % (synnefo_branch)
609 024faf05 Christos Stavrakakis
        _run(cmd, False)
610 024faf05 Christos Stavrakakis
611 525f2979 Ilias Tsitsimpis
    @_check_fabric
612 525f2979 Ilias Tsitsimpis
    def build_synnefo(self):
613 525f2979 Ilias Tsitsimpis
        """Build Synnefo packages"""
614 525f2979 Ilias Tsitsimpis
        self.logger.info("Build Synnefo packages..")
615 525f2979 Ilias Tsitsimpis
        self.logger.debug("Install development packages")
616 525f2979 Ilias Tsitsimpis
        cmd = """
617 525f2979 Ilias Tsitsimpis
        apt-get update
618 525f2979 Ilias Tsitsimpis
        apt-get install zlib1g-dev dpkg-dev debhelper git-buildpackage \
619 769db39e Ilias Tsitsimpis
                python-dev python-all python-pip --yes --force-yes
620 525f2979 Ilias Tsitsimpis
        pip install devflow
621 525f2979 Ilias Tsitsimpis
        """
622 525f2979 Ilias Tsitsimpis
        _run(cmd, False)
623 525f2979 Ilias Tsitsimpis
624 133e5a5b Christos Stavrakakis
        if self.config.get('Global', 'patch_pydist') == "True":
625 525f2979 Ilias Tsitsimpis
            self.logger.debug("Patch pydist.py module")
626 525f2979 Ilias Tsitsimpis
            cmd = r"""
627 525f2979 Ilias Tsitsimpis
            sed -r -i 's/(\(\?P<name>\[A-Za-z\]\[A-Za-z0-9_\.)/\1\\\-/' \
628 525f2979 Ilias Tsitsimpis
                /usr/share/python/debpython/pydist.py
629 525f2979 Ilias Tsitsimpis
            """
630 525f2979 Ilias Tsitsimpis
            _run(cmd, False)
631 525f2979 Ilias Tsitsimpis

632 24ccf5d3 Ilias Tsitsimpis
        # Build synnefo packages
633 24ccf5d3 Ilias Tsitsimpis
        self.logger.debug("Build synnefo packages")
634 525f2979 Ilias Tsitsimpis
        cmd = """
635 24ccf5d3 Ilias Tsitsimpis
        devflow-autopkg snapshot -b ~/synnefo_build-area --no-sign
636 525f2979 Ilias Tsitsimpis
        """
637 24ccf5d3 Ilias Tsitsimpis
        with fabric.cd("synnefo"):
638 525f2979 Ilias Tsitsimpis
            _run(cmd, True)
639 525f2979 Ilias Tsitsimpis

640 24ccf5d3 Ilias Tsitsimpis
        # Install snf-deploy package
641 525f2979 Ilias Tsitsimpis
        self.logger.debug("Install snf-deploy package")
642 525f2979 Ilias Tsitsimpis
        cmd = """
643 525f2979 Ilias Tsitsimpis
        dpkg -i snf-deploy*.deb
644 769db39e Ilias Tsitsimpis
        apt-get -f install --yes --force-yes
645 525f2979 Ilias Tsitsimpis
        """
646 24ccf5d3 Ilias Tsitsimpis
        with fabric.cd("synnefo_build-area"):
647 525f2979 Ilias Tsitsimpis
            with fabric.settings(warn_only=True):
648 525f2979 Ilias Tsitsimpis
                _run(cmd, True)
649 525f2979 Ilias Tsitsimpis

650 24ccf5d3 Ilias Tsitsimpis
        # Setup synnefo packages for snf-deploy
651 525f2979 Ilias Tsitsimpis
        self.logger.debug("Copy synnefo debs to snf-deploy packages dir")
652 525f2979 Ilias Tsitsimpis
        cmd = """
653 525f2979 Ilias Tsitsimpis
        cp ~/synnefo_build-area/*.deb /var/lib/snf-deploy/packages/
654 525f2979 Ilias Tsitsimpis
        """
655 525f2979 Ilias Tsitsimpis
        _run(cmd, False)
656 525f2979 Ilias Tsitsimpis

657 124c300e Christos Stavrakakis
    @_check_fabric
658 124c300e Christos Stavrakakis
    def build_documentation(self):
659 33ad9a0d Ilias Tsitsimpis
        """Build Synnefo documentation"""
660 124c300e Christos Stavrakakis
        self.logger.info("Build Synnefo documentation..")
661 124c300e Christos Stavrakakis
        _run("pip install -U Sphinx", False)
662 124c300e Christos Stavrakakis
        with fabric.cd("synnefo"):
663 33ad9a0d Ilias Tsitsimpis
            _run("devflow-update-version; "
664 33ad9a0d Ilias Tsitsimpis
                 "./ci/make_docs.sh synnefo_documentation", False)
665 124c300e Christos Stavrakakis

666 124c300e Christos Stavrakakis
    def fetch_documentation(self, dest=None):
667 33ad9a0d Ilias Tsitsimpis
        """Fetch Synnefo documentation"""
668 33ad9a0d Ilias Tsitsimpis
        self.logger.info("Fetch Synnefo documentation..")
669 124c300e Christos Stavrakakis
        if dest is None:
670 124c300e Christos Stavrakakis
            dest = "synnefo_documentation"
671 124c300e Christos Stavrakakis
        dest = os.path.abspath(dest)
672 124c300e Christos Stavrakakis
        if not os.path.exists(dest):
673 124c300e Christos Stavrakakis
            os.makedirs(dest)
674 124c300e Christos Stavrakakis
        self.fetch_compressed("synnefo/synnefo_documentation", dest)
675 124c300e Christos Stavrakakis
        self.logger.info("Downloaded documentation to %s" %
676 124c300e Christos Stavrakakis
                         _green(dest))
677 124c300e Christos Stavrakakis

678 525f2979 Ilias Tsitsimpis
    @_check_fabric
679 46a07468 Christos Stavrakakis
    def deploy_synnefo(self, schema=None):
680 525f2979 Ilias Tsitsimpis
        """Deploy Synnefo using snf-deploy"""
681 525f2979 Ilias Tsitsimpis
        self.logger.info("Deploy Synnefo..")
682 46a07468 Christos Stavrakakis
        if schema is None:
683 46a07468 Christos Stavrakakis
            schema = self.config.get('Global', 'schema')
684 6f8b2632 Ilias Tsitsimpis
        self.logger.debug("Will use \"%s\" schema" % schema)
685 525f2979 Ilias Tsitsimpis

686 46a07468 Christos Stavrakakis
        schema_dir = os.path.join(self.ci_dir, "schemas/%s" % schema)
687 46a07468 Christos Stavrakakis
        if not (os.path.exists(schema_dir) and os.path.isdir(schema_dir)):
688 46a07468 Christos Stavrakakis
            raise ValueError("Unknown schema: %s" % schema)
689 46a07468 Christos Stavrakakis

690 525f2979 Ilias Tsitsimpis
        self.logger.debug("Upload schema files to server")
691 4c818bb2 Ilias Tsitsimpis
        _put(os.path.join(schema_dir, "*"), "/etc/snf-deploy/")
692 525f2979 Ilias Tsitsimpis

693 525f2979 Ilias Tsitsimpis
        self.logger.debug("Change password in nodes.conf file")
694 525f2979 Ilias Tsitsimpis
        cmd = """
695 525f2979 Ilias Tsitsimpis
        sed -i 's/^password =.*/password = {0}/' /etc/snf-deploy/nodes.conf
696 525f2979 Ilias Tsitsimpis
        """.format(fabric.env.password)
697 525f2979 Ilias Tsitsimpis
        _run(cmd, False)
698 525f2979 Ilias Tsitsimpis

699 525f2979 Ilias Tsitsimpis
        self.logger.debug("Run snf-deploy")
700 525f2979 Ilias Tsitsimpis
        cmd = """
701 39dd21c2 Christos Stavrakakis
        snf-deploy --disable-colors --autoconf all
702 525f2979 Ilias Tsitsimpis
        """
703 525f2979 Ilias Tsitsimpis
        _run(cmd, True)
704 525f2979 Ilias Tsitsimpis

705 525f2979 Ilias Tsitsimpis
    @_check_fabric
706 525f2979 Ilias Tsitsimpis
    def unit_test(self):
707 525f2979 Ilias Tsitsimpis
        """Run Synnefo unit test suite"""
708 525f2979 Ilias Tsitsimpis
        self.logger.info("Run Synnefo unit test suite")
709 525f2979 Ilias Tsitsimpis
        component = self.config.get('Unit Tests', 'component')
710 525f2979 Ilias Tsitsimpis

711 525f2979 Ilias Tsitsimpis
        self.logger.debug("Install needed packages")
712 525f2979 Ilias Tsitsimpis
        cmd = """
713 525f2979 Ilias Tsitsimpis
        pip install mock
714 525f2979 Ilias Tsitsimpis
        pip install factory_boy
715 525f2979 Ilias Tsitsimpis
        """
716 525f2979 Ilias Tsitsimpis
        _run(cmd, False)
717 525f2979 Ilias Tsitsimpis

718 2255812d Ilias Tsitsimpis
        self.logger.debug("Upload tests.sh file")
719 2255812d Ilias Tsitsimpis
        unit_tests_file = os.path.join(self.ci_dir, "tests.sh")
720 4c818bb2 Ilias Tsitsimpis
        _put(unit_tests_file, ".")
721 525f2979 Ilias Tsitsimpis

722 525f2979 Ilias Tsitsimpis
        self.logger.debug("Run unit tests")
723 525f2979 Ilias Tsitsimpis
        cmd = """
724 2255812d Ilias Tsitsimpis
        bash tests.sh {0}
725 525f2979 Ilias Tsitsimpis
        """.format(component)
726 525f2979 Ilias Tsitsimpis
        _run(cmd, True)
727 525f2979 Ilias Tsitsimpis

728 525f2979 Ilias Tsitsimpis
    @_check_fabric
729 525f2979 Ilias Tsitsimpis
    def run_burnin(self):
730 525f2979 Ilias Tsitsimpis
        """Run burnin functional test suite"""
731 525f2979 Ilias Tsitsimpis
        self.logger.info("Run Burnin functional test suite")
732 525f2979 Ilias Tsitsimpis
        cmd = """
733 525f2979 Ilias Tsitsimpis
        auth_url=$(grep -e '^url =' .kamakirc | cut -d' ' -f3)
734 525f2979 Ilias Tsitsimpis
        token=$(grep -e '^token =' .kamakirc | cut -d' ' -f3)
735 525f2979 Ilias Tsitsimpis
        images_user=$(kamaki image list -l | grep owner | \
736 525f2979 Ilias Tsitsimpis
                      cut -d':' -f2 | tr -d ' ')
737 525f2979 Ilias Tsitsimpis
        snf-burnin --auth-url=$auth_url --token=$token \
738 525f2979 Ilias Tsitsimpis
            --force-flavor=2 --image-id=all \
739 525f2979 Ilias Tsitsimpis
            --system-images-user=$images_user \
740 525f2979 Ilias Tsitsimpis
            {0}
741 525f2979 Ilias Tsitsimpis
        log_folder=$(ls -1d /var/log/burnin/* | tail -n1)
742 525f2979 Ilias Tsitsimpis
        for i in $(ls $log_folder/*/details*); do
743 525f2979 Ilias Tsitsimpis
            echo -e "\\n\\n"
744 525f2979 Ilias Tsitsimpis
            echo -e "***** $i\\n"
745 525f2979 Ilias Tsitsimpis
            cat $i
746 525f2979 Ilias Tsitsimpis
        done
747 525f2979 Ilias Tsitsimpis
        """.format(self.config.get('Burnin', 'cmd_options'))
748 525f2979 Ilias Tsitsimpis
        _run(cmd, True)
749 525f2979 Ilias Tsitsimpis

750 525f2979 Ilias Tsitsimpis
    @_check_fabric
751 124c300e Christos Stavrakakis
    def fetch_compressed(self, src, dest=None):
752 33ad9a0d Ilias Tsitsimpis
        """Create a tarball and fetch it locally"""
753 124c300e Christos Stavrakakis
        self.logger.debug("Creating tarball of %s" % src)
754 124c300e Christos Stavrakakis
        basename = os.path.basename(src)
755 124c300e Christos Stavrakakis
        tar_file = basename + ".tgz"
756 124c300e Christos Stavrakakis
        cmd = "tar czf %s %s" % (tar_file, src)
757 525f2979 Ilias Tsitsimpis
        _run(cmd, False)
758 124c300e Christos Stavrakakis
        if not os.path.exists(dest):
759 124c300e Christos Stavrakakis
            os.makedirs(dest)
760 525f2979 Ilias Tsitsimpis

761 124c300e Christos Stavrakakis
        tmp_dir = tempfile.mkdtemp()
762 124c300e Christos Stavrakakis
        fabric.get(tar_file, tmp_dir)
763 525f2979 Ilias Tsitsimpis

764 124c300e Christos Stavrakakis
        dest_file = os.path.join(tmp_dir, tar_file)
765 124c300e Christos Stavrakakis
        self._check_hash_sum(dest_file, tar_file)
766 124c300e Christos Stavrakakis
        self.logger.debug("Untar packages file %s" % dest_file)
767 ee2419c5 Christos Stavrakakis
        cmd = """
768 ee2419c5 Christos Stavrakakis
        cd %s
769 124c300e Christos Stavrakakis
        tar xzf %s
770 124c300e Christos Stavrakakis
        cp -r %s/* %s
771 124c300e Christos Stavrakakis
        rm -r %s
772 124c300e Christos Stavrakakis
        """ % (tmp_dir, tar_file, src, dest, tmp_dir)
773 ee2419c5 Christos Stavrakakis
        os.system(cmd)
774 124c300e Christos Stavrakakis
        self.logger.info("Downloaded %s to %s" %
775 124c300e Christos Stavrakakis
                         (src, _green(dest)))
776 124c300e Christos Stavrakakis

777 124c300e Christos Stavrakakis
    @_check_fabric
778 124c300e Christos Stavrakakis
    def fetch_packages(self, dest=None):
779 33ad9a0d Ilias Tsitsimpis
        """Fetch Synnefo packages"""
780 124c300e Christos Stavrakakis
        if dest is None:
781 124c300e Christos Stavrakakis
            dest = self.config.get('Global', 'pkgs_dir')
782 8baa4ae7 Ilias Tsitsimpis
        dest = os.path.abspath(os.path.expanduser(dest))
783 124c300e Christos Stavrakakis
        if not os.path.exists(dest):
784 124c300e Christos Stavrakakis
            os.makedirs(dest)
785 124c300e Christos Stavrakakis
        self.fetch_compressed("synnefo_build-area", dest)
786 85bcc8da Christos Stavrakakis
        self.logger.info("Downloaded debian packages to %s" %
787 124c300e Christos Stavrakakis
                         _green(dest))
788 fe402df2 Christos Stavrakakis

789 fe402df2 Christos Stavrakakis

790 fe402df2 Christos Stavrakakis
def parse_typed_option(option, value):
791 fe402df2 Christos Stavrakakis
    try:
792 fe402df2 Christos Stavrakakis
        [type_, val] = value.strip().split(':')
793 fe402df2 Christos Stavrakakis
        if type_ not in ["id", "name"]:
794 fe402df2 Christos Stavrakakis
            raise ValueError
795 fe402df2 Christos Stavrakakis
        return type_, val
796 fe402df2 Christos Stavrakakis
    except ValueError:
797 fe402df2 Christos Stavrakakis
        msg = "Invalid %s format. Must be [id|name]:.+" % option
798 fe402df2 Christos Stavrakakis
        raise ValueError(msg)