Statistics
| Branch: | Tag: | Revision:

root / kamaki / cli / utils / __init__.py @ 8df239df

History | View | Annotate | Download (16.3 kB)

1
# Copyright 2011-2014 GRNET S.A. All rights reserved.
2
#
3
# Redistribution and use in source and binary forms, with or
4
# without modification, are permitted provided that the following
5
# conditions are met:
6
#
7
#   1. Redistributions of source code must retain the above
8
#      copyright notice, this list of conditions and the following
9
#      disclaimer.
10
#
11
#   2. Redistributions in binary form must reproduce the above
12
#      copyright notice, this list of conditions and the following
13
#      disclaimer in the documentation and/or other materials
14
#      provided with the distribution.
15
#
16
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
17
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
19
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
20
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
23
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
24
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
26
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
27
# POSSIBILITY OF SUCH DAMAGE.
28
#
29
# The views and conclusions contained in the software and
30
# documentation are those of the authors and should not be
31
# interpreted as representing official policies, either expressed
32
# or implied, of GRNET S.A.
33

    
34
from sys import stdout, stdin, stderr
35
from re import compile as regex_compile
36
from os import walk, path
37
from json import dumps
38
from kamaki.cli.logger import get_logger
39
from locale import getpreferredencoding
40

    
41
from kamaki.cli.errors import raiseCLIError
42

    
43

    
44
INDENT_TAB = 4
45
log = get_logger(__name__)
46
pref_enc = getpreferredencoding()
47

    
48

    
49
suggest = dict(ansicolors=dict(
50
        active=False,
51
        url='#install-ansicolors-progress',
52
        description='Add colors to console responses'))
53

    
54
try:
55
    from colors import magenta, red, yellow, bold
56
except ImportError:
57
    def dummy(val):
58
        return val
59
    red = yellow = magenta = bold = dummy
60
    suggest['ansicolors']['active'] = True
61

    
62

    
63
def _encode_nicely(somestr, encoding, replacement='?'):
64
    """Encode somestr as 'encoding', but don't raise errors (replace with ?)
65
        This method is slow. Us it only for grace.
66
        :param encoding: (str) encode every character in this encoding
67
        :param replacement: (char) replace each char raising encode-decode errs
68
    """
69
    newstr, err_counter = '', 0
70
    for c in somestr:
71
        try:
72
            newc = c.encode(encoding)
73
            newstr = '%s%s' % (newstr, newc)
74
        except UnicodeError:
75
            newstr = '%s%s' % (newstr, replacement)
76
            err_counter += 1
77
    if err_counter:
78
        log.debug('\t%s character%s failed to be encoded as %s' % (
79
            err_counter, 's' if err_counter > 1 else '', encoding))
80
    return newstr
81

    
82

    
83
def DontRaiseUnicodeError(foo):
84
    def wrap(self, *args, **kwargs):
85
        try:
86
            s = kwargs.pop('s')
87
        except KeyError:
88
            try:
89
                s = args[0]
90
            except IndexError:
91
                return foo(self, *args, **kwargs)
92
            args = args[1:]
93
        try:
94
            s = s.encode(pref_enc)
95
        except UnicodeError as ue:
96
            log.debug('Encoding(%s): %s' % (pref_enc, ue))
97
            s = _encode_nicely(s, pref_enc, replacement='?')
98
        return foo(self, s, *args, **kwargs)
99
    return wrap
100

    
101

    
102
def encode_for_console(s, encoding=pref_enc, replacement='?'):
103
    if encoding.lower() == 'utf-8':
104
        return s
105
    try:
106
        return s.encode(encoding)
107
    except UnicodeError as ue:
108
        log.debug('Encoding(%s): %s' % (encoding, ue))
109
        return _encode_nicely(s, encoding, replacement)
110

    
111

    
112
def suggest_missing(miss=None, exclude=[]):
113
    global suggest
114
    sgs = dict(suggest)
115
    for exc in exclude:
116
        try:
117
            sgs.pop(exc)
118
        except KeyError:
119
            pass
120
    kamaki_docs = 'http://www.synnefo.org/docs/kamaki/latest'
121

    
122
    for k, v in (miss, sgs[miss]) if miss else sgs.items():
123
        if v['active'] and stderr.isatty():
124
            stderr.write('Suggestion: you may like to install %s\n' % k)
125
            stderr.write('\t%s\n' % encode_for_console(v['description']))
126
            stderr.write('\tIt is easy, here are the instructions:\n')
127
            stderr.write('\t%s/installation.html%s\n' % (
128
                kamaki_docs, v['url']))
129
            stderr.flush()
130

    
131

    
132
def guess_mime_type(
133
        filename,
134
        default_content_type='application/octet-stream',
135
        default_encoding=None):
136
    assert filename, 'Cannot guess mimetype for empty filename'
137
    try:
138
        from mimetypes import guess_type
139
        ctype, cenc = guess_type(filename)
140
        return ctype or default_content_type, cenc or default_encoding
141
    except ImportError:
142
        stderr.write('WARNING: Cannot import mimetypes, using defaults\n')
143
        stderr.flush()
144
        return (default_content_type, default_encoding)
145

    
146

    
147
def remove_colors():
148
    global bold
149
    global red
150
    global yellow
151
    global magenta
152

    
153
    def dummy(val):
154
        return val
155
    red = yellow = magenta = bold = dummy
156

    
157

    
158
def pretty_keys(d, delim='_', recursive=False):
159
    """<term>delim<term> to <term> <term> transformation"""
160
    new_d = dict(d)
161
    for k, v in d.items():
162
        new_v = new_d.pop(k)
163
        new_d[k.replace(delim, ' ').strip()] = pretty_keys(
164
            new_v, delim, True) if (
165
                recursive and isinstance(v, dict)) else new_v
166
    return new_d
167

    
168

    
169
def print_json(data, out=stdout, encoding=pref_enc):
170
    """Print a list or dict as json in console
171

172
    :param data: json-dumpable data
173

174
    :param out: Input/Output stream to dump values into
175
    """
176
    s = encode_for_console(dumps(data, indent=INDENT_TAB), encoding)
177
    out.write(s + '\n')
178
    out.flush()
179

    
180

    
181
def print_dict(
182
        d,
183
        exclude=(), indent=0,
184
        with_enumeration=False, recursive_enumeration=False, out=stdout):
185
    """Pretty-print a dictionary object
186
    <indent>key: <non iterable item>
187
    <indent>key:
188
    <indent + INDENT_TAB><pretty-print iterable>
189

190
    :param d: (dict)
191

192
    :param exclude: (iterable of strings) keys to exclude from printing
193

194
    :param indent: (int) initial indentation (recursive)
195

196
    :param with_enumeration: (bool) enumerate 1st-level keys
197

198
    :param recursive_enumeration: (bool) recursively enumerate iterables (does
199
        not enumerate 1st level keys)
200

201
    :param out: Input/Output stream to dump values into
202

203
    :raises CLIError: if preconditions fail
204
    """
205
    assert isinstance(d, dict), 'print_dict input must be a dict'
206
    assert indent >= 0, 'print_dict indent must be >= 0'
207

    
208
    for i, (k, v) in enumerate(d.items()):
209
        k = ('%s' % k).strip()
210
        if k in exclude:
211
            continue
212
        print_str = ' ' * indent
213
        print_str += '%s.' % (i + 1) if with_enumeration else ''
214
        print_str += '%s:' % k
215
        if isinstance(v, dict):
216
            out.write(encode_for_console(print_str) + '\n')
217
            print_dict(
218
                v, exclude, indent + INDENT_TAB,
219
                recursive_enumeration, recursive_enumeration, out)
220
        elif isinstance(v, list) or isinstance(v, tuple):
221
            out.write(encode_for_console(print_str) + '\n')
222
            print_list(
223
                v, exclude, indent + INDENT_TAB,
224
                recursive_enumeration, recursive_enumeration, out)
225
        else:
226
            out.write(encode_for_console('%s %s\n' % (print_str, v)))
227
        out.flush()
228

    
229

    
230
def print_list(
231
        l,
232
        exclude=(), indent=0,
233
        with_enumeration=False, recursive_enumeration=False, out=stdout):
234
    """Pretty-print a list of items
235
    <indent>key: <non iterable item>
236
    <indent>key:
237
    <indent + INDENT_TAB><pretty-print iterable>
238

239
    :param l: (list)
240

241
    :param exclude: (iterable of strings) items to exclude from printing
242

243
    :param indent: (int) initial indentation (recursive)
244

245
    :param with_enumeration: (bool) enumerate 1st-level items
246

247
    :param recursive_enumeration: (bool) recursively enumerate iterables (does
248
        not enumerate 1st level keys)
249

250
    :param out: Input/Output stream to dump values into
251

252
    :raises CLIError: if preconditions fail
253
    """
254
    assert isinstance(l, list) or isinstance(l, tuple), (
255
        'print_list prinbts a list or tuple')
256
    assert indent >= 0, 'print_list indent must be >= 0'
257

    
258
    for i, item in enumerate(l):
259
        print_str = ' ' * indent
260
        print_str += '%s.' % (i + 1) if with_enumeration else ''
261
        if isinstance(item, dict):
262
            if with_enumeration:
263
                out.write(encode_for_console(print_str) + '\n')
264
            elif i and i < len(l):
265
                out.write('\n')
266
            print_dict(
267
                item, exclude,
268
                indent + (INDENT_TAB if with_enumeration else 0),
269
                recursive_enumeration, recursive_enumeration, out)
270
        elif isinstance(item, list) or isinstance(item, tuple):
271
            if with_enumeration:
272
                out.write(encode_for_console(print_str) + '\n')
273
            elif i and i < len(l):
274
                out.write('\n')
275
            print_list(
276
                item, exclude, indent + INDENT_TAB,
277
                recursive_enumeration, recursive_enumeration, out)
278
        else:
279
            item = ('%s' % item).strip()
280
            if item in exclude:
281
                continue
282
            out.write(encode_for_console('%s%s\n' % (print_str, item)))
283
        out.flush()
284
    out.flush()
285

    
286

    
287
def print_items(
288
        items, title=('id', 'name'),
289
        with_enumeration=False, with_redundancy=False, out=stdout):
290
    """print dict or list items in a list, using some values as title
291
    Objects of next level don't inherit enumeration (default: off) or titles
292

293
    :param items: (list) items are lists or dict
294

295
    :param title: (tuple) keys to use their values as title
296

297
    :param with_enumeration: (boolean) enumerate items (order id on title)
298

299
    :param with_redundancy: (boolean) values in title also appear on body
300

301
    :param out: Input/Output stream to dump values into
302
    """
303
    if not items:
304
        return
305
    if not (isinstance(items, dict) or isinstance(items, list) or isinstance(
306
                items, tuple)):
307
        out.write(encode_for_console('%s\n' % items))
308
        out.flush()
309
        return
310

    
311
    for i, item in enumerate(items):
312
        if with_enumeration:
313
            out.write(encode_for_console('%s. ' % (i + 1)))
314
        if isinstance(item, dict):
315
            item = dict(item)
316
            title = sorted(set(title).intersection(item))
317
            pick = item.get if with_redundancy else item.pop
318
            header = ' '.join('%s' % pick(key) for key in title)
319
            out.write(encode_for_console(
320
                (unicode(bold(header) if header else '') + '\n')))
321
            print_dict(item, indent=INDENT_TAB, out=out)
322
        elif isinstance(item, list) or isinstance(item, tuple):
323
            print_list(item, indent=INDENT_TAB, out=out)
324
        else:
325
            out.write(encode_for_console(' %s\n' % item))
326
        out.flush()
327
    out.flush()
328

    
329

    
330
def format_size(size, decimal_factors=False):
331
    units = ('B', 'KB', 'MB', 'GB', 'TB') if decimal_factors else (
332
        'B', 'KiB', 'MiB', 'GiB', 'TiB')
333
    step = 1000 if decimal_factors else 1024
334
    fstep = float(step)
335
    try:
336
        size = float(size)
337
    except (ValueError, TypeError) as err:
338
        raiseCLIError(err, 'Cannot format %s in bytes' % (
339
            ','.join(size) if isinstance(size, tuple) else size))
340
    for i, unit in enumerate(units):
341
        if size < step or i + 1 == len(units):
342
            break
343
        size /= fstep
344
    s = ('%.2f' % size)
345
    s = s.replace('%s' % step, '%s.99' % (step - 1)) if size <= fstep else s
346
    while '.' in s and s[-1] in ('0', '.'):
347
        s = s[:-1]
348
    return s + unit
349

    
350

    
351
def to_bytes(size, format):
352
    """
353
    :param size: (float) the size in the given format
354
    :param format: (case insensitive) KiB, KB, MiB, MB, GiB, GB, TiB, TB
355

356
    :returns: (int) the size in bytes
357
    :raises ValueError: if invalid size or format
358
    :raises AttributeError: if format is not str
359
    :raises TypeError: if size is not arithmetic or convertible to arithmetic
360
    """
361
    format = format.upper()
362
    if format == 'B':
363
        return int(size)
364
    size = float(size)
365
    units_dc = ('KB', 'MB', 'GB', 'TB')
366
    units_bi = ('KIB', 'MIB', 'GIB', 'TIB')
367

    
368
    factor = 1024 if format in units_bi else 1000 if format in units_dc else 0
369
    if not factor:
370
        raise ValueError('Invalid data size format %s' % format)
371
    for prefix in ('K', 'M', 'G', 'T'):
372
        size *= factor
373
        if format.startswith(prefix):
374
            break
375
    return int(size)
376

    
377

    
378
def dict2file(d, f, depth=0):
379
    for k, v in d.items():
380
        f.write('%s%s: ' % (' ' * INDENT_TAB * depth, k))
381
        if isinstance(v, dict):
382
            f.write('\n')
383
            dict2file(v, f, depth + 1)
384
        elif isinstance(v, list) or isinstance(v, tuple):
385
            f.write('\n')
386
            list2file(v, f, depth + 1)
387
        else:
388
            f.write('%s\n' % v)
389

    
390

    
391
def list2file(l, f, depth=1):
392
    for item in l:
393
        if isinstance(item, dict):
394
            dict2file(item, f, depth + 1)
395
        elif isinstance(item, list) or isinstance(item, tuple):
396
            list2file(item, f, depth + 1)
397
        else:
398
            f.write('%s%s\n' % (' ' * INDENT_TAB * depth, item))
399

    
400
# Split input auxiliary
401

    
402

    
403
def _parse_with_regex(line, regex):
404
    re_parser = regex_compile(regex)
405
    return (re_parser.split(line), re_parser.findall(line))
406

    
407

    
408
def _get_from_parsed(parsed_str):
409
    try:
410
        parsed_str = parsed_str.strip()
411
    except:
412
        return None
413
    return ([parsed_str[1:-1]] if (
414
        parsed_str[0] == parsed_str[-1] and parsed_str[0] in ("'", '"')) else (
415
            parsed_str.split(' '))) if parsed_str else None
416

    
417

    
418
def split_input(line):
419
    if not line:
420
        return []
421
    reg_expr = '\'.*?\'|".*?"|^[\S]*$'
422
    (trivial_parts, interesting_parts) = _parse_with_regex(line, reg_expr)
423
    assert(len(trivial_parts) == 1 + len(interesting_parts))
424
    terms = []
425
    for i, tpart in enumerate(trivial_parts):
426
        part = _get_from_parsed(tpart)
427
        if part:
428
            terms += part
429
        try:
430
            part = _get_from_parsed(interesting_parts[i])
431
        except IndexError:
432
            break
433
        if part:
434
            if tpart and not tpart[-1].endswith(' '):
435
                terms[-1] += ' '.join(part)
436
            else:
437
                terms += part
438
    return terms
439

    
440

    
441
def ask_user(msg, true_resp=('y', ), out=stdout, user_in=stdin):
442
    """Print msg and read user response
443

444
    :param true_resp: (tuple of chars)
445

446
    :returns: (bool) True if reponse in true responses, False otherwise
447
    """
448
    yep = ', '.join(true_resp)
449
    nope = '<not %s>' % yep if 'n' in true_resp or 'N' in true_resp else 'N'
450
    out.write(encode_for_console('%s [%s/%s]: ' % (msg, yep, nope)))
451
    out.flush()
452
    user_response = user_in.readline()
453
    return user_response[0].lower() in [s.lower() for s in true_resp]
454

    
455

    
456
def get_path_size(testpath):
457
    if path.isfile(testpath):
458
        return path.getsize(testpath)
459
    total_size = 0
460
    for top, dirs, files in walk(path.abspath(testpath)):
461
        for f in files:
462
            f = path.join(top, f)
463
            if path.isfile(f):
464
                total_size += path.getsize(f)
465
    return total_size
466

    
467

    
468
def remove_from_items(list_of_dicts, key_to_remove):
469
    for item in list_of_dicts:
470
        assert isinstance(item, dict), 'Item %s not a dict' % item
471
        item.pop(key_to_remove, None)
472

    
473

    
474
def filter_dicts_by_dict(
475
    list_of_dicts, filters,
476
    exact_match=True, case_sensitive=False):
477
    """
478
    :param list_of_dicts: (list) each dict contains "raw" key-value pairs
479

480
    :param filters: (dict) filters in key-value form
481

482
    :param exact_match: (bool) if false, check if the filter value is part of
483
        the actual value
484

485
    :param case_sensitive: (bool) refers to values only (not keys)
486

487
    :returns: (list) only the dicts that match all filters
488
    """
489
    new_dicts = []
490
    for d in list_of_dicts:
491
        if set(filters).difference(d):
492
            continue
493
        match = True
494
        for k, v in filters.items():
495
            dv, v = ('%s' % d[k]), ('%s' % v)
496
            if not case_sensitive:
497
                dv, v = dv.lower(), v.lower()
498
            if not ((
499
                    exact_match and v == dv) or (
500
                    (not exact_match) and v in dv)):
501
                match = False
502
                break
503
        if match:
504
            new_dicts.append(d)
505
    return new_dicts