root / kamaki / cli / utils / __init__.py @ df55e7aa
History | View | Annotate | Download (16.1 kB)
1 |
# Copyright 2011-2014 GRNET S.A. All rights reserved.
|
---|---|
2 |
#
|
3 |
# Redistribution and use in source and binary forms, with or
|
4 |
# without modification, are permitted provided that the following
|
5 |
# conditions are met:
|
6 |
#
|
7 |
# 1. Redistributions of source code must retain the above
|
8 |
# copyright notice, this list of conditions and the following
|
9 |
# disclaimer.
|
10 |
#
|
11 |
# 2. Redistributions in binary form must reproduce the above
|
12 |
# copyright notice, this list of conditions and the following
|
13 |
# disclaimer in the documentation and/or other materials
|
14 |
# provided with the distribution.
|
15 |
#
|
16 |
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
|
17 |
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
18 |
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
|
19 |
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
|
20 |
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
21 |
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
22 |
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
|
23 |
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
|
24 |
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
25 |
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
|
26 |
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
27 |
# POSSIBILITY OF SUCH DAMAGE.
|
28 |
#
|
29 |
# The views and conclusions contained in the software and
|
30 |
# documentation are those of the authors and should not be
|
31 |
# interpreted as representing official policies, either expressed
|
32 |
# or implied, of GRNET S.A.
|
33 |
|
34 |
from sys import stdout, stdin, stderr |
35 |
from re import compile as regex_compile |
36 |
from os import walk, path |
37 |
from json import dumps |
38 |
from kamaki.cli.logger import get_logger |
39 |
from locale import getpreferredencoding |
40 |
|
41 |
from kamaki.cli.errors import raiseCLIError |
42 |
|
43 |
|
44 |
INDENT_TAB = 4
|
45 |
log = get_logger(__name__) |
46 |
pref_enc = getpreferredencoding() |
47 |
|
48 |
|
49 |
suggest = dict(ansicolors=dict( |
50 |
active=False,
|
51 |
url='#install-ansicolors',
|
52 |
description='Add colors to console responses'))
|
53 |
|
54 |
try:
|
55 |
from colors import magenta, red, yellow, bold |
56 |
except ImportError: |
57 |
def dummy(val): |
58 |
return val
|
59 |
red = yellow = magenta = bold = dummy |
60 |
suggest['ansicolors']['active'] = True |
61 |
|
62 |
|
63 |
def _encode_nicely(somestr, encoding, replacement='?'): |
64 |
"""Encode somestr as 'encoding', but don't raise errors (replace with ?)
|
65 |
This method is slow. Us it only for grace.
|
66 |
:param encoding: (str) encode every character in this encoding
|
67 |
:param replacement: (char) replace each char raising encode-decode errs
|
68 |
"""
|
69 |
newstr, err_counter = '', 0 |
70 |
for c in somestr: |
71 |
try:
|
72 |
newc = c.encode(encoding) |
73 |
newstr = '%s%s' % (newstr, newc)
|
74 |
except UnicodeError: |
75 |
newstr = '%s%s' % (newstr, replacement)
|
76 |
err_counter += 1
|
77 |
if err_counter:
|
78 |
log.debug('\t%s character%s failed to be encoded as %s' % (
|
79 |
err_counter, 's' if err_counter > 1 else '', encoding)) |
80 |
return newstr
|
81 |
|
82 |
|
83 |
def DontRaiseUnicodeError(foo): |
84 |
def wrap(self, *args, **kwargs): |
85 |
try:
|
86 |
s = kwargs.pop('s')
|
87 |
except KeyError: |
88 |
try:
|
89 |
s = args[0]
|
90 |
except IndexError: |
91 |
return foo(self, *args, **kwargs) |
92 |
args = args[1:]
|
93 |
try:
|
94 |
s = s.encode(pref_enc) |
95 |
except UnicodeError as ue: |
96 |
log.debug('Encoding(%s): %s' % (pref_enc, ue))
|
97 |
s = _encode_nicely(s, pref_enc, replacement='?')
|
98 |
return foo(self, s, *args, **kwargs) |
99 |
return wrap
|
100 |
|
101 |
|
102 |
def encode_for_console(s, encoding=pref_enc, replacement='?'): |
103 |
if encoding.lower() == 'utf-8': |
104 |
return s
|
105 |
try:
|
106 |
return s.encode(encoding)
|
107 |
except UnicodeError as ue: |
108 |
log.debug('Encoding(%s): %s' % (encoding, ue))
|
109 |
return _encode_nicely(s, encoding, replacement)
|
110 |
|
111 |
|
112 |
def suggest_missing(miss=None, exclude=[]): |
113 |
global suggest
|
114 |
sgs = dict(suggest)
|
115 |
for exc in exclude: |
116 |
try:
|
117 |
sgs.pop(exc) |
118 |
except KeyError: |
119 |
pass
|
120 |
kamaki_docs = 'http://www.synnefo.org/docs/kamaki/latest'
|
121 |
|
122 |
for k, v in (miss, sgs[miss]) if miss else sgs.items(): |
123 |
if v['active'] and stderr.isatty(): |
124 |
stderr.write('Suggestion: you may like to install %s\n' % k)
|
125 |
stderr.write('\t%s\n' % encode_for_console(v['description'])) |
126 |
stderr.write('\tIt is easy, here are the instructions:\n')
|
127 |
stderr.write('\t%s/installation.html%s\n' % (
|
128 |
kamaki_docs, v['url']))
|
129 |
stderr.flush() |
130 |
|
131 |
|
132 |
def guess_mime_type( |
133 |
filename, |
134 |
default_content_type='application/octet-stream',
|
135 |
default_encoding=None):
|
136 |
assert filename, 'Cannot guess mimetype for empty filename' |
137 |
try:
|
138 |
from mimetypes import guess_type |
139 |
ctype, cenc = guess_type(filename) |
140 |
return ctype or default_content_type, cenc or default_encoding |
141 |
except ImportError: |
142 |
stderr.write('WARNING: Cannot import mimetypes, using defaults\n')
|
143 |
stderr.flush() |
144 |
return (default_content_type, default_encoding)
|
145 |
|
146 |
|
147 |
def remove_colors(): |
148 |
global bold
|
149 |
global red
|
150 |
global yellow
|
151 |
global magenta
|
152 |
|
153 |
def dummy(val): |
154 |
return val
|
155 |
red = yellow = magenta = bold = dummy |
156 |
|
157 |
|
158 |
def pretty_keys(d, delim='_', recursive=False): |
159 |
"""<term>delim<term> to <term> <term> transformation"""
|
160 |
new_d = dict(d)
|
161 |
for k, v in d.items(): |
162 |
new_v = new_d.pop(k) |
163 |
new_d[k.replace(delim, ' ').strip()] = pretty_keys(
|
164 |
new_v, delim, True) if ( |
165 |
recursive and isinstance(v, dict)) else new_v |
166 |
return new_d
|
167 |
|
168 |
|
169 |
def print_json(data, out=stdout, encoding=pref_enc): |
170 |
"""Print a list or dict as json in console
|
171 |
|
172 |
:param data: json-dumpable data
|
173 |
|
174 |
:param out: Input/Output stream to dump values into
|
175 |
"""
|
176 |
out.write(dumps(data, indent=INDENT_TAB)) |
177 |
out.write('\n')
|
178 |
out.flush() |
179 |
|
180 |
|
181 |
def print_dict( |
182 |
d, |
183 |
exclude=(), indent=0,
|
184 |
with_enumeration=False, recursive_enumeration=False, out=stdout): |
185 |
"""Pretty-print a dictionary object
|
186 |
<indent>key: <non iterable item>
|
187 |
<indent>key:
|
188 |
<indent + INDENT_TAB><pretty-print iterable>
|
189 |
|
190 |
:param d: (dict)
|
191 |
|
192 |
:param exclude: (iterable of strings) keys to exclude from printing
|
193 |
|
194 |
:param indent: (int) initial indentation (recursive)
|
195 |
|
196 |
:param with_enumeration: (bool) enumerate 1st-level keys
|
197 |
|
198 |
:param recursive_enumeration: (bool) recursively enumerate iterables (does
|
199 |
not enumerate 1st level keys)
|
200 |
|
201 |
:param out: Input/Output stream to dump values into
|
202 |
|
203 |
:raises CLIError: if preconditions fail
|
204 |
"""
|
205 |
assert isinstance(d, dict), 'print_dict input must be a dict' |
206 |
assert indent >= 0, 'print_dict indent must be >= 0' |
207 |
|
208 |
for i, (k, v) in enumerate(d.items()): |
209 |
k = ('%s' % k).strip()
|
210 |
if k in exclude: |
211 |
continue
|
212 |
print_str = ' ' * indent
|
213 |
print_str += '%s.' % (i + 1) if with_enumeration else '' |
214 |
print_str += '%s:' % k
|
215 |
if isinstance(v, dict): |
216 |
out.write(print_str + '\n')
|
217 |
print_dict( |
218 |
v, exclude, indent + INDENT_TAB, |
219 |
recursive_enumeration, recursive_enumeration, out) |
220 |
elif isinstance(v, list) or isinstance(v, tuple): |
221 |
out.write(print_str + '\n')
|
222 |
print_list( |
223 |
v, exclude, indent + INDENT_TAB, |
224 |
recursive_enumeration, recursive_enumeration, out) |
225 |
else:
|
226 |
out.write('%s %s\n' % (print_str, v))
|
227 |
out.flush() |
228 |
|
229 |
|
230 |
def print_list( |
231 |
l, |
232 |
exclude=(), indent=0,
|
233 |
with_enumeration=False, recursive_enumeration=False, out=stdout): |
234 |
"""Pretty-print a list of items
|
235 |
<indent>key: <non iterable item>
|
236 |
<indent>key:
|
237 |
<indent + INDENT_TAB><pretty-print iterable>
|
238 |
|
239 |
:param l: (list)
|
240 |
|
241 |
:param exclude: (iterable of strings) items to exclude from printing
|
242 |
|
243 |
:param indent: (int) initial indentation (recursive)
|
244 |
|
245 |
:param with_enumeration: (bool) enumerate 1st-level items
|
246 |
|
247 |
:param recursive_enumeration: (bool) recursively enumerate iterables (does
|
248 |
not enumerate 1st level keys)
|
249 |
|
250 |
:param out: Input/Output stream to dump values into
|
251 |
|
252 |
:raises CLIError: if preconditions fail
|
253 |
"""
|
254 |
assert isinstance(l, list) or isinstance(l, tuple), ( |
255 |
'print_list prinbts a list or tuple')
|
256 |
assert indent >= 0, 'print_list indent must be >= 0' |
257 |
|
258 |
for i, item in enumerate(l): |
259 |
print_str = ' ' * indent
|
260 |
print_str += '%s.' % (i + 1) if with_enumeration else '' |
261 |
if isinstance(item, dict): |
262 |
if with_enumeration:
|
263 |
out.write(print_str + '\n')
|
264 |
elif i and i < len(l): |
265 |
out.write('\n')
|
266 |
print_dict( |
267 |
item, exclude, |
268 |
indent + (INDENT_TAB if with_enumeration else 0), |
269 |
recursive_enumeration, recursive_enumeration, out) |
270 |
elif isinstance(item, list) or isinstance(item, tuple): |
271 |
if with_enumeration:
|
272 |
out.write(print_str + '\n')
|
273 |
elif i and i < len(l): |
274 |
out.write('\n')
|
275 |
print_list( |
276 |
item, exclude, indent + INDENT_TAB, |
277 |
recursive_enumeration, recursive_enumeration, out) |
278 |
else:
|
279 |
item = ('%s' % item).strip()
|
280 |
if item in exclude: |
281 |
continue
|
282 |
out.write('%s%s\n' % (print_str, item))
|
283 |
out.flush() |
284 |
out.flush() |
285 |
|
286 |
|
287 |
def print_items( |
288 |
items, title=('id', 'name'), |
289 |
with_enumeration=False, with_redundancy=False, out=stdout): |
290 |
"""print dict or list items in a list, using some values as title
|
291 |
Objects of next level don't inherit enumeration (default: off) or titles
|
292 |
|
293 |
:param items: (list) items are lists or dict
|
294 |
|
295 |
:param title: (tuple) keys to use their values as title
|
296 |
|
297 |
:param with_enumeration: (boolean) enumerate items (order id on title)
|
298 |
|
299 |
:param with_redundancy: (boolean) values in title also appear on body
|
300 |
|
301 |
:param out: Input/Output stream to dump values into
|
302 |
"""
|
303 |
if not items: |
304 |
return
|
305 |
if not (isinstance(items, dict) or isinstance(items, list) or isinstance( |
306 |
items, tuple)):
|
307 |
out.write('%s\n' % items)
|
308 |
out.flush() |
309 |
return
|
310 |
|
311 |
for i, item in enumerate(items): |
312 |
if with_enumeration:
|
313 |
out.write('%s. ' % (i + 1)) |
314 |
if isinstance(item, dict): |
315 |
item = dict(item)
|
316 |
title = sorted(set(title).intersection(item)) |
317 |
pick = item.get if with_redundancy else item.pop |
318 |
header = ' '.join('%s' % pick(key) for key in title) |
319 |
out.write((unicode(bold(header) if header else '') + '\n')) |
320 |
print_dict(item, indent=INDENT_TAB, out=out) |
321 |
elif isinstance(item, list) or isinstance(item, tuple): |
322 |
print_list(item, indent=INDENT_TAB, out=out) |
323 |
else:
|
324 |
out.write(' %s\n' % item)
|
325 |
out.flush() |
326 |
out.flush() |
327 |
|
328 |
|
329 |
def format_size(size, decimal_factors=False): |
330 |
units = ('B', 'KB', 'MB', 'GB', 'TB') if decimal_factors else ( |
331 |
'B', 'KiB', 'MiB', 'GiB', 'TiB') |
332 |
step = 1000 if decimal_factors else 1024 |
333 |
fstep = float(step)
|
334 |
try:
|
335 |
size = float(size)
|
336 |
except (ValueError, TypeError) as err: |
337 |
raiseCLIError(err, 'Cannot format %s in bytes' % (
|
338 |
','.join(size) if isinstance(size, tuple) else size)) |
339 |
for i, unit in enumerate(units): |
340 |
if size < step or i + 1 == len(units): |
341 |
break
|
342 |
size /= fstep |
343 |
s = ('%.2f' % size)
|
344 |
s = s.replace('%s' % step, '%s.99' % (step - 1)) if size <= fstep else s |
345 |
while '.' in s and s[-1] in ('0', '.'): |
346 |
s = s[:-1]
|
347 |
return s + unit
|
348 |
|
349 |
|
350 |
def to_bytes(size, format): |
351 |
"""
|
352 |
:param size: (float) the size in the given format
|
353 |
:param format: (case insensitive) KiB, KB, MiB, MB, GiB, GB, TiB, TB
|
354 |
|
355 |
:returns: (int) the size in bytes
|
356 |
:raises ValueError: if invalid size or format
|
357 |
:raises AttributeError: if format is not str
|
358 |
:raises TypeError: if size is not arithmetic or convertible to arithmetic
|
359 |
"""
|
360 |
format = format.upper() |
361 |
if format == 'B': |
362 |
return int(size) |
363 |
size = float(size)
|
364 |
units_dc = ('KB', 'MB', 'GB', 'TB') |
365 |
units_bi = ('KIB', 'MIB', 'GIB', 'TIB') |
366 |
|
367 |
factor = 1024 if format in units_bi else 1000 if format in units_dc else 0 |
368 |
if not factor: |
369 |
raise ValueError('Invalid data size format %s' % format) |
370 |
for prefix in ('K', 'M', 'G', 'T'): |
371 |
size *= factor |
372 |
if format.startswith(prefix):
|
373 |
break
|
374 |
return int(size) |
375 |
|
376 |
|
377 |
def dict2file(d, f, depth=0): |
378 |
for k, v in d.items(): |
379 |
f.write('%s%s: ' % (' ' * INDENT_TAB * depth, k)) |
380 |
if isinstance(v, dict): |
381 |
f.write('\n')
|
382 |
dict2file(v, f, depth + 1)
|
383 |
elif isinstance(v, list) or isinstance(v, tuple): |
384 |
f.write('\n')
|
385 |
list2file(v, f, depth + 1)
|
386 |
else:
|
387 |
f.write('%s\n' % v)
|
388 |
|
389 |
|
390 |
def list2file(l, f, depth=1): |
391 |
for item in l: |
392 |
if isinstance(item, dict): |
393 |
dict2file(item, f, depth + 1)
|
394 |
elif isinstance(item, list) or isinstance(item, tuple): |
395 |
list2file(item, f, depth + 1)
|
396 |
else:
|
397 |
f.write('%s%s\n' % (' ' * INDENT_TAB * depth, item)) |
398 |
|
399 |
# Split input auxiliary
|
400 |
|
401 |
|
402 |
def _parse_with_regex(line, regex): |
403 |
re_parser = regex_compile(regex) |
404 |
return (re_parser.split(line), re_parser.findall(line))
|
405 |
|
406 |
|
407 |
def _get_from_parsed(parsed_str): |
408 |
try:
|
409 |
parsed_str = parsed_str.strip() |
410 |
except:
|
411 |
return None |
412 |
return ([parsed_str[1:-1]] if ( |
413 |
parsed_str[0] == parsed_str[-1] and parsed_str[0] in ("'", '"')) else ( |
414 |
parsed_str.split(' '))) if parsed_str else None |
415 |
|
416 |
|
417 |
def split_input(line): |
418 |
if not line: |
419 |
return []
|
420 |
reg_expr = '\'.*?\'|".*?"|^[\S]*$'
|
421 |
(trivial_parts, interesting_parts) = _parse_with_regex(line, reg_expr) |
422 |
assert(len(trivial_parts) == 1 + len(interesting_parts)) |
423 |
terms = [] |
424 |
for i, tpart in enumerate(trivial_parts): |
425 |
part = _get_from_parsed(tpart) |
426 |
if part:
|
427 |
terms += part |
428 |
try:
|
429 |
part = _get_from_parsed(interesting_parts[i]) |
430 |
except IndexError: |
431 |
break
|
432 |
if part:
|
433 |
if tpart and not tpart[-1].endswith(' '): |
434 |
terms[-1] += ' '.join(part) |
435 |
else:
|
436 |
terms += part |
437 |
return terms
|
438 |
|
439 |
|
440 |
def ask_user(msg, true_resp=('y', ), out=stdout, user_in=stdin): |
441 |
"""Print msg and read user response
|
442 |
|
443 |
:param true_resp: (tuple of chars)
|
444 |
|
445 |
:returns: (bool) True if reponse in true responses, False otherwise
|
446 |
"""
|
447 |
yep = ', '.join(true_resp)
|
448 |
nope = '<not %s>' % yep if 'n' in true_resp or 'N' in true_resp else 'N' |
449 |
out.write('%s [%s/%s]: ' % (msg, yep, nope))
|
450 |
out.flush() |
451 |
user_response = user_in.readline() |
452 |
return user_response[0].lower() in [s.lower() for s in true_resp] |
453 |
|
454 |
|
455 |
def get_path_size(testpath): |
456 |
if path.isfile(testpath):
|
457 |
return path.getsize(testpath)
|
458 |
total_size = 0
|
459 |
for top, dirs, files in walk(path.abspath(testpath)): |
460 |
for f in files: |
461 |
f = path.join(top, f) |
462 |
if path.isfile(f):
|
463 |
total_size += path.getsize(f) |
464 |
return total_size
|
465 |
|
466 |
|
467 |
def remove_from_items(list_of_dicts, key_to_remove): |
468 |
for item in list_of_dicts: |
469 |
assert isinstance(item, dict), 'Item %s not a dict' % item |
470 |
item.pop(key_to_remove, None)
|
471 |
|
472 |
|
473 |
def filter_dicts_by_dict( |
474 |
list_of_dicts, filters, |
475 |
exact_match=True, case_sensitive=False): |
476 |
"""
|
477 |
:param list_of_dicts: (list) each dict contains "raw" key-value pairs
|
478 |
|
479 |
:param filters: (dict) filters in key-value form
|
480 |
|
481 |
:param exact_match: (bool) if false, check if the filter value is part of
|
482 |
the actual value
|
483 |
|
484 |
:param case_sensitive: (bool) refers to values only (not keys)
|
485 |
|
486 |
:returns: (list) only the dicts that match all filters
|
487 |
"""
|
488 |
new_dicts = [] |
489 |
for d in list_of_dicts: |
490 |
if set(filters).difference(d): |
491 |
continue
|
492 |
match = True
|
493 |
for k, v in filters.items(): |
494 |
dv, v = ('%s' % d[k]), ('%s' % v) |
495 |
if not case_sensitive: |
496 |
dv, v = dv.lower(), v.lower() |
497 |
if not (( |
498 |
exact_match and v == dv) or ( |
499 |
(not exact_match) and v in dv)): |
500 |
match = False
|
501 |
break
|
502 |
if match:
|
503 |
new_dicts.append(d) |
504 |
return new_dicts
|