1 # Copyright 2012-2013 GRNET S.A. All rights reserved.
3 # Redistribution and use in source and binary forms, with or
4 # without modification, are permitted provided that the following
7 # 1. Redistributions of source code must retain the above
8 # copyright notice, this list of conditions and the following
11 # 2. Redistributions in binary form must reproduce the above
12 # copyright notice, this list of conditions and the following
13 # disclaimer in the documentation and/or other materials
14 # provided with the distribution.
16 # THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
17 # OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
19 # PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
20 # CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21 # SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22 # LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
23 # USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
24 # AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25 # LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
26 # ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
27 # POSSIBILITY OF SUCH DAMAGE.
29 # The views and conclusions contained in the software and
30 # documentation are those of the authors and should not be
31 # interpreted as representing official policies, either expressed
32 # or implied, of GRNET S.A.command
34 from json import load, dumps
36 from logging import getLogger
37 from io import StringIO
38 from pydoc import pager
40 from kamaki.cli import command
41 from kamaki.cli.command_tree import CommandTree
42 from kamaki.cli.utils import filter_dicts_by_dict
43 from kamaki.clients.image import ImageClient
44 from kamaki.clients.pithos import PithosClient
45 from kamaki.clients import ClientError
46 from kamaki.cli.argument import (
47 FlagArgument, ValueArgument, RepeatableArgument, KeyValueArgument,
48 IntArgument, ProgressBarArgument)
49 from kamaki.cli.commands.cyclades import _init_cyclades
50 from kamaki.cli.errors import (
51 raiseCLIError, CLIBaseUrlError, CLIInvalidArgument)
52 from kamaki.cli.commands import _command_init, errors, addLogSettings
53 from kamaki.cli.commands import (
54 _optional_output_cmd, _optional_json, _name_filter, _id_filter)
57 image_cmds = CommandTree('image', 'Cyclades/Plankton API image commands')
58 imagecompute_cmds = CommandTree(
59 'imagecompute', 'Cyclades/Compute API image commands')
60 _commands = [image_cmds, imagecompute_cmds]
64 'Kamaki commands to:',
65 ' get current user id: /user authenticate',
66 ' check available containers: /file list',
67 ' create a new container: /file create <container>',
68 ' check container contents: /file list <container>',
69 ' upload files: /file upload <image file> <container>',
70 ' register an image: /image register <image name> <container>:<path>']
72 about_image_id = ['To see a list of available image ids: /image list']
75 log = getLogger(__name__)
78 class _init_image(_command_init):
82 if getattr(self, 'cloud', None):
83 img_url = self._custom_url('image') or self._custom_url('plankton')
85 token = self._custom_token('image') or self._custom_token(
86 'plankton') or self.config.get_cloud(self.cloud, 'token')
87 self.client = ImageClient(base_url=img_url, token=token)
89 if getattr(self, 'auth_base', False):
90 plankton_endpoints = self.auth_base.get_service_endpoints(
91 self._custom_type('image') or self._custom_type(
92 'plankton') or 'image',
93 self._custom_version('image') or self._custom_version(
95 base_url = plankton_endpoints['publicURL']
96 token = self.auth_base.token
98 raise CLIBaseUrlError(service='plankton')
99 self.client = ImageClient(base_url=base_url, token=token)
105 # Plankton Image Commands
108 def _validate_image_meta(json_dict, return_str=False):
110 :param json_dict" (dict) json-formated, of the form
111 {"key1": "val1", "key2": "val2", ...}
113 :param return_str: (boolean) if true, return a json dump
115 :returns: (dict) if return_str is not True, else return str
117 :raises TypeError, AttributeError: Invalid json format
119 :raises AssertionError: Valid json but invalid image properties dict
121 json_str = dumps(json_dict, indent=2)
122 for k, v in json_dict.items():
123 if k.lower() == 'properties':
124 for pk, pv in v.items():
125 prop_ok = not (isinstance(pv, dict) or isinstance(pv, list))
126 assert prop_ok, 'Invalid property value for key %s' % pk
127 key_ok = not (' ' in k or '-' in k)
128 assert key_ok, 'Invalid property key %s' % k
130 meta_ok = not (isinstance(v, dict) or isinstance(v, list))
131 assert meta_ok, 'Invalid value for meta key %s' % k
132 meta_ok = ' ' not in k
133 assert meta_ok, 'Invalid meta key [%s]' % k
134 json_dict[k] = '%s' % v
135 return json_str if return_str else json_dict
138 def _load_image_meta(filepath):
140 :param filepath: (str) the (relative) path of the metafile
142 :returns: (dict) json_formated
144 :raises TypeError, AttributeError: Invalid json format
146 :raises AssertionError: Valid json but invalid image properties dict
148 with open(path.abspath(filepath)) as f:
151 return _validate_image_meta(meta_dict)
152 except AssertionError:
153 log.debug('Failed to load properties from file %s' % filepath)
157 def _validate_image_location(location):
159 :param location: (str) pithos://<user-id>/<container>/<image-path>
161 :returns: (<user-id>, <container>, <image-path>)
163 :raises AssertionError: if location is invalid
166 msg = 'Invalid prefix for location %s , try: %s' % (location, prefix)
167 assert location.startswith(prefix), msg
168 service, sep, rest = location.partition('://')
169 assert sep and rest, 'Location %s is missing user-id' % location
170 uuid, sep, rest = rest.partition('/')
171 assert sep and rest, 'Location %s is missing container' % location
172 container, sep, img_path = rest.partition('/')
173 assert sep and img_path, 'Location %s is missing image path' % location
174 return uuid, container, img_path
178 class image_list(_init_image, _optional_json, _name_filter, _id_filter):
179 """List images accessible by user"""
183 'status', 'container_format', 'disk_format', 'size')
186 detail=FlagArgument('show detailed output', ('-l', '--details')),
187 container_format=ValueArgument(
188 'filter by container format',
189 '--container-format'),
190 disk_format=ValueArgument('filter by disk format', '--disk-format'),
191 size_min=IntArgument('filter by minimum size', '--size-min'),
192 size_max=IntArgument('filter by maximum size', '--size-max'),
193 status=ValueArgument('filter by status', '--status'),
194 owner=ValueArgument('filter by owner', '--owner'),
195 owner_name=ValueArgument('filter by owners username', '--owner-name'),
197 'order by FIELD ( - to reverse order)',
200 limit=IntArgument('limit number of listed images', ('-n', '--number')),
202 'output results in pages (-n to set items per page, default 10)',
204 enum=FlagArgument('Enumerate results', '--enumerate'),
205 prop=KeyValueArgument('filter by property key=value', ('--property')),
206 prop_like=KeyValueArgument(
207 'fliter by property key=value where value is part of actual value',
208 ('--property-like')),
209 image_ID_for_members=ValueArgument(
210 'List members of an image', '--members-of')
213 def _filter_by_owner(self, images):
214 ouuid = self['owner'] or self._username2uuid(self['owner_name'])
215 return filter_dicts_by_dict(images, dict(owner=ouuid))
217 def _add_owner_name(self, images):
218 uuids = self._uuids2usernames(
219 list(set([img['owner'] for img in images])))
221 img['owner'] += ' (%s)' % uuids[img['owner']]
224 def _filter_by_properties(self, images):
227 props = [dict(img['properties'])]
229 props = filter_dicts_by_dict(props, self['prop'])
230 if props and self['prop_like']:
231 props = filter_dicts_by_dict(
232 props, self['prop_like'], exact_match=False)
234 new_images.append(img)
237 def _members(self, image_id):
238 members = self.client.list_members(image_id)
239 if not (self['json_output'] or self['output_format']):
240 uuids = [member['member_id'] for member in members]
241 usernames = self._uuids2usernames(uuids)
242 for member in members:
243 member['member_id'] += ' (%s)' % usernames[member['member_id']]
244 self._print(members, title=('member_id',))
247 @errors.cyclades.connection
249 super(self.__class__, self)._run()
250 if self['image_ID_for_members']:
251 return self._members(self['image_ID_for_members'])
259 'status']).intersection(self.arguments):
260 filters[arg] = self[arg]
262 order = self['order']
263 detail = self['detail'] or (
264 self['prop'] or self['prop_like']) or (
265 self['owner'] or self['owner_name'])
267 images = self.client.list_public(detail, filters, order)
269 if self['owner'] or self['owner_name']:
270 images = self._filter_by_owner(images)
271 if self['prop'] or self['prop_like']:
272 images = self._filter_by_properties(images)
273 images = self._filter_by_id(images)
274 images = self._non_exact_name_filter(images)
276 if self['detail'] and not (
277 self['json_output'] or self['output_format']):
278 images = self._add_owner_name(images)
279 elif detail and not self['detail']:
281 for key in set(img).difference(self.PERMANENTS):
283 kwargs = dict(with_enumeration=self['enum'])
285 images = images[:self['limit']]
287 kwargs['out'] = StringIO()
289 self._print(images, **kwargs)
291 pager(kwargs['out'].getvalue())
294 super(self.__class__, self)._run()
299 class image_info(_init_image, _optional_json):
300 """Get image metadata"""
303 @errors.plankton.connection
305 def _run(self, image_id):
306 meta = self.client.get_meta(image_id)
307 if not (self['json_output'] or self['output_format']):
308 meta['owner'] += ' (%s)' % self._uuid2username(meta['owner'])
309 self._print(meta, self.print_dict)
311 def main(self, image_id):
312 super(self.__class__, self)._run()
313 self._run(image_id=image_id)
317 class image_modify(_init_image, _optional_output_cmd):
318 """Add / update metadata and properties for an image
319 The original image preserves the values that are not affected
323 image_name=ValueArgument('Change name', '--name'),
324 disk_format=ValueArgument('Change disk format', '--disk-format'),
325 container_format=ValueArgument(
326 'Change container format', '--container-format'),
327 status=ValueArgument('Change status', '--status'),
328 publish=FlagArgument('Publish the image', '--publish'),
329 unpublish=FlagArgument('Unpublish the image', '--unpublish'),
330 property_to_set=KeyValueArgument(
331 'set property in key=value form (can be repeated)',
332 ('-p', '--property-set')),
333 property_to_del=RepeatableArgument(
334 'Delete property by key (can be repeated)', '--property-del'),
335 member_ID_to_add=RepeatableArgument(
336 'Add member to image (can be repeated)', '--member-add'),
337 member_ID_to_remove=RepeatableArgument(
338 'Remove a member (can be repeated)', '--member-del'),
341 'image_name', 'disk_format', 'container_format', 'status', 'publish',
342 'unpublish', 'property_to_set', 'member_ID_to_add',
343 'member_ID_to_remove']
346 @errors.plankton.connection
348 def _run(self, image_id):
349 for mid in (self['member_ID_to_add'] or []):
350 self.client.add_member(image_id, mid)
351 for mid in (self['member_ID_to_remove'] or []):
352 self.client.remove_member(image_id, mid)
353 meta = self.client.get_meta(image_id)
354 for k, v in self['property_to_set'].items():
355 meta['properties'][k.upper()] = v
356 for k in (self['property_to_del'] or []):
357 meta['properties'][k.upper()] = None
358 self._optional_output(self.client.update_image(
360 name=self['image_name'],
361 disk_format=self['disk_format'],
362 container_format=self['container_format'],
363 status=self['status'],
364 public=self['publish'] or self['unpublish'] or None,
365 **meta['properties']))
366 if self['with_output']:
367 self._optional_output(self.get_image_details(image_id))
369 def main(self, image_id):
370 super(self.__class__, self)._run()
371 self._run(image_id=image_id)
374 class PithosLocationArgument(ValueArgument):
375 """Resolve pithos url, return in the form pithos://uuid/container/path"""
378 self, help=None, parsed_name=None, default=None, user_uuid=None):
379 super(PithosLocationArgument, self).__init__(
380 help=help, parsed_name=parsed_name, default=default)
381 self.uuid, self.container, self.path = user_uuid, None, None
383 def setdefault(self, term, value):
384 if not getattr(self, term, None):
385 setattr(self, term, value)
389 return 'pithos://%s/%s/%s' % (self.uuid, self.container, self.path)
392 def value(self, location):
394 from kamaki.cli.commands.pithos import _pithos_container as pc
396 uuid, self.container, self.path = pc._resolve_pithos_url(
398 self.uuid = uuid or self.uuid
399 for term in ('container', 'path'):
400 assert getattr(self, term, None), 'No %s' % term
401 except Exception as e:
402 raise CLIInvalidArgument(
403 'Invalid Pithos+ location %s (%s)' % (location, e),
405 'The image location must be a valid Pithos+',
406 'location. There are two valid formats:',
407 ' pithos://USER_UUID/CONTAINER/PATH',
410 'To see all containers:',
411 ' [kamaki] container list',
412 'To list the contents of a container:',
413 ' [kamaki] container list CONTAINER'])
417 class image_register(_init_image, _optional_json):
418 """(Re)Register an image file to an Image service
419 The image file must be stored at a pithos repository
420 Some metadata can be set by user (e.g., disk-format) while others are set
421 only automatically (e.g., image id). There are also some custom user
422 metadata, called properties.
423 A register command creates a remote meta file at
424 /<container>/<image path>.meta
425 Users may download and edit this file and use it to re-register one or more
427 In case of a meta file, runtime arguments for metadata or properties
428 override meta file settings.
431 container_info_cache = {}
434 checksum=ValueArgument('Set image checksum', '--checksum'),
435 container_format=ValueArgument(
436 'Set container format', '--container-format'),
437 disk_format=ValueArgument('Set disk format', '--disk-format'),
438 owner_name=ValueArgument('Set user uuid by user name', '--owner-name'),
439 properties=KeyValueArgument(
440 'Add property (user-specified metadata) in key=value form'
442 ('-p', '--property')),
443 is_public=FlagArgument('Mark image as public', '--public'),
444 size=IntArgument('Set image size in bytes', '--size'),
445 metafile=ValueArgument(
446 'Load metadata from a json-formated file <img-file>.meta :'
447 '{"key1": "val1", "key2": "val2", ..., "properties: {...}"}',
449 metafile_force=FlagArgument(
450 'Overide remote metadata file', ('-f', '--force')),
451 no_metafile_upload=FlagArgument(
452 'Do not store metadata in remote meta file',
453 ('--no-metafile-upload')),
454 container=ValueArgument(
455 'Pithos+ container containing the image file',
456 ('-C', '--container')),
457 uuid=ValueArgument('Custom user uuid', '--uuid'),
458 local_image_path=ValueArgument(
459 'Local image file path to upload and register '
460 '(still need target file in the form /container/remote-path )',
461 '--upload-image-file'),
462 progress_bar=ProgressBarArgument(
463 'Do not use progress bar', '--no-progress-bar', default=False),
464 name=ValueArgument('The name of the new image', '--name'),
465 pithos_location=PithosLocationArgument(
466 'The Pithos+ image location to put the image at. Format: '
467 'pithos://USER_UUID/CONTAINER/IMAGE or '
471 required = ('name', 'pithos_location')
473 def _get_pithos_client(self, locator):
474 if self['no_metafile_upload']:
476 ptoken = self.client.token
477 if getattr(self, 'auth_base', False):
478 pithos_endpoints = self.auth_base.get_service_endpoints(
480 purl = pithos_endpoints['publicURL']
482 purl = self.config.get_cloud('pithos', 'url')
484 raise CLIBaseUrlError(service='pithos')
485 return PithosClient(purl, ptoken, locator.uuid, locator.container)
487 def _load_params_from_file(self, location):
488 params, properties = dict(), dict()
489 pfile = self['metafile']
492 for k, v in _load_image_meta(pfile).items():
493 key = k.lower().replace('-', '_')
494 if key == 'properties':
495 for pk, pv in v.items():
496 properties[pk.upper().replace('-', '_')] = pv
499 elif key == 'location':
505 except Exception as e:
506 raiseCLIError(e, 'Invalid json metadata config file')
507 return params, properties, location
509 def _load_params_from_args(self, params, properties):
516 'is_public']).intersection(self.arguments):
517 params[key] = self[key]
518 for k, v in self['properties'].items():
519 properties[k.upper().replace('-', '_')] = v
522 @errors.plankton.connection
523 def _run(self, name, location):
524 locator = self.arguments['pithos_location']
525 if self['local_image_path']:
526 with open(self['local_image_path']) as f:
527 pithos = self._get_pithos_client(locator)
528 (pbar, upload_cb) = self._safe_progress_bar('Uploading')
530 hash_bar = pbar.clone()
531 hash_cb = hash_bar.get_generator('Calculating hashes')
532 pithos.upload_object(
534 hash_cb=hash_cb, upload_cb=upload_cb,
535 container_info_cache=self.container_info_cache)
538 (params, properties, new_loc) = self._load_params_from_file(location)
539 if location != new_loc:
540 locator.value = new_loc
541 self._load_params_from_args(params, properties)
542 pclient = self._get_pithos_client(locator)
544 #check if metafile exists
545 meta_path = '%s.meta' % locator.path
546 if pclient and not self['metafile_force']:
548 pclient.get_object_info(meta_path)
550 'Metadata file /%s/%s already exists, abort' % (
551 locator.container, meta_path),
552 details=['Registration ABORTED', 'Try -f to overwrite'])
553 except ClientError as ce:
559 r = self.client.register(name, location, params, properties)
560 except ClientError as ce:
561 if ce.status in (400, 404):
563 ce, 'Nonexistent image file location\n\t%s' % location,
565 'Does the image file %s exist at container %s ?' % (
567 locator.container)] + howto_image_file)
569 r['owner'] += ' (%s)' % self._uuid2username(r['owner'])
570 self._print(r, self.print_dict)
572 #upload the metadata file
575 meta_headers = pclient.upload_from_string(
576 meta_path, dumps(r, indent=2),
577 container_info_cache=self.container_info_cache)
580 'Failed to dump metafile /%s/%s' % (
581 locator.container, meta_path))
583 if self['json_output'] or self['output_format']:
584 self.print_json(dict(
585 metafile_location='/%s/%s' % (
586 locator.container, meta_path),
587 headers=meta_headers))
589 self.error('Metadata file uploaded as /%s/%s (version %s)' % (
592 meta_headers['x-object-version']))
595 super(self.__class__, self)._run()
596 self.arguments['pithos_location'].setdefault(
597 'uuid', self.auth_base.user_term('id'))
598 self._run(self['name'], self['pithos_location'])
602 class image_unregister(_init_image, _optional_output_cmd):
603 """Unregister an image (does not delete the image file)"""
606 @errors.plankton.connection
608 def _run(self, image_id):
609 self._optional_output(self.client.unregister(image_id))
611 def main(self, image_id):
612 super(self.__class__, self)._run()
613 self._run(image_id=image_id)
616 # Compute Image Commands
618 @command(imagecompute_cmds)
619 class imagecompute_list(
620 _init_cyclades, _optional_json, _name_filter, _id_filter):
623 PERMANENTS = ('id', 'name')
626 detail=FlagArgument('show detailed output', ('-l', '--details')),
627 limit=IntArgument('limit number listed images', ('-n', '--number')),
628 more=FlagArgument('handle long lists of results', '--more'),
629 enum=FlagArgument('Enumerate results', '--enumerate'),
630 user_id=ValueArgument('filter by user_id', '--user-id'),
631 user_name=ValueArgument('filter by username', '--user-name'),
632 meta=KeyValueArgument(
633 'filter by metadata key=value (can be repeated)', ('--metadata')),
634 meta_like=KeyValueArgument(
635 'filter by metadata key=value (can be repeated)',
639 def _filter_by_metadata(self, images):
642 meta = [dict(img['metadata'])]
644 meta = filter_dicts_by_dict(meta, self['meta'])
645 if meta and self['meta_like']:
646 meta = filter_dicts_by_dict(
647 meta, self['meta_like'], exact_match=False)
649 new_images.append(img)
652 def _filter_by_user(self, images):
653 uuid = self['user_id'] or self._username2uuid(self['user_name'])
654 return filter_dicts_by_dict(images, dict(user_id=uuid))
656 def _add_name(self, images, key='user_id'):
657 uuids = self._uuids2usernames(
658 list(set([img[key] for img in images])))
660 img[key] += ' (%s)' % uuids[img[key]]
664 @errors.cyclades.connection
666 withmeta = bool(self['meta'] or self['meta_like'])
667 withuser = bool(self['user_id'] or self['user_name'])
668 detail = self['detail'] or withmeta or withuser
669 images = self.client.list_images(detail)
670 images = self._filter_by_name(images)
671 images = self._filter_by_id(images)
673 images = self._filter_by_user(images)
675 images = self._filter_by_metadata(images)
676 if self['detail'] and not (
677 self['json_output'] or self['output_format']):
678 images = self._add_name(self._add_name(images, 'tenant_id'))
679 elif detail and not self['detail']:
681 for key in set(img).difference(self.PERMANENTS):
683 kwargs = dict(with_enumeration=self['enum'])
685 images = images[:self['limit']]
687 kwargs['out'] = StringIO()
689 self._print(images, **kwargs)
691 pager(kwargs['out'].getvalue())
694 super(self.__class__, self)._run()
698 @command(imagecompute_cmds)
699 class imagecompute_info(_init_cyclades, _optional_json):
700 """Get detailed information on an image"""
703 @errors.cyclades.connection
705 def _run(self, image_id):
706 image = self.client.get_image_details(image_id)
707 uuids = [image['user_id'], image['tenant_id']]
708 usernames = self._uuids2usernames(uuids)
709 image['user_id'] += ' (%s)' % usernames[image['user_id']]
710 image['tenant_id'] += ' (%s)' % usernames[image['tenant_id']]
711 self._print(image, self.print_dict)
713 def main(self, image_id):
714 super(self.__class__, self)._run()
715 self._run(image_id=image_id)
718 @command(imagecompute_cmds)
719 class imagecompute_delete(_init_cyclades, _optional_output_cmd):
720 """Delete an image (WARNING: image file is also removed)"""
723 @errors.cyclades.connection
725 def _run(self, image_id):
726 self._optional_output(self.client.delete_image(image_id))
728 def main(self, image_id):
729 super(self.__class__, self)._run()
730 self._run(image_id=image_id)
733 @command(imagecompute_cmds)
734 class imagecompute_modify(_init_cyclades, _optional_output_cmd):
735 """Modify image properties (metadata)"""
738 property_to_add=KeyValueArgument(
739 'Add property in key=value format (can be repeated)',
741 property_to_del=RepeatableArgument(
742 'Delete property by key (can be repeated)',
745 required = ['property_to_add', 'property_to_del']
748 @errors.cyclades.connection
750 def _run(self, image_id):
751 if self['property_to_add']:
752 self.client.update_image_metadata(
753 image_id, **self['property_to_add'])
754 for key in (self['property_to_del'] or []):
755 self.client.delete_image_metadata(image_id, key)
756 if self['with_output']:
757 self._optional_output(self.client.get_image_details(image_id))
759 def main(self, image_id):
760 super(self.__class__, self)._run()
761 self._run(image_id=image_id)