Revision edc1182f

b/kamaki/cli/commands/errors.py
513 513
    @classmethod
514 514
    def local_path(this, foo):
515 515
        def _raise(self, *args, **kwargs):
516
            local_path = kwargs.get('local_path', '<None>')
516
            local_path = kwargs.get('local_path', None)
517 517
            try:
518 518
                return foo(self, *args, **kwargs)
519 519
            except IOError as ioe:
b/kamaki/cli/commands/pithos.py
33 33

  
34 34
from io import StringIO
35 35
from pydoc import pager
36
from os import path, walk, makedirs
36 37

  
37 38
from kamaki.clients.pithos import PithosClient, ClientError
38 39

  
......
41 42
from kamaki.cli.commands import (
42 43
    _command_init, errors, addLogSettings, DontRaiseKeyError, _optional_json,
43 44
    _name_filter, _optional_output_cmd)
44
from kamaki.cli.errors import (
45
    CLIBaseUrlError, CLIError)
45
from kamaki.cli.errors import CLIBaseUrlError, CLIError, CLIInvalidArgument
46 46
from kamaki.cli.argument import (
47
    FlagArgument, IntArgument, ValueArgument, DateArgument)
48
from kamaki.cli.utils import (format_size, bold)
47
    FlagArgument, IntArgument, ValueArgument, DateArgument,
48
    ProgressBarArgument, RepeatableArgument)
49
from kamaki.cli.utils import (
50
    format_size, bold, get_path_size, guess_mime_type)
49 51

  
50 52
file_cmds = CommandTree('file', 'Pithos+/Storage object level API commands')
51 53
container_cmds = CommandTree(
......
142 144
        /CONTAINER/OBJECT_PATH
143 145
        return account, container, path
144 146
        """
145
        account, container, path, prefix = '', '', url, 'pithos://'
147
        account, container, obj_path, prefix = '', '', url, 'pithos://'
146 148
        if url.startswith(prefix):
147 149
            account, sep, url = url[len(prefix):].partition('/')
148 150
            url = '/%s' % url
149 151
        if url.startswith('/'):
150
            container, sep, path = url[1:].partition('/')
151
        return account, container, path
152
            container, sep, obj_path = url[1:].partition('/')
153
        return account, container, obj_path
152 154

  
153 155
    def _run(self, url=None):
154 156
        acc, con, self.path = self._resolve_pithos_url(url or '')
......
193 195
            empty_space = ' ' * (len(str(len(object_list))) - len(str(index)))
194 196
            if 'subdir' in obj:
195 197
                continue
196
            if obj['content_type'] == 'application/directory':
197
                isDir, size = True, 'D'
198
            if self._is_dir(obj):
199
                size = 'D'
198 200
            else:
199
                isDir, size = False, format_size(obj['bytes'])
201
                size = format_size(obj['bytes'])
200 202
                pretty_obj['bytes'] = '%s (%s)' % (obj['bytes'], size)
201 203
            oname = obj['name'] if self['more'] else bold(obj['name'])
202 204
            prfx = ('%s%s. ' % (empty_space, index)) if self['enum'] else ''
......
206 208
                self.writeln()
207 209
            else:
208 210
                oname = '%s%9s %s' % (prfx, size, oname)
209
                oname += '/' if isDir else u''
211
                oname += '/' if self._is_dir(obj) else u''
210 212
                self.writeln(oname)
211 213

  
212 214
    @errors.generic.all
......
509 511
        super(file_move, self)._run(
510 512
            source_path_or_url, destination_path_or_url or '')
511 513
        self._run()
514

  
515

  
516
@command(file_cmds)
517
class file_append(_pithos_container, _optional_output_cmd):
518
    """Append local file to (existing) remote object
519
    The remote object should exist.
520
    If the remote object is a directory, it is transformed into a file.
521
    In the later case, objects under the directory remain intact.
522
    """
523

  
524
    arguments = dict(
525
        progress_bar=ProgressBarArgument(
526
            'do not show progress bar', ('-N', '--no-progress-bar'),
527
            default=False),
528
        max_threads=IntArgument('default: 1', '--threads'),
529
    )
530

  
531
    @errors.generic.all
532
    @errors.pithos.connection
533
    @errors.pithos.container
534
    @errors.pithos.object_path
535
    def _run(self, local_path):
536
        if self['max_threads'] > 0:
537
            self.client.MAX_THREADS = int(self['max_threads'])
538
        (progress_bar, upload_cb) = self._safe_progress_bar('Appending')
539
        try:
540
            with open(local_path, 'rb') as f:
541
                self._optional_output(
542
                    self.client.append_object(self.path, f, upload_cb))
543
        finally:
544
            self._safe_progress_bar_finish(progress_bar)
545

  
546
    def main(self, local_path, remote_path_or_url):
547
        super(self.__class__, self)._run(remote_path_or_url)
548
        self._run(local_path)
549

  
550

  
551
@command(file_cmds)
552
class file_truncate(_pithos_container, _optional_output_cmd):
553
    """Truncate remote file up to size"""
554

  
555
    arguments = dict(
556
        size_in_bytes=IntArgument('Length of file after truncation', '--size')
557
    )
558
    required = ('size_in_bytes', )
559

  
560
    @errors.generic.all
561
    @errors.pithos.connection
562
    @errors.pithos.container
563
    @errors.pithos.object_path
564
    @errors.pithos.object_size
565
    def _run(self, size):
566
        self._optional_output(self.client.truncate_object(self.path, size))
567

  
568
    def main(self, path_or_url):
569
        super(self.__class__, self)._run(path_or_url)
570
        self._run(size=self['size_in_bytes'])
571

  
572

  
573
@command(file_cmds)
574
class file_overwrite(_pithos_container, _optional_output_cmd):
575
    """Overwrite part of a remote file"""
576

  
577
    arguments = dict(
578
        progress_bar=ProgressBarArgument(
579
            'do not show progress bar', ('-N', '--no-progress-bar'),
580
            default=False),
581
        start_position=IntArgument('File position in bytes', '--from'),
582
        end_position=IntArgument('File position in bytes', '--to')
583
    )
584
    required = ('start_position', 'end_position')
585

  
586
    @errors.generic.all
587
    @errors.pithos.connection
588
    @errors.pithos.container
589
    @errors.pithos.object_path
590
    @errors.pithos.object_size
591
    def _run(self, local_path, start, end):
592
        start, end = int(start), int(end)
593
        (progress_bar, upload_cb) = self._safe_progress_bar(
594
            'Overwrite %s bytes' % (end - start))
595
        try:
596
            with open(path.abspath(local_path), 'rb') as f:
597
                self._optional_output(self.client.overwrite_object(
598
                    obj=self.path,
599
                    start=start,
600
                    end=end,
601
                    source_file=f,
602
                    upload_cb=upload_cb))
603
        finally:
604
            self._safe_progress_bar_finish(progress_bar)
605

  
606
    def main(self, local_path, path_or_url):
607
        super(self.__class__, self)._run(path_or_url)
608
        self.path = self.path or path.basename(local_path)
609
        self._run(
610
            local_path=local_path,
611
            start=self['start_position'],
612
            end=self['end_position'])
613

  
614

  
615
@command(file_cmds)
616
class file_upload(_pithos_container, _optional_output_cmd):
617
    """Upload a file"""
618

  
619
    arguments = dict(
620
        max_threads=IntArgument('default: 5', '--threads'),
621
        content_encoding=ValueArgument(
622
            'set MIME content type', '--content-encoding'),
623
        content_disposition=ValueArgument(
624
            'specify objects presentation style', '--content-disposition'),
625
        content_type=ValueArgument('specify content type', '--content-type'),
626
        uuid_for_read_permission=RepeatableArgument(
627
            'Give read access to a user of group (can be repeated)',
628
            '--read-permission'),
629
        uuid_for_write_permission=RepeatableArgument(
630
            'Give write access to a user of group (can be repeated)',
631
            '--write-permission'),
632
        public=FlagArgument('make object publicly accessible', '--public'),
633
        overwrite=FlagArgument('Force (over)write', ('-f', '--force')),
634
        recursive=FlagArgument(
635
            'Recursively upload directory *contents* + subdirectories',
636
            ('-r', '--recursive')),
637
        unchunked=FlagArgument(
638
            'Upload file as one block (not recommended)', '--unchunked'),
639
        md5_checksum=ValueArgument(
640
            'Confirm upload with a custom checksum (MD5)', '--etag'),
641
        use_hashes=FlagArgument(
642
            'Source file contains hashmap not data', '--source-is-hashmap'),
643
    )
644

  
645
    def _sharing(self):
646
        sharing = dict()
647
        readlist = self['uuid_for_read_permission']
648
        if readlist:
649
            sharing['read'] = self['uuid_for_read_permission']
650
        writelist = self['uuid_for_write_permission']
651
        if writelist:
652
            sharing['write'] = self['uuid_for_write_permission']
653
        return sharing or None
654

  
655
    def _check_container_limit(self, path):
656
        cl_dict = self.client.get_container_limit()
657
        container_limit = int(cl_dict['x-container-policy-quota'])
658
        r = self.client.container_get()
659
        used_bytes = sum(int(o['bytes']) for o in r.json)
660
        path_size = get_path_size(path)
661
        if container_limit and path_size > (container_limit - used_bytes):
662
            raise CLIError(
663
                'Container %s (limit(%s) - used(%s)) < (size(%s) of %s)' % (
664
                    self.client.container,
665
                    format_size(container_limit),
666
                    format_size(used_bytes),
667
                    format_size(path_size),
668
                    path),
669
                details=[
670
                    'Check accound limit: /file quota',
671
                    'Check container limit:',
672
                    '\t/file containerlimit get %s' % self.client.container,
673
                    'Increase container limit:',
674
                    '\t/file containerlimit set <new limit> %s' % (
675
                        self.client.container)])
676

  
677
    def _src_dst(self, local_path, remote_path, objlist=None):
678
        lpath = path.abspath(local_path)
679
        short_path = path.basename(path.abspath(local_path))
680
        rpath = remote_path or short_path
681
        if path.isdir(lpath):
682
            if not self['recursive']:
683
                raise CLIError('%s is a directory' % lpath, details=[
684
                    'Use %s to upload directories & contents' % '/'.join(
685
                        self.arguments['recursive'].parsed_name)])
686
            robj = self.client.container_get(path=rpath)
687
            if not self['overwrite']:
688
                if robj.json:
689
                    raise CLIError(
690
                        'Objects/files prefixed as %s already exist' % rpath,
691
                        details=['Existing objects:'] + ['\t%s:\t%s' % (
692
                            o['name'],
693
                            o['content_type'][12:]) for o in robj.json] + [
694
                            'Use -f to add, overwrite or resume'])
695
                else:
696
                    try:
697
                        topobj = self.client.get_object_info(rpath)
698
                        if not self._is_dir(topobj):
699
                            raise CLIError(
700
                                'Object /%s/%s exists but not a directory' % (
701
                                    self.container, rpath),
702
                                details=['Use -f to overwrite'])
703
                    except ClientError as ce:
704
                        if ce.status not in (404, ):
705
                            raise
706
            self._check_container_limit(lpath)
707
            prev = ''
708
            for top, subdirs, files in walk(lpath):
709
                if top != prev:
710
                    prev = top
711
                    try:
712
                        rel_path = rpath + top.split(lpath)[1]
713
                    except IndexError:
714
                        rel_path = rpath
715
                    self.error('mkdir %s:%s' % (
716
                        self.client.container, rel_path))
717
                    self.client.create_directory(rel_path)
718
                for f in files:
719
                    fpath = path.join(top, f)
720
                    if path.isfile(fpath):
721
                        rel_path = rel_path.replace(path.sep, '/')
722
                        pathfix = f.replace(path.sep, '/')
723
                        yield open(fpath, 'rb'), '%s/%s' % (rel_path, pathfix)
724
                    else:
725
                        self.error('%s is not a regular file' % fpath)
726
        else:
727
            if not path.isfile(lpath):
728
                raise CLIError(('%s is not a regular file' % lpath) if (
729
                    path.exists(lpath)) else '%s does not exist' % lpath)
730
            try:
731
                robj = self.client.get_object_info(rpath)
732
                if remote_path and self._is_dir(robj):
733
                    rpath += '/%s' % (short_path.replace(path.sep, '/'))
734
                    self.client.get_object_info(rpath)
735
                if not self['overwrite']:
736
                    raise CLIError(
737
                        'Object /%s/%s already exists' % (
738
                            self.container, rpath),
739
                        details=['use -f to overwrite / resume'])
740
            except ClientError as ce:
741
                if ce.status not in (404, ):
742
                    raise
743
            self._check_container_limit(lpath)
744
            yield open(lpath, 'rb'), rpath
745

  
746
    def _run(self, local_path, remote_path):
747
        if self['max_threads'] > 0:
748
            self.client.MAX_THREADS = int(self['max_threads'])
749
        params = dict(
750
            content_encoding=self['content_encoding'],
751
            content_type=self['content_type'],
752
            content_disposition=self['content_disposition'],
753
            sharing=self._sharing(),
754
            public=self['public'])
755
        uploaded, container_info_cache = list, dict()
756
        rpref = 'pithos://%s' if self['account'] else ''
757
        for f, rpath in self._src_dst(local_path, remote_path):
758
            self.error('%s --> %s/%s/%s' % (
759
                f.name, rpref, self.client.container, rpath))
760
            if not (self['content_type'] and self['content_encoding']):
761
                ctype, cenc = guess_mime_type(f.name)
762
                params['content_type'] = self['content_type'] or ctype
763
                params['content_encoding'] = self['content_encoding'] or cenc
764
            if self['unchunked']:
765
                r = self.client.upload_object_unchunked(
766
                    rpath, f,
767
                    etag=self['md5_checksum'], withHashFile=self['use_hashes'],
768
                    **params)
769
                if self['with_output'] or self['json_output']:
770
                    r['name'] = '/%s/%s' % (self.client.container, rpath)
771
                    uploaded.append(r)
772
            else:
773
                try:
774
                    (progress_bar, upload_cb) = self._safe_progress_bar(
775
                        'Uploading %s' % f.name.split(path.sep)[-1])
776
                    if progress_bar:
777
                        hash_bar = progress_bar.clone()
778
                        hash_cb = hash_bar.get_generator(
779
                            'Calculating block hashes')
780
                    else:
781
                        hash_cb = None
782
                    r = self.client.upload_object(
783
                        rpath, f,
784
                        hash_cb=hash_cb,
785
                        upload_cb=upload_cb,
786
                        container_info_cache=container_info_cache,
787
                        **params)
788
                    if self['with_output'] or self['json_output']:
789
                        r['name'] = '/%s/%s' % (self.client.container, rpath)
790
                        uploaded.append(r)
791
                except Exception:
792
                    self._safe_progress_bar_finish(progress_bar)
793
                    raise
794
                finally:
795
                    self._safe_progress_bar_finish(progress_bar)
796
        self._optional_output(uploaded)
797
        self.error('Upload completed')
798

  
799
    def main(self, local_path, remote_path_or_url):
800
        super(self.__class__, self)._run(remote_path_or_url)
801
        remote_path = self.path or path.basename(path.abspath(local_path))
802
        self._run(local_path=local_path, remote_path=remote_path)
803

  
804

  
805
class RangeArgument(ValueArgument):
806
    """
807
    :value type: string of the form <start>-<end> where <start> and <end> are
808
        integers
809
    :value returns: the input string, after type checking <start> and <end>
810
    """
811

  
812
    @property
813
    def value(self):
814
        return getattr(self, '_value', self.default)
815

  
816
    @value.setter
817
    def value(self, newvalues):
818
        if newvalues:
819
            self._value = getattr(self, '_value', self.default)
820
            for newvalue in newvalues.split(','):
821
                self._value = ('%s,' % self._value) if self._value else ''
822
                start, sep, end = newvalue.partition('-')
823
                if sep:
824
                    if start:
825
                        start, end = (int(start), int(end))
826
                        if start > end:
827
                            raise CLIInvalidArgument(
828
                                'Invalid range %s' % newvalue, details=[
829
                                'Valid range formats',
830
                                '  START-END', '  UP_TO', '  -FROM',
831
                                'where all values are integers'])
832
                        self._value += '%s-%s' % (start, end)
833
                    else:
834
                        self._value += '-%s' % int(end)
835
                else:
836
                    self._value += '%s' % int(start)
837

  
838

  
839
@command(file_cmds)
840
class file_cat(_pithos_container):
841
    """Fetch remote file contents"""
842

  
843
    arguments = dict(
844
        range=RangeArgument('show range of data', '--range'),
845
        if_match=ValueArgument('show output if ETags match', '--if-match'),
846
        if_none_match=ValueArgument(
847
            'show output if ETags match', '--if-none-match'),
848
        if_modified_since=DateArgument(
849
            'show output modified since then', '--if-modified-since'),
850
        if_unmodified_since=DateArgument(
851
            'show output unmodified since then', '--if-unmodified-since'),
852
        object_version=ValueArgument(
853
            'Get contents of the chosen version', '--object-version')
854
    )
855

  
856
    @errors.generic.all
857
    @errors.pithos.connection
858
    @errors.pithos.container
859
    @errors.pithos.object_path
860
    def _run(self):
861
        self.client.download_object(
862
            self.path, self._out,
863
            range_str=self['range'],
864
            version=self['object_version'],
865
            if_match=self['if_match'],
866
            if_none_match=self['if_none_match'],
867
            if_modified_since=self['if_modified_since'],
868
            if_unmodified_since=self['if_unmodified_since'])
869

  
870
    def main(self, path_or_url):
871
        super(self.__class__, self)._run(path_or_url)
872
        self._run()
b/kamaki/clients/pithos/__init__.py
662 662
    def _dump_blocks_sync(
663 663
            self, obj, remote_hashes, blocksize, total_size, dst, crange,
664 664
            **args):
665
        if not total_size:
666
            return
665 667
        for blockid, blockhash in enumerate(remote_hashes):
666 668
            if blockhash:
667 669
                start = blocksize * blockid

Also available in: Unified diff