Revision 5655d560
| b/Changelog | ||
|---|---|---|
| 19 | 19 |
This operation was implemented by accident, due to the symetry between |
| 20 | 20 |
move and copy |
| 21 | 21 |
- Add optional output for file methods [#3756, #3732]: |
| 22 |
mkdir, touch, create, move, create, copy, move |
|
| 22 |
mkdir, touch, create, move, create, copy, move, append, delete, purge, |
|
| 23 |
info, meta, upload |
|
| 24 |
- Transliterate permissions and metadata methods to (get, set delete) groups |
|
| 23 | 25 |
|
| 24 | 26 |
Features: |
| 25 | 27 |
|
| b/kamaki/cli/commands/pithos.py | ||
|---|---|---|
| 505 | 505 |
meta=KeyValueArgument( |
| 506 | 506 |
'set container metadata (can be repeated)', |
| 507 | 507 |
'--meta'), |
| 508 |
with_output=FlagArgument('show request headers', ('--with-output')),
|
|
| 508 |
with_output=FlagArgument('show response headers', ('--with-output')),
|
|
| 509 | 509 |
json_output=FlagArgument('show headers in json', ('-j', '--json'))
|
| 510 | 510 |
) |
| 511 | 511 |
|
| ... | ... | |
| 514 | 514 |
@errors.pithos.container |
| 515 | 515 |
def _run(self, container): |
| 516 | 516 |
r = self.client.create_container( |
| 517 |
container=container, |
|
| 518 |
sizelimit=self['limit'], |
|
| 519 |
versioning=self['versioning'], |
|
| 520 |
metadata=self['meta']) |
|
| 517 |
container=container, sizelimit=self['limit'], |
|
| 518 |
versioning=self['versioning'], metadata=self['meta']) |
|
| 521 | 519 |
if self['json_output']: |
| 522 | 520 |
print_json(r) |
| 523 | 521 |
elif self['with_output']: |
| ... | ... | |
| 728 | 726 |
source_version=ValueArgument( |
| 729 | 727 |
'copy specific version', |
| 730 | 728 |
('-S', '--source-version')),
|
| 731 |
with_output=FlagArgument('show request headers', ('--with-output')),
|
|
| 729 |
with_output=FlagArgument('show response headers', ('--with-output')),
|
|
| 732 | 730 |
json_output=FlagArgument('show headers in json', ('-j', '--json'))
|
| 733 | 731 |
) |
| 734 | 732 |
|
| ... | ... | |
| 823 | 821 |
'Suffix of src to replace with add_suffix, if matched', |
| 824 | 822 |
'--suffix-to-replace', |
| 825 | 823 |
default=''), |
| 826 |
with_output=FlagArgument('show request headers', ('--with-output')),
|
|
| 824 |
with_output=FlagArgument('show response headers', ('--with-output')),
|
|
| 827 | 825 |
json_output=FlagArgument('show headers in json', ('-j', '--json'))
|
| 828 | 826 |
) |
| 829 | 827 |
|
| ... | ... | |
| 879 | 877 |
progress_bar=ProgressBarArgument( |
| 880 | 878 |
'do not show progress bar', |
| 881 | 879 |
('-N', '--no-progress-bar'),
|
| 882 |
default=False) |
|
| 880 |
default=False), |
|
| 881 |
with_output=FlagArgument('show response headers', ('--with-output')),
|
|
| 882 |
json_output=FlagArgument('show headers in json', ('-j', '--json'))
|
|
| 883 | 883 |
) |
| 884 | 884 |
|
| 885 | 885 |
@errors.generic.all |
| ... | ... | |
| 890 | 890 |
(progress_bar, upload_cb) = self._safe_progress_bar('Appending')
|
| 891 | 891 |
try: |
| 892 | 892 |
f = open(local_path, 'rb') |
| 893 |
self.client.append_object(self.path, f, upload_cb) |
|
| 893 |
r = self.client.append_object(self.path, f, upload_cb) |
|
| 894 |
if self['json_output']: |
|
| 895 |
print_json(r) |
|
| 896 |
elif self['with_output']: |
|
| 897 |
print_items(r) |
|
| 894 | 898 |
except Exception: |
| 895 | 899 |
self._safe_progress_bar_finish(progress_bar) |
| 896 | 900 |
raise |
| ... | ... | |
| 1061 | 1065 |
recursive=FlagArgument( |
| 1062 | 1066 |
'Recursively upload directory *contents* + subdirectories', |
| 1063 | 1067 |
('-R', '--recursive')),
|
| 1064 |
details=FlagArgument( |
|
| 1065 |
'Show a detailed list of uploaded objects at the end', |
|
| 1066 |
('-l', '--details'))
|
|
| 1068 |
with_output=FlagArgument( |
|
| 1069 |
'Show uploaded objects response headers', |
|
| 1070 |
('--with-output')),
|
|
| 1071 |
json_output=FlagArgument('show headers in json', ('-j', '--json'))
|
|
| 1067 | 1072 |
) |
| 1068 | 1073 |
|
| 1069 | 1074 |
def _check_container_limit(self, path): |
| ... | ... | |
| 1176 | 1181 |
rpath, f, |
| 1177 | 1182 |
etag=self['etag'], withHashFile=self['use_hashes'], |
| 1178 | 1183 |
**params) |
| 1179 |
if self['details']:
|
|
| 1184 |
if self['with_output'] or self['json_output']:
|
|
| 1180 | 1185 |
r['name'] = '%s: %s' % (self.client.container, rpath) |
| 1181 | 1186 |
uploaded.append(r) |
| 1182 | 1187 |
else: |
| ... | ... | |
| 1195 | 1200 |
upload_cb=upload_cb, |
| 1196 | 1201 |
container_info_cache=container_info_cache, |
| 1197 | 1202 |
**params) |
| 1198 |
if self['details']:
|
|
| 1203 |
if self['with_output'] or self['json_output']:
|
|
| 1199 | 1204 |
r['name'] = '%s: %s' % (self.client.container, rpath) |
| 1200 | 1205 |
uploaded.append(r) |
| 1201 | 1206 |
except Exception: |
| ... | ... | |
| 1203 | 1208 |
raise |
| 1204 | 1209 |
finally: |
| 1205 | 1210 |
self._safe_progress_bar_finish(progress_bar) |
| 1206 |
if self['details']: |
|
| 1211 |
if self['json_output']: |
|
| 1212 |
print_json(uploaded) |
|
| 1213 |
elif self['with_output']: |
|
| 1207 | 1214 |
print_items(uploaded) |
| 1208 | 1215 |
else: |
| 1209 | 1216 |
print('Upload completed')
|
| ... | ... | |
| 1232 | 1239 |
'--if-unmodified-since'), |
| 1233 | 1240 |
object_version=ValueArgument( |
| 1234 | 1241 |
'get the specific version', |
| 1235 |
('-j', '--object-version'))
|
|
| 1242 |
('-O', '--object-version'))
|
|
| 1236 | 1243 |
) |
| 1237 | 1244 |
|
| 1238 | 1245 |
@errors.generic.all |
| ... | ... | |
| 1286 | 1293 |
'--if-unmodified-since'), |
| 1287 | 1294 |
object_version=ValueArgument( |
| 1288 | 1295 |
'get the specific version', |
| 1289 |
('-j', '--object-version')),
|
|
| 1296 |
('-O', '--object-version')),
|
|
| 1290 | 1297 |
poolsize=IntArgument('set pool size', '--with-pool-size'),
|
| 1291 | 1298 |
progress_bar=ProgressBarArgument( |
| 1292 | 1299 |
'do not show progress bar', |
| ... | ... | |
| 1417 | 1424 |
download_cb) = self._safe_progress_bar( |
| 1418 | 1425 |
'Download %s' % rpath) |
| 1419 | 1426 |
self.client.download_object( |
| 1420 |
rpath, |
|
| 1421 |
f, |
|
| 1427 |
rpath, f, |
|
| 1422 | 1428 |
download_cb=download_cb, |
| 1423 | 1429 |
range_str=self['range'], |
| 1424 | 1430 |
version=self['object_version'], |
| ... | ... | |
| 1442 | 1448 |
finally: |
| 1443 | 1449 |
stdout.flush() |
| 1444 | 1450 |
timeout += 0.1 |
| 1445 |
|
|
| 1446 | 1451 |
print('\nDownload canceled by user')
|
| 1447 | 1452 |
if local_path is not None: |
| 1448 | 1453 |
print('to resume, re-run with --resume')
|
| ... | ... | |
| 1474 | 1479 |
'--if-unmodified-since'), |
| 1475 | 1480 |
object_version=ValueArgument( |
| 1476 | 1481 |
'get the specific version', |
| 1477 |
('-j', '--object-version'))
|
|
| 1482 |
('-O', '--object-version')),
|
|
| 1483 |
json_output=FlagArgument('show headers in json', ('-j', '--json'))
|
|
| 1478 | 1484 |
) |
| 1479 | 1485 |
|
| 1480 | 1486 |
@errors.generic.all |
| ... | ... | |
| 1489 | 1495 |
if_none_match=self['if_none_match'], |
| 1490 | 1496 |
if_modified_since=self['if_modified_since'], |
| 1491 | 1497 |
if_unmodified_since=self['if_unmodified_since']) |
| 1492 |
print_dict(data) |
|
| 1498 |
printer = print_json if self['json_output'] else print_dict |
|
| 1499 |
printer(data) |
|
| 1493 | 1500 |
|
| 1494 | 1501 |
def main(self, container___path): |
| 1495 | 1502 |
super(self.__class__, self)._run( |
| ... | ... | |
| 1522 | 1529 |
yes=FlagArgument('Do not prompt for permission', '--yes'),
|
| 1523 | 1530 |
recursive=FlagArgument( |
| 1524 | 1531 |
'empty dir or container and delete (if dir)', |
| 1525 |
('-R', '--recursive'))
|
|
| 1532 |
('-R', '--recursive')),
|
|
| 1533 |
with_output=FlagArgument('show response headers', ('--with-output')),
|
|
| 1534 |
json_output=FlagArgument('show headers in json', ('-j', '--json'))
|
|
| 1526 | 1535 |
) |
| 1527 | 1536 |
|
| 1528 | 1537 |
def __init__(self, arguments={}):
|
| ... | ... | |
| 1537 | 1546 |
@errors.pithos.container |
| 1538 | 1547 |
@errors.pithos.object_path |
| 1539 | 1548 |
def _run(self): |
| 1549 |
r = {}
|
|
| 1540 | 1550 |
if self.path: |
| 1541 | 1551 |
if self['yes'] or ask_user( |
| 1542 | 1552 |
'Delete %s:%s ?' % (self.container, self.path)): |
| 1543 |
self.client.del_object( |
|
| 1553 |
r = self.client.del_object(
|
|
| 1544 | 1554 |
self.path, |
| 1545 | 1555 |
until=self['until'], |
| 1546 | 1556 |
delimiter=self['delimiter']) |
| ... | ... | |
| 1552 | 1562 |
else: |
| 1553 | 1563 |
ask_msg = 'Delete container' |
| 1554 | 1564 |
if self['yes'] or ask_user('%s %s ?' % (ask_msg, self.container)):
|
| 1555 |
self.client.del_container( |
|
| 1565 |
r = self.client.del_container(
|
|
| 1556 | 1566 |
until=self['until'], |
| 1557 | 1567 |
delimiter=self['delimiter']) |
| 1558 | 1568 |
else: |
| 1559 | 1569 |
print('Aborted')
|
| 1570 |
return |
|
| 1571 |
if self['json_output']: |
|
| 1572 |
print_json(r) |
|
| 1573 |
elif self['with_output']: |
|
| 1574 |
print_dict(r) |
|
| 1560 | 1575 |
|
| 1561 | 1576 |
def main(self, container____path__=None): |
| 1562 | 1577 |
super(self.__class__, self)._run(container____path__) |
| ... | ... | |
| 1576 | 1591 |
|
| 1577 | 1592 |
arguments = dict( |
| 1578 | 1593 |
yes=FlagArgument('Do not prompt for permission', '--yes'),
|
| 1579 |
force=FlagArgument('purge even if not empty', ('-F', '--force'))
|
|
| 1594 |
force=FlagArgument('purge even if not empty', ('-F', '--force')),
|
|
| 1595 |
with_output=FlagArgument('show response headers', ('--with-output')),
|
|
| 1596 |
json_output=FlagArgument('show headers in json', ('-j', '--json'))
|
|
| 1580 | 1597 |
) |
| 1581 | 1598 |
|
| 1582 | 1599 |
@errors.generic.all |
| ... | ... | |
| 1585 | 1602 |
def _run(self): |
| 1586 | 1603 |
if self['yes'] or ask_user('Purge container %s?' % self.container):
|
| 1587 | 1604 |
try: |
| 1588 |
self.client.purge_container() |
|
| 1605 |
r = self.client.purge_container()
|
|
| 1589 | 1606 |
except ClientError as ce: |
| 1590 | 1607 |
if ce.status in (409,): |
| 1591 | 1608 |
if self['force']: |
| 1592 | 1609 |
self.client.del_container(delimiter='/') |
| 1593 |
self.client.purge_container() |
|
| 1610 |
r = self.client.purge_container()
|
|
| 1594 | 1611 |
else: |
| 1595 | 1612 |
raiseCLIError(ce, details=['Try -F to force-purge']) |
| 1596 | 1613 |
else: |
| 1597 | 1614 |
raise |
| 1615 |
if self['json_output']: |
|
| 1616 |
print_json(r) |
|
| 1617 |
elif self['with_output']: |
|
| 1618 |
print_dict(r) |
|
| 1598 | 1619 |
else: |
| 1599 | 1620 |
print('Aborted')
|
| 1600 | 1621 |
|
| ... | ... | |
| 1630 | 1651 |
class file_unpublish(_file_container_command): |
| 1631 | 1652 |
"""Unpublish an object""" |
| 1632 | 1653 |
|
| 1654 |
arguments = dict( |
|
| 1655 |
with_output=FlagArgument('show response headers', ('--with-output')),
|
|
| 1656 |
json_output=FlagArgument('show headers in json', ('-j', '--json'))
|
|
| 1657 |
) |
|
| 1658 |
|
|
| 1633 | 1659 |
@errors.generic.all |
| 1634 | 1660 |
@errors.pithos.connection |
| 1635 | 1661 |
@errors.pithos.container |
| 1636 | 1662 |
@errors.pithos.object_path |
| 1637 | 1663 |
def _run(self): |
| 1638 |
self.client.unpublish_object(self.path) |
|
| 1664 |
r = self.client.unpublish_object(self.path) |
|
| 1665 |
if self['json_output']: |
|
| 1666 |
print_json(r) |
|
| 1667 |
elif self['with_output']: |
|
| 1668 |
print_dict(r) |
|
| 1639 | 1669 |
|
| 1640 | 1670 |
def main(self, container___path): |
| 1641 | 1671 |
super(self.__class__, self)._run( |
| ... | ... | |
| 1645 | 1675 |
|
| 1646 | 1676 |
|
| 1647 | 1677 |
@command(pithos_cmds) |
| 1648 |
class file_permissions(_file_container_command):
|
|
| 1649 |
"""Get read and write permissions of an object
|
|
| 1650 |
Permissions are lists of users and user groups. There is read and write
|
|
| 1678 |
class file_permissions(_pithos_init):
|
|
| 1679 |
"""Manage user and group accessibility for objects
|
|
| 1680 |
Permissions are lists of users and user groups. There are read and write
|
|
| 1651 | 1681 |
permissions. Users and groups with write permission have also read |
| 1652 | 1682 |
permission. |
| 1653 | 1683 |
""" |
| 1654 | 1684 |
|
| 1685 |
|
|
| 1686 |
@command(pithos_cmds) |
|
| 1687 |
class file_permissions_get(_file_container_command): |
|
| 1688 |
"""Get read and write permissions of an object""" |
|
| 1689 |
|
|
| 1655 | 1690 |
@errors.generic.all |
| 1656 | 1691 |
@errors.pithos.connection |
| 1657 | 1692 |
@errors.pithos.container |
| ... | ... | |
| 1668 | 1703 |
|
| 1669 | 1704 |
|
| 1670 | 1705 |
@command(pithos_cmds) |
| 1671 |
class file_setpermissions(_file_container_command):
|
|
| 1706 |
class file_permissions_set(_file_container_command):
|
|
| 1672 | 1707 |
"""Set permissions for an object |
| 1673 | 1708 |
New permissions overwrite existing permissions. |
| 1674 | 1709 |
Permission format: |
| 1675 | 1710 |
- read=<username>[,usergroup[,...]] |
| 1676 | 1711 |
- write=<username>[,usegroup[,...]] |
| 1677 | 1712 |
E.g. to give read permissions for file F to users A and B and write for C: |
| 1678 |
. /file setpermissions F read=A,B write=C
|
|
| 1713 |
. /file permissions set F read=A,B write=C
|
|
| 1679 | 1714 |
""" |
| 1680 | 1715 |
|
| 1681 | 1716 |
@errors.generic.all |
| ... | ... | |
| 1712 | 1747 |
|
| 1713 | 1748 |
|
| 1714 | 1749 |
@command(pithos_cmds) |
| 1715 |
class file_delpermissions(_file_container_command):
|
|
| 1750 |
class file_permissions_delete(_file_container_command):
|
|
| 1716 | 1751 |
"""Delete all permissions set on object |
| 1717 |
To modify permissions, use /file setpermssions
|
|
| 1752 |
To modify permissions, use /file permissions set
|
|
| 1718 | 1753 |
""" |
| 1719 | 1754 |
|
| 1720 | 1755 |
@errors.generic.all |
| ... | ... | |
| 1742 | 1777 |
arguments = dict( |
| 1743 | 1778 |
object_version=ValueArgument( |
| 1744 | 1779 |
'show specific version \ (applies only for objects)', |
| 1745 |
('-j', '--object-version'))
|
|
| 1780 |
('-O', '--object-version')),
|
|
| 1781 |
json_output=FlagArgument('show headers in json', ('-j', '--json'))
|
|
| 1746 | 1782 |
) |
| 1747 | 1783 |
|
| 1748 | 1784 |
@errors.generic.all |
| ... | ... | |
| 1758 | 1794 |
r = self.client.get_object_info( |
| 1759 | 1795 |
self.path, |
| 1760 | 1796 |
version=self['object_version']) |
| 1761 |
print_dict(r) |
|
| 1797 |
printer = print_json if self['json_output'] else print_dict |
|
| 1798 |
printer(r) |
|
| 1762 | 1799 |
|
| 1763 | 1800 |
def main(self, container____path__=None): |
| 1764 | 1801 |
super(self.__class__, self)._run(container____path__) |
| ... | ... | |
| 1766 | 1803 |
|
| 1767 | 1804 |
|
| 1768 | 1805 |
@command(pithos_cmds) |
| 1769 |
class file_meta(_file_container_command): |
|
| 1806 |
class file_metadata(_pithos_init): |
|
| 1807 |
"""Metadata are attached on objects. They are formed as key:value pairs. |
|
| 1808 |
They can have arbitary values. |
|
| 1809 |
""" |
|
| 1810 |
|
|
| 1811 |
|
|
| 1812 |
@command(pithos_cmds) |
|
| 1813 |
class file_metadata_get(_file_container_command): |
|
| 1770 | 1814 |
"""Get metadata for account, containers or objects""" |
| 1771 | 1815 |
|
| 1772 | 1816 |
arguments = dict( |
| ... | ... | |
| 1774 | 1818 |
until=DateArgument('show metadata until then', '--until'),
|
| 1775 | 1819 |
object_version=ValueArgument( |
| 1776 | 1820 |
'show specific version \ (applies only for objects)', |
| 1777 |
('-j', '--object-version'))
|
|
| 1821 |
('-O', '--object-version')),
|
|
| 1822 |
json_output=FlagArgument('show headers in json', ('-j', '--json'))
|
|
| 1778 | 1823 |
) |
| 1779 | 1824 |
|
| 1780 | 1825 |
@errors.generic.all |
| ... | ... | |
| 1789 | 1834 |
else: |
| 1790 | 1835 |
r = self.client.get_account_meta(until=until) |
| 1791 | 1836 |
r = pretty_keys(r, '-') |
| 1792 |
if r: |
|
| 1793 |
print(bold(self.client.account)) |
|
| 1837 |
print(bold(self.client.account)) |
|
| 1794 | 1838 |
elif self.path is None: |
| 1795 | 1839 |
if self['detail']: |
| 1796 | 1840 |
r = self.client.get_container_info(until=until) |
| ... | ... | |
| 1811 | 1855 |
r = self.client.get_object_meta( |
| 1812 | 1856 |
self.path, |
| 1813 | 1857 |
version=self['object_version']) |
| 1814 |
if r: |
|
| 1815 | 1858 |
r = pretty_keys(pretty_keys(r, '-')) |
| 1816 | 1859 |
if r: |
| 1817 |
print_dict(r) |
|
| 1860 |
printer = print_json if self['json_output'] else print_dict |
|
| 1861 |
printer(r) |
|
| 1818 | 1862 |
|
| 1819 | 1863 |
def main(self, container____path__=None): |
| 1820 | 1864 |
super(self.__class__, self)._run(container____path__) |
| ... | ... | |
| 1822 | 1866 |
|
| 1823 | 1867 |
|
| 1824 | 1868 |
@command(pithos_cmds) |
| 1825 |
class file_setmeta(_file_container_command): |
|
| 1826 |
"""Set a piece of metadata for account, container or object |
|
| 1827 |
Metadata are formed as key:value pairs |
|
| 1828 |
""" |
|
| 1869 |
class file_metadata_set(_file_container_command): |
|
| 1870 |
"""Set a piece of metadata for account, container or object""" |
|
| 1829 | 1871 |
|
| 1830 | 1872 |
@errors.generic.all |
| 1831 | 1873 |
@errors.pithos.connection |
| ... | ... | |
| 1845 | 1887 |
|
| 1846 | 1888 |
|
| 1847 | 1889 |
@command(pithos_cmds) |
| 1848 |
class file_delmeta(_file_container_command):
|
|
| 1890 |
class file_metadata_delete(_file_container_command):
|
|
| 1849 | 1891 |
"""Delete metadata with given key from account, container or object |
| 1850 |
Metadata are formed as key:value objects |
|
| 1851 |
- to get metadata of current account: /file meta |
|
| 1852 |
- to get metadata of a container: /file meta <container> |
|
| 1853 |
- to get metadata of an object: /file meta <container>:<path> |
|
| 1892 |
- to get metadata of current account: /file metadata get |
|
| 1893 |
- to get metadata of a container: /file metadata get <container> |
|
| 1894 |
- to get metadata of an object: /file metadata get <container>:<path> |
|
| 1854 | 1895 |
""" |
| 1855 | 1896 |
|
| 1856 | 1897 |
@errors.generic.all |
| b/kamaki/clients/livetest/pithos.py | ||
|---|---|---|
| 393 | 393 |
def _test_0050_container_put(self): |
| 394 | 394 |
self.client.container = self.c2 |
| 395 | 395 |
|
| 396 |
r = self.client.container_put()
|
|
| 397 |
self.assertEqual(r.status_code, 202)
|
|
| 396 |
r = self.client.create_container()
|
|
| 397 |
self.assertTrue(isinstance(r, dict))
|
|
| 398 | 398 |
|
| 399 | 399 |
r = self.client.get_container_limit(self.client.container) |
| 400 | 400 |
cquota = r.values()[0] |
| 401 | 401 |
newquota = 2 * int(cquota) |
| 402 | 402 |
|
| 403 |
r = self.client.container_put(quota=newquota)
|
|
| 404 |
self.assertEqual(r.status_code, 202)
|
|
| 403 |
r = self.client.create_container(sizelimit=newquota)
|
|
| 404 |
self.assertTrue(isinstance(r, dict))
|
|
| 405 | 405 |
|
| 406 | 406 |
r = self.client.get_container_limit(self.client.container) |
| 407 | 407 |
xquota = int(r.values()[0]) |
| 408 | 408 |
self.assertEqual(newquota, xquota) |
| 409 | 409 |
|
| 410 |
r = self.client.container_put(versioning='auto')
|
|
| 411 |
self.assertEqual(r.status_code, 202)
|
|
| 410 |
r = self.client.create_container(versioning='auto')
|
|
| 411 |
self.assertTrue(isinstance(r, dict))
|
|
| 412 | 412 |
|
| 413 | 413 |
r = self.client.get_container_versioning(self.client.container) |
| 414 | 414 |
nvers = r.values()[0] |
| ... | ... | |
| 421 | 421 |
nvers = r.values()[0] |
| 422 | 422 |
self.assertEqual('none', nvers)
|
| 423 | 423 |
|
| 424 |
r = self.client.container_put(metadata={'m1': 'v1', 'm2': 'v2'})
|
|
| 425 |
self.assertEqual(r.status_code, 202)
|
|
| 424 |
r = self.client.create_container(metadata={'m1': 'v1', 'm2': 'v2'})
|
|
| 425 |
self.assertTrue(isinstance(r, dict))
|
|
| 426 | 426 |
|
| 427 | 427 |
r = self.client.get_container_meta(self.client.container) |
| 428 | 428 |
self.assertTrue('x-container-meta-m1' in r)
|
| b/kamaki/clients/pithos/__init__.py | ||
|---|---|---|
| 101 | 101 |
cnt_back_up = self.container |
| 102 | 102 |
try: |
| 103 | 103 |
self.container = container or cnt_back_up |
| 104 |
self.container_delete(until=unicode(time())) |
|
| 104 |
r = self.container_delete(until=unicode(time()))
|
|
| 105 | 105 |
finally: |
| 106 | 106 |
self.container = cnt_back_up |
| 107 |
return r.headers |
|
| 107 | 108 |
|
| 108 | 109 |
def upload_object_unchunked( |
| 109 | 110 |
self, obj, f, |
| ... | ... | |
| 838 | 839 |
ret = [''] * num_of_blocks |
| 839 | 840 |
self._init_thread_limit() |
| 840 | 841 |
flying = dict() |
| 841 |
for blockid, blockhash in enumerate(remote_hashes): |
|
| 842 |
start = blocksize * blockid |
|
| 843 |
is_last = start + blocksize > total_size |
|
| 844 |
end = (total_size - 1) if is_last else (start + blocksize - 1) |
|
| 845 |
(start, end) = _range_up(start, end, range_str) |
|
| 846 |
if start < end: |
|
| 847 |
self._watch_thread_limit(flying.values()) |
|
| 848 |
flying[blockid] = self._get_block_async(obj, **restargs) |
|
| 849 |
for runid, thread in flying.items(): |
|
| 850 |
if (blockid + 1) == num_of_blocks: |
|
| 851 |
thread.join() |
|
| 852 |
elif thread.isAlive(): |
|
| 853 |
continue |
|
| 854 |
if thread.exception: |
|
| 855 |
raise thread.exception |
|
| 856 |
ret[runid] = thread.value.content |
|
| 857 |
self._cb_next() |
|
| 858 |
flying.pop(runid) |
|
| 859 |
return ''.join(ret) |
|
| 842 |
try: |
|
| 843 |
for blockid, blockhash in enumerate(remote_hashes): |
|
| 844 |
start = blocksize * blockid |
|
| 845 |
is_last = start + blocksize > total_size |
|
| 846 |
end = (total_size - 1) if is_last else (start + blocksize - 1) |
|
| 847 |
(start, end) = _range_up(start, end, range_str) |
|
| 848 |
if start < end: |
|
| 849 |
self._watch_thread_limit(flying.values()) |
|
| 850 |
flying[blockid] = self._get_block_async(obj, **restargs) |
|
| 851 |
for runid, thread in flying.items(): |
|
| 852 |
if (blockid + 1) == num_of_blocks: |
|
| 853 |
thread.join() |
|
| 854 |
elif thread.isAlive(): |
|
| 855 |
continue |
|
| 856 |
if thread.exception: |
|
| 857 |
raise thread.exception |
|
| 858 |
ret[runid] = thread.value.content |
|
| 859 |
self._cb_next() |
|
| 860 |
flying.pop(runid) |
|
| 861 |
return ''.join(ret) |
|
| 862 |
except KeyboardInterrupt: |
|
| 863 |
sendlog.info('- - - wait for threads to finish')
|
|
| 864 |
for thread in activethreads(): |
|
| 865 |
thread.join() |
|
| 860 | 866 |
|
| 861 | 867 |
#Command Progress Bar method |
| 862 | 868 |
def _cb_next(self, step=1): |
| ... | ... | |
| 1028 | 1034 |
raise ClientError( |
| 1029 | 1035 |
'Container "%s" is not empty' % self.container, |
| 1030 | 1036 |
r.status_code) |
| 1037 |
return r.headers |
|
| 1031 | 1038 |
|
| 1032 | 1039 |
def get_container_versioning(self, container=None): |
| 1033 | 1040 |
""" |
| ... | ... | |
| 1128 | 1135 |
:param delimiter: (str) |
| 1129 | 1136 |
""" |
| 1130 | 1137 |
self._assert_container() |
| 1131 |
self.object_delete(obj, until=until, delimiter=delimiter) |
|
| 1138 |
r = self.object_delete(obj, until=until, delimiter=delimiter) |
|
| 1139 |
return r.headers |
|
| 1132 | 1140 |
|
| 1133 | 1141 |
def set_object_meta(self, obj, metapairs): |
| 1134 | 1142 |
""" |
| ... | ... | |
| 1163 | 1171 |
""" |
| 1164 | 1172 |
:param obj: (str) remote object path |
| 1165 | 1173 |
""" |
| 1166 |
self.object_post(obj, update=True, public=False) |
|
| 1174 |
r = self.object_post(obj, update=True, public=False) |
|
| 1175 |
return r.headers |
|
| 1167 | 1176 |
|
| 1168 | 1177 |
def get_object_info(self, obj, version=None): |
| 1169 | 1178 |
""" |
| ... | ... | |
| 1255 | 1264 |
filesize = fstat(source_file.fileno()).st_size |
| 1256 | 1265 |
nblocks = 1 + (filesize - 1) // blocksize |
| 1257 | 1266 |
offset = 0 |
| 1267 |
headers = {}
|
|
| 1258 | 1268 |
if upload_cb: |
| 1259 |
upload_gen = upload_cb(nblocks) |
|
| 1260 |
upload_gen.next() |
|
| 1261 |
for i in range(nblocks): |
|
| 1262 |
block = source_file.read(min(blocksize, filesize - offset)) |
|
| 1263 |
offset += len(block) |
|
| 1264 |
self.object_post( |
|
| 1265 |
obj, |
|
| 1266 |
update=True, |
|
| 1267 |
content_range='bytes */*', |
|
| 1268 |
content_type='application/octet-stream', |
|
| 1269 |
content_length=len(block), |
|
| 1270 |
data=block) |
|
| 1269 |
self.progress_bar_gen = upload_cb(nblocks) |
|
| 1270 |
self._cb_next() |
|
| 1271 |
flying = {}
|
|
| 1272 |
self._init_thread_limit() |
|
| 1273 |
try: |
|
| 1274 |
for i in range(nblocks): |
|
| 1275 |
block = source_file.read(min(blocksize, filesize - offset)) |
|
| 1276 |
offset += len(block) |
|
| 1271 | 1277 |
|
| 1272 |
if upload_cb: |
|
| 1273 |
upload_gen.next() |
|
| 1278 |
self._watch_thread_limit(flying.values()) |
|
| 1279 |
unfinished = {}
|
|
| 1280 |
flying[i] = SilentEvent( |
|
| 1281 |
method=self.object_post, |
|
| 1282 |
obj=obj, |
|
| 1283 |
update=True, |
|
| 1284 |
content_range='bytes */*', |
|
| 1285 |
content_type='application/octet-stream', |
|
| 1286 |
content_length=len(block), |
|
| 1287 |
data=block) |
|
| 1288 |
flying[i].start() |
|
| 1289 |
|
|
| 1290 |
for key, thread in flying.items(): |
|
| 1291 |
if thread.isAlive(): |
|
| 1292 |
if i < nblocks: |
|
| 1293 |
unfinished[key] = thread |
|
| 1294 |
continue |
|
| 1295 |
thread.join() |
|
| 1296 |
if thread.exception: |
|
| 1297 |
raise thread.exception |
|
| 1298 |
headers[key] = thread.value.headers |
|
| 1299 |
self._cb_next() |
|
| 1300 |
flying = unfinished |
|
| 1301 |
except KeyboardInterrupt: |
|
| 1302 |
sendlog.info('- - - wait for threads to finish')
|
|
| 1303 |
for thread in activethreads(): |
|
| 1304 |
thread.join() |
|
| 1305 |
return headers.values() |
|
| 1274 | 1306 |
|
| 1275 | 1307 |
def truncate_object(self, obj, upto_bytes): |
| 1276 | 1308 |
""" |
| b/kamaki/clients/pithos/test.py | ||
|---|---|---|
| 1159 | 1159 |
|
| 1160 | 1160 |
# Pithos+ only methods |
| 1161 | 1161 |
|
| 1162 |
@patch('%s.container_put' % pithos_pkg, return_value=FR())
|
|
| 1163 |
def test_create_container(self, CP): |
|
| 1164 |
FR.headers = container_info |
|
| 1165 |
cont = 'an0th3r_c0n741n3r' |
|
| 1166 |
|
|
| 1167 |
r = self.client.create_container() |
|
| 1168 |
self.assert_dicts_are_equal(r, container_info) |
|
| 1169 |
CP.assert_called_once_with(quota=None, versioning=None, metadata=None) |
|
| 1170 |
|
|
| 1171 |
bu_cont = self.client.container |
|
| 1172 |
r = self.client.create_container(cont) |
|
| 1173 |
self.assertEqual(self.client.container, bu_cont) |
|
| 1174 |
self.assert_dicts_are_equal(r, container_info) |
|
| 1175 |
self.assertEqual( |
|
| 1176 |
CP.mock_calls[-1], |
|
| 1177 |
call(quota=None, versioning=None, metadata=None)) |
|
| 1178 |
|
|
| 1179 |
meta = dict(k1='v1', k2='v2') |
|
| 1180 |
r = self.client.create_container(cont, 42, 'auto', meta) |
|
| 1181 |
self.assertEqual(self.client.container, bu_cont) |
|
| 1182 |
self.assert_dicts_are_equal(r, container_info) |
|
| 1183 |
self.assertEqual( |
|
| 1184 |
CP.mock_calls[-1], |
|
| 1185 |
call(quota=42, versioning='auto', metadata=meta)) |
|
| 1186 |
|
|
| 1162 | 1187 |
@patch('%s.container_delete' % pithos_pkg, return_value=FR())
|
| 1163 | 1188 |
def test_purge_container(self, CD): |
| 1164 | 1189 |
self.client.purge_container() |
| ... | ... | |
| 1672 | 1697 |
upload_cb=append_gen if turn else None) |
| 1673 | 1698 |
self.assertEqual((turn + 1) * num_of_blocks, len(post.mock_calls)) |
| 1674 | 1699 |
(args, kwargs) = post.mock_calls[-1][1:3] |
| 1675 |
self.assertEqual(args, (obj,))
|
|
| 1700 |
self.assertEqual(kwargs['obj'], obj)
|
|
| 1676 | 1701 |
self.assertEqual(kwargs['content_length'], len(kwargs['data'])) |
| 1677 | 1702 |
fsize = num_of_blocks * int(kwargs['content_length']) |
| 1678 | 1703 |
self.assertEqual(fsize, file_size) |
Also available in: Unified diff