global-include */templates/* */fixtures/* */static/*
global-exclude */.DS_Store
-exclude */settings.py
-include */settings.py.dist
+include pithos/settings.d/*
prune docs
-prune tools
+prune other
This server is useful during development, but should not be used for deployment.
To deploy Pithos using Apache, take a look at the Administrator Guide in docs.
+
+Using the tools
+---------------
+
+In the pithos/tools directory you will find the following utilities:
+
+ pithos-sh Pithos shell
+ pithos-sync Pithos synchronization client
+ pithos-fs Pithos FUSE implementation
+ pithos-test Pithos server tests
+
+Also, the pithos/lib directory contains a python library that can be used
+to access Pithos and manage stored objects. All tools use the included lib.
+
+Connection options can be set via environmental variables:
+
+ PITHOS_USER Login user
+ PITHOS_AUTH Login token
+ PITHOS_SERVER Pithos server (default: plus.pithos.grnet.gr)
+ PITHOS_API Pithos server path (default: v1)
+ PITHOS_SYNC_CONTAINER Container to sync with (default: pithos)
:members:
:undoc-members:
-Hashfiler
-~~~~~~~~~
+Store
+~~~~~
-.. automodule:: pithos.backends.lib.hashfiler
+.. automodule:: pithos.backends.lib.hashfiler.store
:show-inheritance:
:members:
:undoc-members:
Client Library
==============
-.. automodule:: tools.lib.client
+.. automodule:: pithos.lib.client
:show-inheritance:
:members:
:undoc-members:
from pithos import settings
from pithos.backends.modular import ModularBackend
-from lib.hashmap import HashMap
-from lib.migrate import Migration, Cache
+from pithos.lib.hashmap import HashMap
+
+from migrate import Migration, Cache
import os
from pithos.backends.modular import CLUSTER_NORMAL, CLUSTER_HISTORY, CLUSTER_DELETED
from pithos.backends.lib.sqlalchemy.node import Node, ROOTNODE
-from lib.transfer import upload
-from lib.hashmap import HashMap
-from lib.client import Fault
-from lib.migrate import Migration, Cache
+from pithos.lib.transfer import upload
+from pithos.lib.hashmap import HashMap
+from pithos.lib.client import Fault
+
+from migrate import Migration, Cache
+
from calendar import timegm
from decimal import Decimal
from collections import defaultdict
ot.create_objects()
f.close()
-
\ No newline at end of file
+
# interpreted as representing official policies, either expressed
# or implied, of GRNET S.A.
-from lib.migrate import Migration
-
from sqlalchemy import Table
from sqlalchemy.sql import select
from pithos.im.models import User
+from migrate import Migration
+
import base64
class UserMigration(Migration):
if __name__ == "__main__":
db = ''
m = UserMigration(db)
- m.execute()
\ No newline at end of file
+ m.execute()
+# Copyright (c) Django Software Foundation and individual contributors.
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without modification,
+# are permitted provided that the following conditions are met:
+#
+# 1. Redistributions of source code must retain the above copyright notice,
+# this list of conditions and the following disclaimer.
+#
+# 2. Redistributions in binary form must reproduce the above copyright
+# notice, this list of conditions and the following disclaimer in the
+# documentation and/or other materials provided with the distribution.
+#
+# 3. Neither the name of Django nor the names of its contributors may be used
+# to endorse or promote products derived from this software without
+# specific prior written permission.
+#
+# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
+# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+
+VERSION = (0, 8, 2, 'alpha', 0)
+
+def get_version():
+ version = '%s.%s' % (VERSION[0], VERSION[1])
+ if VERSION[2]:
+ version = '%s.%s' % (version, VERSION[2])
+ if VERSION[3:] == ('alpha', 0):
+ version = '%s pre-alpha' % version
+ else:
+ if VERSION[3] != 'final':
+ version = '%s %s %s' % (version, VERSION[3], VERSION[4])
+ return version
from django.core.files.uploadhandler import FileUploadHandler
from django.core.files.uploadedfile import UploadedFile
-from pithos.api.compat import parse_http_date_safe, parse_http_date
+from pithos.lib.compat import parse_http_date_safe, parse_http_date
+
from pithos.api.faults import (Fault, NotModified, BadRequest, Unauthorized, Forbidden, ItemNotFound,
Conflict, LengthRequired, PreconditionFailed, RequestEntityTooLarge,
RangeNotSatisfiable, ServiceUnavailable)
import os
import time
import logging
-import hashlib
import binascii
from base import NotAllowedError, QuotaError, BaseBackend
+from pithos.lib.hashmap import HashMap
+
( CLUSTER_NORMAL, CLUSTER_HISTORY, CLUSTER_DELETED ) = range(3)
inf = float('inf')
logger = logging.getLogger(__name__)
-class HashMap(list):
-
- def __init__(self, blocksize, blockhash):
- super(HashMap, self).__init__()
- self.blocksize = blocksize
- self.blockhash = blockhash
-
- def _hash_raw(self, v):
- h = hashlib.new(self.blockhash)
- h.update(v)
- return h.digest()
-
- def hash(self):
- if len(self) == 0:
- return self._hash_raw('')
- if len(self) == 1:
- return self.__getitem__(0)
-
- h = list(self)
- s = 2
- while s < len(h):
- s = s * 2
- h += [('\x00' * len(h[0]))] * (s - len(h))
- while len(h) > 1:
- h = [self._hash_raw(h[x] + h[x + 1]) for x in range(0, len(h), 2)]
- return h[0]
-
-
def backend_method(func=None, autocommit=1):
if func is None:
def fn(func):
sys.stderr.write("Error: Can't find the file 'settings.py' in the directory containing %r. It appears you've customized things.\nYou'll have to run django-admin.py, passing it your settings module.\n(If the file settings.py does indeed exist, it's causing an ImportError somehow.)\n" % __file__)
sys.exit(1)
-if __name__ == "__main__":
+def main():
execute_manager(settings)
+
+if __name__ == "__main__":
+ main()
if exists(conf):
execfile(conf)
+elif exists('/etc/pithos/settings.local'):
+ execfile('/etc/pithos/settings.local')
--- /dev/null
+Tools in this dir depend on being able to import pithos.
+You need to adjust PYTHONPATH accordingly for this to work.
+e.g.
+ export PYTHONPATH=$HOME/pithos
from sys import argv
from time import time
-from lib.compat import parse_http_date
-
-from lib.client import OOS_Client, Fault
-from lib.fuse import FUSE, FuseOSError, Operations
-from lib.util import get_user, get_auth, get_server
+from pithos.lib.compat import parse_http_date
+from pithos.lib.client import OOS_Client, Fault
+from pithos.lib.fuse import FUSE, FuseOSError, Operations
+from pithos.lib.util import get_user, get_auth, get_server
epoch = int(time())
from os import environ
from sys import argv, exit, stdin, stdout
from datetime import datetime
-from lib.client import Pithos_Client, Fault
-from lib.util import get_user, get_auth, get_server, get_api
-from lib.transfer import upload, download
+
+from pithos.lib.client import Pithos_Client, Fault
+from pithos.lib.util import get_user, get_auth, get_server, get_api
+from pithos.lib.transfer import upload, download
import json
import logging
from shutil import copyfile
from time import time
-from lib.transfer import download, upload
-from lib.client import Pithos_Client, Fault
-from lib.hashmap import merkle
-from lib.util import get_user, get_auth, get_server
+from pithos.lib.transfer import download, upload
+from pithos.lib.client import Pithos_Client, Fault
+from pithos.lib.hashmap import merkle
+from pithos.lib.util import get_user, get_auth, get_server
DEFAULT_CONTAINER = 'pithos'
# interpreted as representing official policies, either expressed
# or implied, of GRNET S.A.
-from lib.client import Pithos_Client, Fault
-from lib.util import get_user, get_auth, get_server, get_api
+from pithos.lib.client import Pithos_Client, Fault
+from pithos.lib.util import get_user, get_auth, get_server, get_api
+
from xml.dom import minidom
from StringIO import StringIO
from hashlib import new as newhasher
all_files = 1
build-dir = docs/build
source-dir = docs/source
-
import os
from setuptools import setup, find_packages
+from pithos import get_version
-VERSION = os.popen("git describe --abbrev=0 --tags").read().strip(' \nv')
+
+def read(fname):
+ return open(os.path.join(os.path.dirname(__file__), fname)).read()
+
+
+VERSION = get_version().replace(' ', '')
+
+INSTALL_REQUIRES = [
+ 'Django==1.2.3',
+ 'South==0.7',
+ 'httplib2==0.6.0',
+ 'SQLAlchemy==0.6.3',
+ 'MySQL-python==1.2.2',
+ 'psycopg2==2.2.1'
+]
setup(
name='Pithos',
version=VERSION,
- description='Pithos file storage service',
+ description='Pithos file storage service and tools',
+ long_description=read('README'),
author='GRNET',
author_email='pithos@grnet.gr',
url='http://code.grnet.gr/projects/pithos',
packages=find_packages(),
+ #install_requires = INSTALL_REQUIRES,
license='BSD',
classifiers=[
'Development Status :: 3 - Alpha',
'Programming Language :: Python',
'Topic :: Utilities',
'License :: OSI Approved :: BSD License',
+ ],
+ entry_points={
+ 'console_scripts': ['pithos-manage = pithos.manage:main']
+ },
+ scripts=[
+ 'pithos/tools/pithos-sh',
+ 'pithos/tools/pithos-sync',
+ 'pithos/tools/pithos-fs',
+ 'pithos/tools/pithos-test'
]
)
+++ /dev/null
-exclude lib/migrate*
+++ /dev/null
-README
-======
-
-Pithos is a file storage service, built by GRNET.
-Learn more about Pithos at: http://code.grnet.gr/projects/pithos
-
-Here you will find the following tools:
-
- pithos-sh Pithos shell
- pithos-sync Pithos synchronization client
- pithos-fs Pithos FUSE implementation
-
-Also, the lib folder contains a python library that can be
-used to access Pithos and manage stored objects.
-All tools use the included lib.
-
-Connection options can be set via environmental variables:
-
- PITHOS_USER Login user
- PITHOS_AUTH Login token
- PITHOS_SERVER Pithos server (default: plus.pithos.grnet.gr)
- PITHOS_API Pithos server path (default: v1)
- PITHOS_SYNC_CONTAINER Container to sync with (default: pithos)
+++ /dev/null
-# Copyright (c) Django Software Foundation and individual contributors.
-# All rights reserved.
-#
-# Redistribution and use in source and binary forms, with or without modification,
-# are permitted provided that the following conditions are met:
-#
-# 1. Redistributions of source code must retain the above copyright notice,
-# this list of conditions and the following disclaimer.
-#
-# 2. Redistributions in binary form must reproduce the above copyright
-# notice, this list of conditions and the following disclaimer in the
-# documentation and/or other materials provided with the distribution.
-#
-# 3. Neither the name of Django nor the names of its contributors may be used
-# to endorse or promote products derived from this software without
-# specific prior written permission.
-#
-# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
-# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
-# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
-# DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR
-# ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
-# (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
-# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
-# ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
-# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
-# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-
-import re
-import datetime
-import calendar
-
-MONTHS = 'jan feb mar apr may jun jul aug sep oct nov dec'.split()
-__D = r'(?P<day>\d{2})'
-__D2 = r'(?P<day>[ \d]\d)'
-__M = r'(?P<mon>\w{3})'
-__Y = r'(?P<year>\d{4})'
-__Y2 = r'(?P<year>\d{2})'
-__T = r'(?P<hour>\d{2}):(?P<min>\d{2}):(?P<sec>\d{2})'
-RFC1123_DATE = re.compile(r'^\w{3}, %s %s %s %s GMT$' % (__D, __M, __Y, __T))
-RFC850_DATE = re.compile(r'^\w{6,9}, %s-%s-%s %s GMT$' % (__D, __M, __Y2, __T))
-ASCTIME_DATE = re.compile(r'^\w{3} %s %s %s %s$' % (__M, __D2, __T, __Y))
-
-def parse_http_date(date):
- """
- Parses a date format as specified by HTTP RFC2616 section 3.3.1.
-
- The three formats allowed by the RFC are accepted, even if only the first
- one is still in widespread use.
-
- Returns an floating point number expressed in seconds since the epoch, in
- UTC.
- """
- # emails.Util.parsedate does the job for RFC1123 dates; unfortunately
- # RFC2616 makes it mandatory to support RFC850 dates too. So we roll
- # our own RFC-compliant parsing.
- for regex in RFC1123_DATE, RFC850_DATE, ASCTIME_DATE:
- m = regex.match(date)
- if m is not None:
- break
- else:
- raise ValueError("%r is not in a valid HTTP date format" % date)
- try:
- year = int(m.group('year'))
- if year < 100:
- if year < 70:
- year += 2000
- else:
- year += 1900
- month = MONTHS.index(m.group('mon').lower()) + 1
- day = int(m.group('day'))
- hour = int(m.group('hour'))
- min = int(m.group('min'))
- sec = int(m.group('sec'))
- result = datetime.datetime(year, month, day, hour, min, sec)
- return calendar.timegm(result.utctimetuple())
- except Exception:
- raise ValueError("%r is not a valid date" % date)
-
-def parse_http_date_safe(date):
- """
- Same as parse_http_date, but returns None if the input is invalid.
- """
- try:
- return parse_http_date(date)
- except Exception:
- pass
+++ /dev/null
-#!/usr/bin/env python
-
-import os
-
-from setuptools import setup, find_packages
-
-VERSION = os.popen("git describe --abbrev=0 --tags").read().strip(' \nv')
-
-setup(
- name='Pithos Tools',
- version=VERSION,
- description='Pithos file storage service tools',
- author='GRNET',
- author_email='pithos@grnet.gr',
- url='http://code.grnet.gr/projects/pithos',
- scripts=['pithos-sh', 'pithos-sync', 'pithos-fs'],
- packages=['lib'],
- license='BSD',
- classifiers=[
- 'Development Status :: 3 - Alpha',
- 'Operating System :: OS Independent',
- 'Programming Language :: Python',
- 'Topic :: Utilities',
- 'License :: OSI Approved :: BSD License',
- ]
-)
projects / pithos / commitdiff