Synnefo

# Copyright 2012 GRNET S.A. All rights reserved.

#

# Redistribution and use in source and binary forms, with or

# without modification, are permitted provided that the following

# conditions are met:

#

#   1. Redistributions of source code must retain the above

#      copyright notice, this list of conditions and the following

#      disclaimer.

#

#   2. Redistributions in binary form must reproduce the above

#      copyright notice, this list of conditions and the following

#      disclaimer in the documentation and/or other materials

#      provided with the distribution.

#

# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS

# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED

# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR

# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR

# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,

# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT

# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF

# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED

# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT

# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN

# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE

# POSSIBILITY OF SUCH DAMAGE.

#

# The views and conclusions contained in the software and

# documentation are those of the authors and should not be

# interpreted as representing official policies, either expressed

# or implied, of GRNET S.A.

from pithos.api.util import get_backend, split_container_object_string

import re

import hashlib

import os

def data_read_iterator(str, size=1024):

    offset = 0

    while True:

        data = str[offset:offset + size]

        offset = offset + size

        if not data:

            break

        yield data

class ManageAccounts():

    def __init__(self):

        self.backend = get_backend()

    def cleanup(self):

        self.backend.close()

    def existing_accounts(self):

        return sorted([path for path, _ in self.backend.node.node_accounts()])

    def duplicate_accounts(self):

        accounts = self.existing_accounts()

        duplicates = []

        for i in range(len(accounts)):

            account = accounts[i]

            matcher = re.compile(account, re.IGNORECASE)

            duplicate = filter(matcher.match, (i for i in accounts[i + 1:] \

                if len(i) == len(account)))

            if duplicate:

                duplicate.insert(0, account)

                duplicates.append(duplicate)

        return duplicates

    def list_all_containers(self, account, step=10):

        containers = []

        marker = None

        while 1:

            more = self.backend.list_containers(account, account, limit=10,

                                                marker=marker)

            if not more:

                break

            containers.extend(more)

            marker = more[-1]

        return containers

    def list_all_container_objects(self, account, container, virtual=False):

        objects = []

        marker = None

        while 1:

            more = self.backend.list_objects(account, account, container,

                                             marker=marker, virtual=virtual)

            if not more:

                break

            objects.extend((i[0] for i in more))

            marker = more[-1][0]

        return objects

    def list_all_objects(self, account, virtual=False):

        containers = self.list_all_containers(account)

        objects = []

        extend = objects.extend

        for c in containers:

            more = self.list_all_container_objects(account, c, virtual=virtual)

            extend([os.path.join(c, i) for i in more])

        return objects

    def list_past_versions(self, account, container, name):

        versions = self.backend.list_versions(account, account, container,

                                              name)

        # do not return the current version

        return list(x[0] for x in versions[:-1])

    def move_object(self, src_account, src_container, src_name,

                    dest_account, dry=True, silent=False):

        if src_account not in self.existing_accounts():

            raise NameError('%s does not exist' % src_account)

        if dest_account not in self.existing_accounts():

            raise NameError('%s does not exist' % dest_account)

        trans = self.backend.wrapper.conn.begin()

        try:

            self._copy_object(src_account, src_container, src_name,

                              dest_account, move=True)

            if dry:

                if not silent:

                    print "Skipping database commit."

                trans.rollback()

            else:

                trans.commit()

                if not silent:

                    print "%s is deleted." % src_account

        except:

            trans.rollback()

            raise

    def _copy_object(self, src_account, src_container, src_name,

                    dest_account, move=False):

        path = os.path.join(src_container, src_name)

        fullpath = os.path.join(src_account, path)

        dest_container = src_container

        dest_name = src_name

        meta = self.backend.get_object_meta(src_account, src_account,

                                            src_container, src_name, 'pithos',

                                            version=None)

        content_type = meta.get('type')

        # get source object history

        versions = self.list_past_versions(src_account, src_container,

                                           src_name)

        # get source object permissions

        permissions = self.backend.permissions.access_get(fullpath)

        # get source object public

        public = self.backend.get_object_public(src_account, src_account,

                                                src_container, src_name)

        if dest_container in self.backend.list_containers(dest_account,

                                                          dest_account):

            # Note: if dest_container contains an object with the same name

            # a new version with the contents of the source object will be

            # created and the one in the destination container will pass to

            # history

            self.backend.copy_object(dest_account, src_account, src_container,

                                     src_name, dest_account, dest_container,

                                     dest_name, content_type, 'pithos',

                                     meta={}, replace_meta=False,

                                     permissions=permissions)

        else:

            # create destination container and retry

            self.backend.put_container(dest_account, dest_account,

                                       dest_container)

            self.backend.copy_object(dest_account, src_account, src_container,

                                     src_name, dest_account, dest_container,

                                     dest_name, content_type, 'pithos',

                                     meta={}, replace_meta=False,

                                     permissions=permissions)

        if move:

            self.backend.delete_object(src_account, src_account,

                                       src_container, src_name)

        dest_path, dest_node = self.backend._lookup_object(dest_account,

                                                           dest_container,

                                                           dest_name)

        assert dest_path == '/'.join([dest_account, path])

        # turn history versions to point to the newly created node

        for serial in versions:

            self.backend.node.version_put_property(serial, 'node', dest_node)

        if public:

            # set destination object public

            fullpath = '/'.join([dest_account, dest_container, dest_name])

            self.backend.permissions.public_set(

                fullpath,

                self.backend.public_url_security,

                self.backend.public_url_alphabet

            )

    def _merge_account(self, src_account, dest_account, delete_src=False):

            # TODO: handle exceptions

            # copy all source objects

            for path in self.list_all_objects(src_account):

                src_container, src_name = split_container_object_string(

                    '/%s' % path)

                # give read permissions to the dest_account

                permissions = self.backend.get_object_permissions(

                    src_account, src_account, src_container, src_name)

                if permissions:

                    permissions = permissions[2]

                permissions['read'] = permissions.get('read', [])

                permissions['read'].append(dest_account)

                self.backend.update_object_permissions(src_account,

                                                       src_account,

                                                       src_container,

                                                       src_name,

                                                       permissions)

                self._copy_object(src_account, src_container, src_name,

                                 dest_account, move=delete_src)

            # move groups also

            groups = self.backend.get_account_groups(src_account, src_account)

            (v.replace(src_account, dest_account) for v in groups.values())

            self.backend.update_account_groups(dest_account, dest_account,

                                               groups)

            if delete_src:

                self._delete_account(src_account)

    def merge_account(self, src_account, dest_account, only_stats=True,

                      dry=True, silent=False, delete_src=False):

        if src_account not in self.existing_accounts():

            raise NameError('%s does not exist' % src_account)

        if dest_account not in self.existing_accounts():

            raise NameError('%s does not exist' % dest_account)

        if only_stats:

            print "The following %s's entries will be moved to %s:" \

                % (src_account, dest_account)

            print "Objects: %r" % self.list_all_objects(src_account)

            print "Groups: %r" \

                % self.backend.get_account_groups(src_account,

                                                  src_account).keys()

            return

        trans = self.backend.wrapper.conn.begin()

        try:

            self._merge_account(src_account, dest_account, delete_src)

            if dry:

                if not silent:

                    print "Skipping database commit."

                trans.rollback()

            else:

                trans.commit()

                if not silent:

                    msg = "%s merged into %s."

                    print msg % (src_account, dest_account)

        except:

            trans.rollback()

            raise

    def delete_container_contents(self, account, container):

        self.backend.delete_container(account, account, container,

                                      delimiter='/')

    def delete_container(self, account, container):

        self.backend.delete_container(account, account, container)

    def _delete_account(self, account):

        for c in self.list_all_containers(account):

            self.delete_container_contents(account, c)

            self.delete_container(account, c)

        self.backend.delete_account(account, account)

    def delete_account(self, account, only_stats=True, dry=True, silent=False):

        if account not in self.existing_accounts():

            raise NameError('%s does not exist' % account)

        if only_stats:

            print "The following %s's entries will be removed:" % account

            print "Objects: %r" % self.list_all_objects(account)

            print "Groups: %r" \

                % self.backend.get_account_groups(account, account).keys()

            return

        trans = self.backend.wrapper.conn.begin()

        try:

            self._delete_account(account)

            if dry:

                if not silent:

                    print "Skipping database commit."

                trans.rollback()

            else:

                trans.commit()

                if not silent:

                    print "%s is deleted." % account

        except:

            trans.rollback()

            raise

    def create_account(self, account):

        return self.backend._lookup_account(account, create=True)

    def create_update_object(self, account, container, name, content_type,

                             data, meta=None, permissions=None, request_user=None):

        meta = meta or {}

        permissions = permissions or {}

        md5 = hashlib.md5()

        size = 0

        hashmap = []

        for block_data in data_read_iterator(data, self.backend.block_size):

            size += len(block_data)

            hashmap.append(self.backend.put_block(block_data))

            md5.update(block_data)

        checksum = md5.hexdigest().lower()

        request_user = request_user or account

        return self.backend.update_object_hashmap(request_user, account,

                                                  container, name, size,

                                                  content_type, hashmap,

                                                  checksum, 'pithos', meta,

                                                  True, permissions)

# Copyright 2012 GRNET S.A. All rights reserved.

#

# Redistribution and use in source and binary forms, with or

# without modification, are permitted provided that the following

# conditions are met:

#

#   1. Redistributions of source code must retain the above

#      copyright notice, this list of conditions and the following

#      disclaimer.

#

#   2. Redistributions in binary form must reproduce the above

#      copyright notice, this list of conditions and the following

#      disclaimer in the documentation and/or other materials

#      provided with the distribution.

#

# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS

# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED

# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR

# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR

# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,

# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT

# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF

# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED

# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT

# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN

# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE

# POSSIBILITY OF SUCH DAMAGE.

#

# The views and conclusions contained in the software and

# documentation are those of the authors and should not be

# interpreted as representing official policies, either expressed

# or implied, of GRNET S.A.

from pithos.api.manage_accounts import ManageAccounts

from synnefo.webproject.management.utils import pprint_table

import argparse

import os

import sys

def _double_list_str(l):

    return '\n'.join(', '.join(sublist) for sublist in l)

def list(args):

    try:

        utils = ManageAccounts()

        if args.only_duplicate:

            accounts = utils.duplicate_accounts()

        else:

            accounts = utils.existing_accounts()

        headers = ['uuid']

        table = [(a,) for a in accounts]

        if args.output_format != "json" and not args.headers:

            headers = None

        pprint_table(sys.stdout, table, headers, args.output_format)

    except Exception, e:

        sys.stderr.write('%s\n' % e)

    finally:

        utils.cleanup()

def delete(args):

    try:

        utils = ManageAccounts()

        utils.delete_account(args.delete_account, only_stats=True)

        confirm = raw_input(

            "Type 'yes' if you are sure you want "

            "to remove those entries: "

        )

        if not confirm == 'yes':

            return

        else:

            utils.delete_account(

                args.delete_account, only_stats=False, dry=args.dry

            )

    except Exception, e:

        sys.stderr.write('%s\n' % e)

    finally:

        utils.cleanup()

def merge(args):

    try:

        utils = ManageAccounts()

        utils.merge_account(

            args.src_account, args.dest_account, only_stats=True

        )

        confirm = raw_input(

            "Type 'yes' if you are sure you want"

            " to move those entries to %s: " % args.dest_account

        )

        if not confirm == 'yes':

            return

        else:

            utils.merge_account(

                args.src_account, args.dest_account, only_stats=False,

                dry=args.dry

            )

    except Exception, e:

        sys.stderr.write('%s\n' % e)

    finally:

        utils.cleanup()

def export_quota(args):

    try:

        utils = ManageAccounts()

        d = utils.backend.node.node_account_quotas()

        location = os.path.abspath(os.path.expanduser(args.location))

        f = open(location, 'w')

        for uuid, capacity in d.iteritems():

            f.write(' '.join([uuid, 'pithos.diskspace', capacity]))

            f.write('\n')

    except Exception, e:

        sys.stderr.write('%s\n' % e)

    finally:

        utils.cleanup()

def set_container_quota(args):

    try:

        utils = ManageAccounts()

        try:

            quota = int(args.quota)

        except:

            raise ValueError('Invalid quota')

        accounts = [args.account] if args.account \

            else utils.existing_accounts()

        failed = []

        def update_container_policy(account):

            trans = utils.backend.wrapper.conn.begin()

            try:

                utils.backend.update_container_policy(

                    account, account, args.container, {'quota': quota}

                )

                if args.dry:

                    print "Skipping database commit."

                    trans.rollback()

                else:

                    trans.commit()

            except Exception, e:

                failed.append((account, e))

        map(update_container_policy, accounts)

        if failed:

            sys.stdout.write(

                'Failed for the following accounts:\n'

            )

            pprint_table(sys.stdout, failed, headers=[])

    except Exception, e:

        sys.stderr.write('%s\n' % e)

    finally:

        utils.cleanup()

def main(argv=None):

    # create the top-level parser

    parser = argparse.ArgumentParser()

    subparsers = parser.add_subparsers()

    # create the parser for the "list" command

    parser_list = subparsers.add_parser(

        'list', description="List existing accounts"

    )

    parser_list.add_argument(

        '--dublicate', dest='only_duplicate', action="store_true",

        default=False, help="Display only case insensitive duplicate accounts."

    )

    parser_list.add_argument(

        "--no-headers", dest="headers", action="store_false", default=True,

        help="Do not display headers"

    )

    parser_list.add_argument(

        "--output-format", dest="output_format", metavar="[pretty, csv, json]",

        default="pretty", choices=["pretty", "csv", "json"],

        help="Select the output format: pretty [the default], tabs"

             " [tab-separated output], csv [comma-separated output]")

    parser_list.set_defaults(func=list)

    # create the parser for the "delete" command

    parser_delete = subparsers.add_parser('delete')

    parser_delete.add_argument('delete_account')

    parser_delete.add_argument(

        '--dry', action="store_true", default=False,

        help="Do not commit database changes."

    )

    parser_delete.set_defaults(func=delete)

    # create the parser for the "merge" command

    parser_merge = subparsers.add_parser('merge')

    parser_merge.add_argument('src_account')

    parser_merge.add_argument('dest_account')

    parser_merge.add_argument(

        '--dry', action="store_true", default=False,

        help="Do not commit database changes."

    )

    parser_merge.set_defaults(func=merge)

    # create the parser for the "export-quota" command

    parser_export_quota = subparsers.add_parser('export-quota')

    parser_export_quota.add_argument(

        '--location', dest='location', required=True,

        help="Where to save the exported quotas"

    )

    parser_export_quota.set_defaults(func=export_quota)

    # create the parser for the "set-container-quota" command

    parser_set_container_quota = subparsers.add_parser(

        'set-container-quota',

        description="Set container quota for all the existing accounts"

    )

    parser_set_container_quota.add_argument(

        '--account', dest='account',

        help="Set container quota for a specific account"

    )

    parser_set_container_quota.add_argument(

        '--dry', action="store_true", default=False,

        help="Do not commit database changes."

    )

    parser_set_container_quota.add_argument('container')

    parser_set_container_quota.add_argument('quota')

    parser_set_container_quota.set_defaults(func=set_container_quota)

    args = parser.parse_args()

    args.func(args)

if __name__ == '__main__':

    main(sys.argv)

# Copyright 2011 GRNET S.A. All rights reserved.

#

# Redistribution and use in source and binary forms, with or

# without modification, are permitted provided that the following

# conditions are met:

#

#   1. Redistributions of source code must retain the above

#      copyright notice, this list of conditions and the following

#      disclaimer.

#

#   2. Redistributions in binary form must reproduce the above

#      copyright notice, this list of conditions and the following

#      disclaimer in the documentation and/or other materials

#      provided with the distribution.

#

# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS

# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED

# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR

# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR

# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,

# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT

# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF

# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED

# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT

# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN

# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE

# POSSIBILITY OF SUCH DAMAGE.

#

# The views and conclusions contained in the software and

# documentation are those of the authors and should not be

# interpreted as representing official policies, either expressed

# or implied, of GRNET S.A.

import unittest

import uuid

import random

import string

import os

from collections import defaultdict

from pithos.api.manage_accounts import ManageAccounts

def get_random_data(length=500):

    char_set = string.ascii_uppercase + string.digits

    return ''.join(random.choice(char_set) for x in xrange(length))

class ManageAccountsTests(unittest.TestCase):

    def setUp(self):

        self.utils = ManageAccounts()

        self.accounts = ('account1', 'Account1', 'account2', 'account3')

        for i in self.accounts:

            self.utils.create_account(i)

    def tearDown(self):

        for i in self.accounts:

            self.utils._delete_account(i)

        self.utils.cleanup()

    def _verify_object(self, account, container, object, expected=None,

                       strict=True):

        expected = expected or {}

        self._verify_object_metadata(account, container, object,

                                     expected.get('meta'))

        self._verify_object_history(account, container, object,

                                    expected.get('versions'),

                                    strict=strict)

        self._verify_object_permissions(account, container, object,

                                        expected.get('permissions'))

    def _verify_object_metadata(self, account, container, object, expected):

        object_meta = self.utils.backend.get_object_meta(

            account, account, container, object, 'pithos')

        for k in expected:

            self.assertTrue(k in object_meta)

            self.assertEquals(object_meta[k], expected[k])

    def _verify_object_history(self, account, container, object, expected,

                               strict=True):

        history = self.utils.list_past_versions(account, container, object)

        if strict:

            self.assertEquals(sorted(expected), history)

        else:

            self.assertTrue(set(expected) <= set(history))

    def _verify_object_permissions(self, account, container, object, expected):

        expected = expected or {}

        perms_tuple = self.utils.backend.get_object_permissions(

            account, account, container, object)

        self.assertEqual(len(perms_tuple), 3)

        object_perms = perms_tuple[2]

        for k in expected:

            self.assertTrue(set(expected.get(k)) <= set(object_perms.get(k)))

        for holder in expected.get('read', []):

            if holder == '*':

                continue

            try:

                # check first for a group permission

                owner, group = holder.split(':', 1)

            except ValueError:

                holders = [holder]

            else:

                holders = self.utils.backend.permissions.group_members(owner,

                                                                       group)

            for h in holders:

                try:

                    self.utils.backend.get_object_meta(

                        holder, account, container, object, 'pithos')

                except Exception, e:

                    self.fail(e)

    def test_existing_accounts(self):

        accounts = self.utils.existing_accounts()

        self.assertEquals(sorted(accounts), accounts)

        self.assertTrue(set(['account1', 'account2']) <= set(accounts))

    def test_duplicate_accounts(self):

        duplicates = self.utils.duplicate_accounts()

        self.assertTrue(['Account1', 'account1'] in duplicates)

    def test_list_all_containers(self):

        step = 10

        containers = []

        append = containers.append

        for i in range(3 * step + 1):

            while 1:

                cname = unicode(uuid.uuid4())

                if cname not in containers:

                    append(cname)

                    break

            self.utils.backend.put_container('account1', 'account1', cname)

        self.assertEquals(sorted(containers),

                          self.utils.list_all_containers('account1',

                                                         step=step))

    def test_list_all_container_objects(self):

        containers = ('container1', 'container2')

        objects = defaultdict(list)

        for c in containers:

            self.utils.backend.put_container('account1', 'account1', c)

            step = 10

            append = objects[c].append

            content_type = 'application/octet-stream'

            for i in range(3 * step + 1):

                while 1:

                    oname = unicode(uuid.uuid4())

                    if oname not in objects:

                        append(oname)

                        break

                data = get_random_data(int(random.random()))

                self.utils.create_update_object('account1', c, oname,

                                                content_type, data)

        (self.assertEquals(sorted(objects.get(c)),

                           self.utils.list_all_container_objects('account1', c)

                           ) for c in containers)

    def test_list_all_objects(self):

        containers = ('container1', 'container2')

        objects = []

        append = objects.append

        for c in containers:

            self.utils.backend.put_container('account1', 'account1', c)

            step = 10

            content_type = 'application/octet-stream'

            for i in range(3 * step + 1):

                while 1:

                    oname = unicode(uuid.uuid4())

                    if oname not in objects:

                        append(os.path.join(c, oname))

                        break

                data = get_random_data(int(random.random()))

                self.utils.create_update_object('account1', c, oname,

                                                content_type, data)

        self.assertEquals(len(objects),

                          len(self.utils.list_all_objects('account1')))

        self.assertEquals(sorted(objects),

                          self.utils.list_all_objects('account1'))

    def test_list_past_versions(self):

        self.utils.backend.put_container('account1', 'account1', 'container1')

        versions = []

        append = versions.append

        for i in range(5):

            data = get_random_data(int(random.random()))

            append(self.utils.create_update_object('account1', 'container1',

                                                   'object1',

                                                   'application/octet-stream',

                                                   data))

        self.assertEquals(sorted(versions[:-1]),

                          self.utils.list_past_versions('account1',

                                                        'container1',

                                                        'object1'))

    def test_move(self):

        # create containers

        self.utils.backend.put_container('account1', 'account1', 'container1')

        self.utils.backend.put_container('Account1', 'Account1', 'container1')

        # add group

        self.utils.backend.update_account_groups('Account1', 'Account1',

                                                 {'test': ['account3']})

        # upload object and update it several times

        versions = []

        append = versions.append

        meta = {'foo': 'bar'}

        permissions = {'read': ['account1', 'account2', 'Account1:test'],

                       'write': ['account2', 'Account1:test']}

        for i in range(5):

            data = get_random_data(int(random.random()))

            append(self.utils.create_update_object('Account1', 'container1',

                                                   'object1',

                                                   'application/octet-stream',

                                                   data, meta, permissions))

        self.utils.move_object('Account1', 'container1', 'object1', 'account1',

                               dry=False, silent=True)

        expected = {'meta': meta,

                    'versions': versions[:-1],

                    'permissions': permissions}

        self._verify_object('account1', 'container1', 'object1', expected)

    def test_merge(self):

        # create container

        self.utils.backend.put_container('Account1', 'Account1', 'container0')

        self.utils.backend.put_container('Account1', 'Account1', 'container1')

        # add group

        self.utils.backend.update_account_groups('Account1', 'Account1',

                                                 {'test': ['account3']})

        # upload objects and update them several times

        versions = defaultdict(list)

        meta = {'foo': 'bar'}

        permissions = {'read': ['account2', 'Account1:test'],

                       'write': ['account2', 'Account1:test']}

        for i in range(2):

            container = 'container%s' % i

            versions[container] = {}

            for j in range(3):

                object = 'object%s' % j

                versions[container][object] = []

                append = versions[container][object].append

                for k in range(5):

                    data = get_random_data(int(random.random()))

                    append(self.utils.create_update_object(

                        'Account1', container, object,

                        'application/octet-stream', data, meta, permissions))

        self.utils.merge_account('Account1', 'account1', only_stats=False,

                                 dry=False, silent=True)

        self.assertTrue('Account1' in self.utils.existing_accounts())

        self.assertTrue('account1' in self.utils.existing_accounts())

        # assert container has been created

        try:

            self.utils.backend.get_container_meta('account1', 'account1',

                                                  'container1', 'pithos')

        except NameError, e:

            self.fail(e)

        expected = {'meta': meta,

                    'permissions': permissions}

        for c, o_dict in versions.iteritems():

            for o, versions in o_dict.iteritems():

                expected['versions'] = versions[:-1]

                self._verify_object('account1', c, o, expected)

    def test_merge_existing_dest_container(self):

        # create container

        self.utils.backend.put_container('Account1', 'Account1', 'container1')

        self.utils.backend.put_container('account1', 'account1', 'container1')

        # add group

        self.utils.backend.update_account_groups('Account1', 'Account1',

                                                 {'test': ['account3']})

        # upload objects and update them several times

        versions = defaultdict(list)

        meta = {'foo': 'bar'}

        permissions = {'read': ['account2', 'Account1:test'],

                       'write': ['account2', 'Account1:test']}

        versions = []

        append = versions.append

        for k in range(5):

            data = get_random_data(int(random.random()))

            append(self.utils.create_update_object(

                'Account1', 'container1', 'object1',

                'application/octet-stream', data, meta, permissions))

        self.utils.merge_account('Account1', 'account1', only_stats=False,

                                 dry=False, silent=True)

        self.assertTrue('Account1' in self.utils.existing_accounts())

        self.assertTrue('account1' in self.utils.existing_accounts())

        try:

            self.utils.backend.get_container_meta('account1', 'account1',

                                                  'container1', 'pithos')

        except NameError, e:

            self.fail(e)

        expected = {'meta': meta,

                    'versions': versions[:-1],

                    'permissions': permissions}

        self._verify_object('account1', 'container1', 'object1', expected)

    def test_merge_existing_dest_object(self):

        # create container

        self.utils.backend.put_container('Account1', 'Account1', 'container1')

        self.utils.backend.put_container('account1', 'account1', 'container1')

        # add group

        self.utils.backend.update_account_groups('Account1', 'Account1',

                                                 {'test': ['account3']})

        # upload objects and update them several times

        versions = defaultdict(list)

        meta = {'foo': 'bar'}

        permissions = {'read': ['account2', 'Account1:test'],

                       'write': ['account2', 'Account1:test']}

        container = 'container1'

        object = 'object1'

        versions = []

        append = versions.append

        for k in range(5):

            data = get_random_data(int(random.random()))

            append(self.utils.create_update_object(

                   'Account1', container, object,

                   'application/octet-stream', data, meta, permissions))

            data = get_random_data(int(random.random()))

            self.utils.create_update_object(

                   'account1', container, object,

                   'application/octet-stream', data, meta, permissions)

        self.utils.merge_account('Account1', 'account1', only_stats=False,

                                 dry=False, silent=True)

        self.assertTrue('Account1' in self.utils.existing_accounts())

        self.assertTrue('account1' in self.utils.existing_accounts())

        try:

            self.utils.backend.get_container_meta('account1', 'account1',

                                                  'container1', 'pithos')

        except NameError, e:

            self.fail(e)

        expected = {'meta': meta,

                    'permissions': permissions,

                    'versions': versions[:-1]}

        self._verify_object('account1', container, object, expected,

                            strict=False)

if __name__ == '__main__':

    unittest.main()