Synnefo

# Copyright 2013 GRNET S.A. All rights reserved.

#

# Redistribution and use in source and binary forms, with or

# without modification, are permitted provided that the following

# conditions are met:

#

#   1. Redistributions of source code must retain the above

#      copyright notice, this list of conditions and the following

#      disclaimer.

#

#   2. Redistributions in binary form must reproduce the above

#      copyright notice, this list of conditions and the following

#      disclaimer in the documentation and/or other materials

#      provided with the distribution.

#

# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS

# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED

# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR

# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR

# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,

# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT

# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF

# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED

# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT

# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN

# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE

# POSSIBILITY OF SUCH DAMAGE.

#

# The views and conclusions contained in the software and

# documentation are those of the authors and should not be

# interpreted as representing official policies, either expressed

# or implied, of GRNET S.A.

"""

This is the burnin class that tests the Pithos functionality

"""

import os

import random

import tempfile

from datetime import datetime

from synnefo_tools.burnin.common import BurninTests, Proper

from kamaki.clients import ClientError

# Too many public methods. pylint: disable-msg=R0904

class PithosTestSuite(BurninTests):

    """Test Pithos functionality"""

    containers = Proper(value=None)

    created_container = Proper(value=None)

    now_unformated = Proper(value=datetime.utcnow())

    obj_metakey = Proper(value=None)

    def test_005_account_head(self):

        """HEAD on pithos account"""

        self._set_pithos_account(self._get_uuid())

        pithos = self.clients.pithos

        r = pithos.account_head()

        self.assertEqual(r.status_code, 204)

        self.info('Returns 204')

        r = pithos.account_head(until='1000000000')

        self.assertEqual(r.status_code, 204)

        datestring = unicode(r.headers['x-account-until-timestamp'])

        self.assertEqual(u'Sun, 09 Sep 2001 01:46:40 GMT', datestring)

        self.assertTrue('x-account-policy-quota' in r.headers)

        self.info('Until and policy quota exist')

        for format in pithos.DATE_FORMATS:

            now_formated = self.now_unformated.strftime(format)

            r1 = pithos.account_head(

                if_modified_since=now_formated, success=(204, 304, 412))

            r2 = pithos.account_head(

                if_unmodified_since=now_formated, success=(204, 304, 412))

            self.assertNotEqual(r1.status_code, r2.status_code)

        self.info('If_(un)modified_since is OK')

    def test_010_account_get(self):

        """Test account_get"""

        self.info('Preparation')

        pithos = self.clients.pithos

        for i in range(1, 3):

            cont_name = "cont%s_%s%s" % (

                i, self.run_id or 0, random.randint(1000, 9999))

            self._create_pithos_container(cont_name)

        pithos.container, obj = cont_name, 'shared_file'

        pithos.create_object(obj)

        pithos.set_object_sharing(obj, read_permission='*')

        self.info('Created object /%s/%s' % (cont_name, obj))

        r = pithos.list_containers()

        fullLen = len(r)

        self.assertTrue(fullLen > 2)

        self.info('Normal use is OK')

        r = pithos.account_get(limit=1)

        self.assertEqual(len(r.json), 1)

        self.info('Limit works')

        r = pithos.account_get(marker='cont')

        cont1, cont3 = r.json[0], r.json[2]

        self.info('Marker works')

        r = pithos.account_get(limit=2, marker='cont')

        conames = [container['name'] for container in r.json if (

            container['name'].lower().startswith('cont'))]

        self.assertTrue(cont1['name'] in conames)

        self.assertFalse(cont3['name'] in conames)

        self.info('Marker-limit combination works')

        r = pithos.account_get(show_only_shared=True)

        self.assertTrue(cont_name in [c['name'] for c in r.json])

        self.info('Show-only-shared works')

        r = pithos.account_get(until=1342609206.0)

        self.assertTrue(len(r.json) <= fullLen)

        self.info('Until works')

        for format in pithos.DATE_FORMATS:

            now_formated = self.now_unformated.strftime(format)

            r1 = pithos.account_get(

                if_modified_since=now_formated, success=(200, 304, 412))

            r2 = pithos.account_get(

                if_unmodified_since=now_formated, success=(200, 304, 412))

            self.assertNotEqual(r1.status_code, r2.status_code)

        self.info('If_(un)modified_since is OK')

    def test_015_account_post(self):

        """Test account_post"""

        pithos = self.clients.pithos

        r = pithos.account_post()

        self.assertEqual(r.status_code, 202)

        self.info('Status code is OK')

        rand_num = '%s%s' % (self.run_id or 0, random.randint(1000, 9999))

        grpName = 'grp%s' % rand_num

        u1, u2 = pithos.account, 'invalid-user-uuid-%s' % rand_num

        self.assertRaises(

            ClientError, pithos.set_account_group, grpName, [u1, u2])

        self.info('Invalid uuid is handled correctly')

        pithos.set_account_group(grpName, [u1])

        r = pithos.get_account_group()

        self.assertEqual(r['x-account-group-' + grpName], '%s' % u1)

        self.info('Account group is OK')

        pithos.del_account_group(grpName)

        r = pithos.get_account_group()

        self.assertTrue('x-account-group-' + grpName not in r)

        self.info('Removed account group')

        mprefix = 'meta%s' % rand_num

        pithos.set_account_meta({

            mprefix + '1': 'v1', mprefix + '2': 'v2'})

        r = pithos.get_account_meta()

        self.assertEqual(r['x-account-meta-' + mprefix + '1'], 'v1')

        self.assertEqual(r['x-account-meta-' + mprefix + '2'], 'v2')

        self.info('Account meta is OK')

        pithos.del_account_meta(mprefix + '1')

        r = pithos.get_account_meta()

        self.assertTrue('x-account-meta-' + mprefix + '1' not in r)

        self.assertTrue('x-account-meta-' + mprefix + '2' in r)

        self.info('Selective removal of account meta is OK')

        pithos.del_account_meta(mprefix + '2')

        r = pithos.get_account_meta()

        self.assertTrue('x-account-meta-' + mprefix + '2' not in r)

        self.info('Temporary account meta are removed')

    def test_020_container_head(self):

        """Test container HEAD"""

        pithos = self.clients.pithos

        r = pithos.container_head()

        self.assertEqual(r.status_code, 204)

        self.info('Status code is OK')

        r = pithos.container_head(until=1000000, success=(204, 404))

        self.assertEqual(r.status_code, 404)

        self.info('Until works')

        for format in pithos.DATE_FORMATS:

            now_formated = self.now_unformated.strftime(format)

            r1 = pithos.container_head(

                if_modified_since=now_formated, success=(204, 304, 412))

            r2 = pithos.container_head(

                if_unmodified_since=now_formated, success=(204, 304, 412))

            self.assertNotEqual(r1.status_code, r2.status_code)

        k = 'metakey%s' % random.randint(1000, 9999)

        pithos.set_container_meta({k: 'our value'})

        r = pithos.get_container_meta()

        k = 'x-container-meta-%s' % k

        self.assertIn(k, r)

        self.assertEqual('our value', r[k])

        self.info('Container meta exists')

        self.obj_metakey = 'metakey%s' % random.randint(1000, 9999)

        obj = 'object_with_meta'

        pithos.create_object(obj)

        pithos.set_object_meta(obj, {self.obj_metakey: 'our value'})

        r = pithos.get_container_object_meta()

        self.assertIn('x-container-object-meta', r)

        self.assertIn(

            self.obj_metakey, r['x-container-object-meta'].lower())

        self.info('Container object meta exists')

    def test_025_container_get(self):

        """Test container GET"""

        pithos = self.clients.pithos

        r = pithos.container_get()

        self.assertEqual(r.status_code, 200)

        self.info('Status code is OK')

        fullLen = len(r.json)

        obj1 = 'test%s' % random.randint(1000, 9999)

        pithos.create_object(obj1)

        obj2 = 'test%s' % random.randint(1000, 9999)

        pithos.create_object(obj2)

        obj3 = 'another%s.test' % random.randint(1000, 9999)

        pithos.create_object(obj3)

        r = pithos.container_get(prefix='test')

        self.assertTrue(len(r.json) > 1)

        test_objects = [o for o in r.json if o['name'].startswith('test')]

        self.assertEqual(len(r.json), len(test_objects))

        self.info('Prefix is OK')

        r = pithos.container_get(limit=1)

        self.assertEqual(len(r.json), 1)

        self.info('Limit is OK')

        r = pithos.container_get(marker=obj3[:-5])

        self.assertTrue(len(r.json) > 1)

        aoobjects = [obj for obj in r.json if obj['name'] > obj3[:-5]]

        self.assertEqual(len(r.json), len(aoobjects))

        self.info('Marker is OK')

        r = pithos.container_get(prefix=obj3, delimiter='.')

        self.assertTrue(fullLen > len(r.json))

        self.info('Delimiter is OK')

        r = pithos.container_get(path='/')

        fullLen += 3

        self.assertEqual(fullLen, len(r.json))

        self.info('Path is OK')

        r = pithos.container_get(format='xml')

        self.assertEqual(r.text.split()[4], 'name="' + pithos.container + '">')

        self.info('Format is OK')

        r = pithos.container_get(meta=[self.obj_metakey, ])

        self.assertTrue(len(r.json) > 0)

        self.info('Meta is OK')

        r = pithos.container_get(show_only_shared=True)

        self.assertTrue(len(r.json) < fullLen)

        self.info('Show-only-shared is OK')

        try:

            r = pithos.container_get(until=1000000000)

            datestring = unicode(r.headers['x-account-until-timestamp'])

            self.assertEqual(u'Sun, 09 Sep 2001 01:46:40 GMT', datestring)

            self.info('Until is OK')

        except ClientError:

            pass

    def test_030_container_put(self):

        """Test container PUT"""

        pithos = self.clients.pithos

        pithos.container = 'cont%s%s' % (

            self.run_id or 0, random.randint(1000, 9999))

        self.temp_containers.append(pithos.container)

        r = pithos.create_container()

        self.assertTrue(isinstance(r, dict))

        r = pithos.get_container_limit(pithos.container)

        cquota = r.values()[0]

        newquota = 2 * int(cquota)

        self.info('Limit is OK')

        pithos.del_container()

        r = pithos.create_container(sizelimit=newquota)

        self.assertTrue(isinstance(r, dict))

        r = pithos.get_container_limit(pithos.container)

        xquota = int(r.values()[0])

        self.assertEqual(newquota, xquota)

        self.info('Can set container limit')

        pithos.del_container()

        r = pithos.create_container(versioning='auto')

        self.assertTrue(isinstance(r, dict))

        r = pithos.get_container_versioning(pithos.container)

        nvers = r.values()[0]

        self.assertEqual('auto', nvers)

        self.info('Versioning=auto is OK')

        pithos.del_container()

        r = pithos.container_put(versioning='none')

        self.assertEqual(r.status_code, 201)

        r = pithos.get_container_versioning(pithos.container)

        nvers = r.values()[0]

        self.assertEqual('none', nvers)

        self.info('Versioning=none is OK')

        pithos.del_container()

        r = pithos.create_container(metadata={'m1': 'v1', 'm2': 'v2'})

        self.assertTrue(isinstance(r, dict))

        r = pithos.get_container_meta(pithos.container)

        self.assertTrue('x-container-meta-m1' in r)

        self.assertEqual(r['x-container-meta-m1'], 'v1')

        self.assertTrue('x-container-meta-m2' in r)

        self.assertEqual(r['x-container-meta-m2'], 'v2')

        r = pithos.container_put(metadata={'m1': '', 'm2': 'v2a'})

        self.assertEqual(r.status_code, 202)

        r = pithos.get_container_meta(pithos.container)

        self.assertTrue('x-container-meta-m1' not in r)

        self.assertTrue('x-container-meta-m2' in r)

        self.assertEqual(r['x-container-meta-m2'], 'v2a')

        self.info('Container meta is OK')

        pithos.del_container_meta(pithos.container)

    def test_035_container_post(self):

        """Test container POST"""

        pithos = self.clients.pithos

        r = pithos.container_post()

        self.assertEqual(r.status_code, 202)

        self.info('Status is OK')

        pithos.set_container_meta({'m1': 'v1', 'm2': 'v2'})

        r = pithos.get_container_meta(pithos.container)

        self.assertTrue('x-container-meta-m1' in r)

        self.assertEqual(r['x-container-meta-m1'], 'v1')

        self.assertTrue('x-container-meta-m2' in r)

        self.assertEqual(r['x-container-meta-m2'], 'v2')

        self.info('Set metadata works')

        r = pithos.del_container_meta('m1')

        r = pithos.set_container_meta({'m2': 'v2a'})

        r = pithos.get_container_meta(pithos.container)

        self.assertTrue('x-container-meta-m1' not in r)

        self.assertTrue('x-container-meta-m2' in r)

        self.assertEqual(r['x-container-meta-m2'], 'v2a')

        self.info('Delete metadata works')

        r = pithos.get_container_limit(pithos.container)

        cquota = r.values()[0]

        newquota = 2 * int(cquota)

        r = pithos.set_container_limit(newquota)

        r = pithos.get_container_limit(pithos.container)

        xquota = int(r.values()[0])

        self.assertEqual(newquota, xquota)

        self.info('Set quota works')

        pithos.set_container_versioning('auto')

        r = pithos.get_container_versioning(pithos.container)

        nvers = r.values()[0]

        self.assertEqual('auto', nvers)

        pithos.set_container_versioning('none')

        r = pithos.get_container_versioning(pithos.container)

        nvers = r.values()[0]

        self.assertEqual('none', nvers)

        self.info('Set versioning works')

        f = self._create_large_file(1024 * 1024 * 100)

        self.info('Created file %s of 100 MB' % f.name)

        pithos.create_directory('dir')

        r = pithos.upload_object('/dir/sample.file', f)

        for term in ('content-length', 'content-type', 'x-object-version'):

            self.assertTrue(term in r)

        r = pithos.get_object_info('/dir/sample.file')

        self.assertTrue(int(r['content-length']) > 100000000)

        self.info('Made remote directory /dir and object /dir/sample.file')

        """What is tranfer_encoding? What should I check about it? """

        #TODO

        obj = 'object_with_meta'

        pithos.container = self.temp_containers[-2]

        r = pithos.object_post(

            obj, update='False', metadata={'newmeta': 'newval'})

        r = pithos.get_object_info(obj)

        self.assertTrue('x-object-meta-newmeta' in r)

        self.assertFalse('x-object-meta-%s' % self.obj_metakey not in r)

        self.info('Metadata with update=False works')

    def test_050_stop_test(self):

        """STOP TESTING ALREADY"""

        self.assertTrue(False)

    def test_051_list_containers(self):

        """Test container list actually returns containers"""

        self.containers = self._get_list_of_containers()

        self.assertGreater(len(self.containers), 0)

        self.lala = 1

    def test_052_unique_containers(self):

        """Test if containers have unique names"""

        names = [n['name'] for n in self.containers]

        names = sorted(names)

        self.assertEqual(sorted(list(set(names))), names)

    def test_054_upload_file(self):

        """Test uploading a txt file to Pithos"""

        # Create a tmp file

        with tempfile.TemporaryFile(dir=self.temp_directory) as fout:

            fout.write("This is a temp file")

            fout.seek(0, 0)

            # Upload the file,

            # The container is the one choosen during the `create_container'

            self.clients.pithos.upload_object("test.txt", fout)

            # Verify quotas

            self._check_quotas(diskspace=+os.fstat(fout.fileno()).st_size)

    def test_055_download_file(self):

        """Test downloading the file from Pithos"""

        # Create a tmp directory to save the file

        with tempfile.TemporaryFile(dir=self.temp_directory) as fout:

            self.clients.pithos.download_object("test.txt", fout)

            # Now read the file

            fout.seek(0, 0)

            contents = fout.read()

            # Compare results

            self.info("Comparing contents with the uploaded file")

            self.assertEqual(contents, "This is a temp file")

    def test_056_remove(self):

        """Test removing files and containers from Pithos"""

        self.info("Removing the file %s from container %s",

                  "test.txt", self.created_container)

        # The container is the one choosen during the `create_container'

        content_length = \

            self.clients.pithos.get_object_info("test.txt")['content-length']

        self.clients.pithos.del_object("test.txt")

        # Verify quotas

        self._check_quotas(diskspace=-int(content_length))

        self.info("Removing the container %s", self.created_container)

        self.clients.pithos.purge_container()

        # List containers

        containers = self._get_list_of_containers()

        self.info("Check that the container %s has been deleted",

                  self.created_container)

        names = [n['name'] for n in containers]

        self.assertNotIn(self.created_container, names)

        # We successfully deleted our container, no need to do it

        # in our clean up phase

        self.created_container = None

    @classmethod

    def tearDownClass(cls):  # noqa

        """Clean up"""

        from kamaki.cli.logger import deactivate

        deactivate('kamaki.clients.send')

        deactivate('kamaki.clients.recv')

        pithos = cls.clients.pithos

        for c in getattr(cls, 'temp_containers', []):

            pithos.container = c

            try:

                pithos.del_container(delimiter='/')

                pithos.purge_container(c)

            except ClientError as ce:

                print ('Failed to destroy container (%s)' % ce)