Revision 0a92ff85

b/contrib/snf-pithos-tools/pithos/tools/test.py
2385 2385
        self.assert_not_raises_fault(413, self.upload_random_data, 'c', 'o',
2386 2386
                                 length=1024*1024+1)
2387 2387

  
2388
class TestUsageFreeVersioningAutoContainerPolicy(BaseTestCase):
2389
    """ Challenge free version accounting
2390
       in a container with auto versioning policy
2391

  
2392
       In case of unknown server version accounting policy or
2393
       debit version accounting policy
2394
       enforce all test cases to return immediately
2395
    """
2396
    def setUp(self):
2397
        BaseTestCase.setUp(self)
2398

  
2399
        # TODO Only works if tests are running in the same host with the server. Add support for remote server
2400
        self.stop_execution = False
2401
        try:
2402
            from pithos.api.settings import BACKEND_FREE_VERSIONING
2403
        except ImportError:
2404
            print 'Unable to execute the test: unknown version accounting policy'
2405
            self.stop_execution = True
2406
            return
2407
        else:
2408
            if not BACKEND_FREE_VERSIONING:
2409
                print 'Unable to execute the test: no free version accounting policy'
2410
                self.stop_execution = True
2411
                return
2412

  
2413
        self.create_container('container')
2414
        meta = self.client.retrieve_account_metadata()
2415
        self.initial_usage = int(meta['x-account-bytes-used'])
2416
        print 'Current account usage: %d' % self.initial_usage
2417

  
2418
        self.usage = self.initial_usage
2419
        #self.mtime = {}
2420
        l =  (50, 100, 150, 70, 80)
2421
        self.curr_acc_usage = self.initial_usage
2422
        for i, length in list(enumerate(l)):
2423
            self.upload_random_data('container', 'object', length=length)
2424
            meta = self.client.retrieve_account_metadata()
2425
            self.usage = int(meta['x-account-bytes-used'])
2426
            print 'Current account usage: %d' % self.usage
2427
            self.curr_acc_usage = self.initial_usage + length
2428
            self.assertEqual(self.usage, self.curr_acc_usage)
2429

  
2430
            #t = datetime.datetime.utcnow()
2431
            #self.mtime[i] = int(_time.mktime(t.timetuple()))
2432
            _time.sleep(1)
2433

  
2434
        versions = self.client.retrieve_object_versionlist(
2435
            'container', 'object'
2436
        )['versions']
2437
        self.mtime = [int(i[1]) for i in versions]
2438

  
2439
    def create_container(self, cname):
2440
        self.client.create_container(cname)
2441

  
2442
    def test_delete_object_container(self):
2443
        """
2444
            assume account usage = 0
2445
            scenario:
2446
                upload object (length=50)
2447
                update object (length=100)
2448
                update object (length=150)
2449
                update object (length=70)
2450
                update object (length=80)
2451
                delete object
2452
                delete container
2453
            account usage sequel: 50|100|150|70|80|0|0
2454
        """
2455
        if self.stop_execution:
2456
            return
2457

  
2458
        self.client.delete_object('container', 'object')
2459
        meta = self.client.retrieve_account_metadata()
2460
        self.usage = int(meta['x-account-bytes-used'])
2461
        print 'Current account usage: %d' % self.usage
2462
        self.assertEqual(self.usage, self.initial_usage)
2463

  
2464
        self.client.delete_container('container')
2465
        meta = self.client.retrieve_account_metadata()
2466
        self.usage = int(meta['x-account-bytes-used'])
2467
        print 'Current account usage: %d' % self.usage
2468
        self.assertEqual(self.usage, self.initial_usage)
2469

  
2470
    def test_purge_delete_object(self):
2471
        """
2472
            assume account usage = 0
2473
            scenario:
2474
                upload object (length=50)
2475
                update object (length=100)
2476
                update object (length=150)
2477
                update object (length=70)
2478
                update object (length=80)
2479
                delete object history
2480
                delete object
2481
                delete container
2482
            account usage sequel: 50|100|150|70|80|80|0|0
2483
        """
2484
        if self.stop_execution:
2485
            return
2486

  
2487
        # purge some object history
2488
        i = random.randrange(len(self.mtime))
2489
        self.client.delete_object('container', 'object', until=self.mtime[i])
2490
        meta = self.client.retrieve_account_metadata()
2491
        self.usage = int(meta['x-account-bytes-used'])
2492
        print 'Current account usage: %d' % self.usage
2493
        self.assertEqual(self.usage, self.curr_acc_usage)
2494

  
2495
        self.client.delete_object('container', 'object')
2496
        meta = self.client.retrieve_account_metadata()
2497
        self.usage = int(meta['x-account-bytes-used'])
2498
        print 'Current account usage: %d' % self.usage
2499
        self.assertEqual(self.usage, self.initial_usage)
2500

  
2501
        self.client.delete_container('container')
2502
        meta = self.client.retrieve_account_metadata()
2503
        self.usage = int(meta['x-account-bytes-used'])
2504
        print 'Current account usage: %d' % self.usage
2505
        self.assertEqual(self.usage, self.initial_usage)
2506

  
2507
    def test_delete_object_purge_container_history(self):
2508
        """
2509
            assume account usage = 0
2510
            scenario:
2511
                upload object (length=50)
2512
                update object (length=100)
2513
                update object (length=150)
2514
                update object (length=70)
2515
                update object (length=80)
2516
                delete object
2517
                delete container history
2518
                delete container
2519
            account usage sequel: 50|100|150|70|80|0|0|0
2520
        """
2521
        if self.stop_execution:
2522
            return
2523

  
2524
        self.client.delete_object('container', 'object')
2525
        meta = self.client.retrieve_account_metadata()
2526
        self.usage = int(meta['x-account-bytes-used'])
2527
        print 'Current account usage: %d' % self.usage
2528
        self.assertEqual(self.usage,
2529
                         self.initial_usage)
2530

  
2531
        # purge some container history
2532
        i = random.randrange(len(self.mtime))
2533
        self.client.delete_container('container', until=self.mtime[i])
2534
        meta = self.client.retrieve_account_metadata()
2535
        self.usage = int(meta['x-account-bytes-used'])
2536
        print 'Current account usage: %d' % self.usage
2537
        self.assertEqual(self.usage, self.initial_usage)
2538

  
2539
        self.client.delete_container('container')
2540
        meta = self.client.retrieve_account_metadata()
2541
        self.usage = int(meta['x-account-bytes-used'])
2542
        print 'Current account usage: %d' % self.usage
2543
        self.assertEqual(self.usage, self.initial_usage)
2544

  
2545
    def test_purge_container_delete_object(self):
2546
        """
2547
            assume account usage = 0
2548
            scenario:
2549
                upload object (length=50)
2550
                update object (length=100)
2551
                update object (length=150)
2552
                update object (length=70)
2553
                update object (length=80)
2554
                delete container history
2555
                delete object
2556
                delete container
2557
            account usage sequel: 50|100|150|70|80|80|0|0
2558
        """
2559
        if self.stop_execution:
2560
            return
2561

  
2562
        # purge some container history
2563
        i = random.randrange(len(self.mtime))
2564
        self.client.delete_container('container', until=self.mtime[i])
2565
        meta = self.client.retrieve_account_metadata()
2566
        self.usage = int(meta['x-account-bytes-used'])
2567
        print 'Current account usage: %d' % self.usage
2568
        self.assertEqual( self.usage, self.curr_acc_usage)
2569

  
2570
        self.client.delete_object('container', 'object')
2571
        meta = self.client.retrieve_account_metadata()
2572
        self.usage = int(meta['x-account-bytes-used'])
2573
        print 'Current account usage: %d' % self.usage
2574
        self.assertEqual(self.usage, self.initial_usage)
2575

  
2576
        self.client.delete_container('container')
2577
        meta = self.client.retrieve_account_metadata()
2578
        self.usage = int(meta['x-account-bytes-used'])
2579
        print 'Current account usage: %d' % self.usage
2580
        self.assertEqual(self.usage, self.initial_usage)
2581

  
2582
    def test_successive_purging(self):
2583
        """
2584
            assume account usage = 0
2585
            scenario:
2586
                upload object (length=50)
2587
                update object (length=100)
2588
                update object (length=150)
2589
                update object (length=70)
2590
                update object (length=80)
2591
                delete earlier 2 object history versions
2592
                delete next earlier object history version
2593
                delete rest object history and current version
2594
                delete object (404 status)
2595
                delete container
2596
            account usage sequel: 50|100|150|70|80|80|80|0|0|0
2597
        """
2598
        if self.stop_execution:
2599
            return
2600

  
2601
        # purge some object history
2602
        i = random.randrange(len(self.mtime))
2603
        self.client.delete_object('container', 'object', until=self.mtime[i])
2604
        meta = self.client.retrieve_account_metadata()
2605
        self.usage = int(meta['x-account-bytes-used'])
2606
        print 'Current account usage: %d' % self.usage
2607
        self.assertEqual(self.usage, self.curr_acc_usage)
2608

  
2609
        # purge some container history
2610
        i = random.randrange(len(self.mtime))
2611
        self.client.delete_container('container', until=self.mtime[i])
2612
        meta = self.client.retrieve_account_metadata()
2613
        self.usage = int(meta['x-account-bytes-used'])
2614
        print 'Current account usage: %d' % self.usage
2615
        self.assertEqual(self.usage, self.curr_acc_usage)
2616

  
2617
        # purge some more object history
2618
        i = random.randrange(len(self.mtime))
2619
        self.client.delete_object('container', 'object', until=self.mtime[i])
2620
        meta = self.client.retrieve_account_metadata()
2621
        self.usage = int(meta['x-account-bytes-used'])
2622
        print 'Current account usage: %d' % self.usage
2623
        self.assertEqual(self.usage, self.curr_acc_usage)
2624

  
2625
        # purge some object history and current
2626
        t = datetime.datetime.utcnow()
2627
        now = int(_time.mktime(t.timetuple()))
2628
        self.client.delete_object('container', 'object', until=now)
2629
        meta = self.client.retrieve_account_metadata()
2630
        self.usage = int(meta['x-account-bytes-used'])
2631
        print 'Current account usage: %d' % self.usage
2632
        self.assertEqual(self.usage, self.initial_usage)
2633

  
2634
        # try to delete object and assert object is not found
2635
        self.assert_raises_fault(
2636
            404, self.client.delete_object, 'container', 'object'
2637
        )
2638

  
2639
        meta = self.client.retrieve_account_metadata()
2640
        self.usage = int(meta['x-account-bytes-used'])
2641
        print 'Current account usage: %d' % self.usage
2642
        self.assertEqual(self.usage, self.initial_usage)
2643

  
2644
        self.client.delete_container('container')
2645
        meta = self.client.retrieve_account_metadata()
2646
        self.usage = int(meta['x-account-bytes-used'])
2647
        print 'Current account usage: %d' % self.usage
2648
        self.assertEqual(self.usage, self.initial_usage)
2649

  
2650
    def test_delete_object_empty_container_content(self):
2651
        """
2652
            assume account usage = 0
2653
            scenario:
2654
                upload object (length=50)
2655
                update object (length=100)
2656
                update object (length=150)
2657
                update object (length=70)
2658
                update object (length=80)
2659
                delete object
2660
                delete container contents
2661
                delete container
2662
            account usage sequel: 50|100|150|70|80|0|0|0
2663
        """
2664
        if self.stop_execution:
2665
            return
2666

  
2667
        self.client.delete_object('container', 'object')
2668

  
2669
        meta = self.client.retrieve_account_metadata()
2670
        self.usage = int(meta['x-account-bytes-used'])
2671
        print 'Current account usage: %d' % self.usage
2672
        self.assertEqual(self.usage, self.initial_usage)
2673

  
2674
        self.client.delete_container('container', delimiter='/')
2675
        meta = self.client.retrieve_account_metadata()
2676
        self.usage = int(meta['x-account-bytes-used'])
2677
        print 'Current account usage: %d' % self.usage
2678
        self.assertEqual(self.usage, self.initial_usage)
2679

  
2680
        self.client.delete_container('container')
2681
        meta = self.client.retrieve_account_metadata()
2682
        self.usage = int(meta['x-account-bytes-used'])
2683
        print 'Current account usage: %d' % self.usage
2684
        self.assertEqual(self.usage, self.initial_usage)
2685

  
2686
    def test_delete_container_content(self):
2687
        """
2688
            assume account usage = 0
2689
            scenario:
2690
                upload object (length=50)
2691
                update object (length=100)
2692
                update object (length=150)
2693
                update object (length=70)
2694
                update object (length=80)
2695
                delete container contents
2696
                delete container
2697
            account usage sequel: 50|100|150|70|80|0|0
2698
        """
2699
        if self.stop_execution:
2700
            return
2701

  
2702
        self.client.delete_container('container', delimiter='/')
2703
        meta = self.client.retrieve_account_metadata()
2704
        self.usage = int(meta['x-account-bytes-used'])
2705
        print 'Current account usage: %d' % self.usage
2706
        self.assertEqual(self.usage, self.initial_usage)
2707

  
2708
        self.client.delete_container('container')
2709
        meta = self.client.retrieve_account_metadata()
2710
        self.usage = int(meta['x-account-bytes-used'])
2711
        print 'Current account usage: %d' % self.usage
2712
        self.assertEqual(self.usage, self.initial_usage)
2713

  
2714

  
2715
class TestUsageFreeVersioningNoneContainerPolicy(
2716
    TestUsageFreeVersioningAutoContainerPolicy):
2717
    """ Challenge free version accounting
2718
       in a container with none versioning policy
2719

  
2720
       In case of unknown server version accounting policy or
2721
       debit version accounting policy
2722
       enforce all test cases to return immediately
2723
    """
2724
    def create_container(self, cname):
2725
        self.client.create_container(cname,
2726
                                     policies={'versioning':'none'})
2727

  
2728
class TestUsageDebitVersioningAutoContainerPolicy(BaseTestCase):
2729
    """ Challenge debit version accounting
2730
       in a container with auto versioning policy
2731

  
2732
       In case of unknown server version accounting policy or
2733
       free version accounting policy
2734
       enforce all test cases to return immediately
2735
    """
2736
    def setUp(self):
2737
        BaseTestCase.setUp(self)
2738

  
2739
        self.stop_execution = False
2740
        try:
2741
            from pithos.api.settings import BACKEND_FREE_VERSIONING
2742
        except ImportError:
2743
            print 'Unable to execute the test: unknown version accounting policy'
2744
            self.stop_execution = True
2745
            return
2746
        else:
2747
            if BACKEND_FREE_VERSIONING:
2748
                print 'Unable to execute the test: free version accounting policy'
2749
                self.stop_execution = True
2750
                return
2751

  
2752
        self.create_container('container')
2753
        meta = self.client.retrieve_account_metadata()
2754
        self.initial_usage = int(meta['x-account-bytes-used'])
2755
        print 'Current account usage: %d' % self.initial_usage
2756

  
2757
        self.usage = self.initial_usage
2758
        self.l = (50, 100, 150, 70, 80)
2759
        self.curr_acc_usage = self.initial_usage
2760
        for i, length in list(enumerate(self.l)):
2761
            self.upload_random_data('container', 'object', length=length)
2762
            meta = self.client.retrieve_account_metadata()
2763
            self.usage = int(meta['x-account-bytes-used'])
2764
            print 'Current account usage: %d' % self.usage
2765
            self.curr_acc_usage += length
2766
            self.assertEqual(self.usage, self.curr_acc_usage)
2767
            _time.sleep(1)
2768

  
2769
        versions = self.client.retrieve_object_versionlist(
2770
            'container', 'object'
2771
        )['versions']
2772
        self.mtime = [int(i[1]) for i in versions]
2773

  
2774
    def create_container(self, cname):
2775
        self.client.create_container(cname)
2776

  
2777
    def test_delete_object_container(self):
2778
        """
2779
            assume account usage = 0
2780
            scenario:
2781
                upload object (length=50)
2782
                update object (length=100)
2783
                update object (length=150)
2784
                update object (length=70)
2785
                update object (length=80)
2786
                delete object
2787
                delete container
2788
            account usage sequel: 50|150|300|370|450|450|0
2789
        """
2790
        if self.stop_execution:
2791
            return
2792

  
2793
        self.client.delete_object('container', 'object')
2794
        meta = self.client.retrieve_account_metadata()
2795
        self.usage = int(meta['x-account-bytes-used'])
2796
        print 'Current account usage: %d' % self.usage
2797
        self.assertEqual(self.usage, self.initial_usage + sum(self.l))
2798

  
2799
        self.client.delete_container('container')
2800
        meta = self.client.retrieve_account_metadata()
2801
        self.usage = int(meta['x-account-bytes-used'])
2802
        print 'Current account usage: %d' % self.usage
2803
        self.assertEqual(self.usage, self.initial_usage)
2804

  
2805
    def test_purge_delete_object(self):
2806
        """
2807
            assume account usage = 0
2808
            scenario:
2809
                upload object (length=50)
2810
                update object (length=100)
2811
                update object (length=150)
2812
                update object (length=70)
2813
                update object (length=80)
2814
                delete 3 earlier object versions
2815
                delete object
2816
                delete container
2817
            account usage sequel: 50|150|300|370|450|150|150|0
2818
        """
2819
        if self.stop_execution:
2820
            return
2821

  
2822
        # purge some object history
2823
        i = 3
2824
        self.client.delete_object('container', 'object', until=self.mtime[i])
2825
        meta = self.client.retrieve_account_metadata()
2826
        self.usage = int(meta['x-account-bytes-used'])
2827
        print 'Current account usage: %d' % self.usage
2828
        self.assertEqual(self.usage, self.curr_acc_usage - sum(self.l[:i]))
2829

  
2830
        self.client.delete_object('container', 'object')
2831
        meta = self.client.retrieve_account_metadata()
2832
        self.usage = int(meta['x-account-bytes-used'])
2833
        print 'Current account usage: %d' % self.usage
2834
        self.assertEqual(self.usage, self.curr_acc_usage - sum(self.l[:i]))
2835

  
2836
        self.client.delete_container('container')
2837
        meta = self.client.retrieve_account_metadata()
2838
        self.usage = int(meta['x-account-bytes-used'])
2839
        print 'Current account usage: %d' % self.usage
2840
        self.assertEqual(self.usage, self.initial_usage)
2841

  
2842
    def test_delete_object_purge_container_history(self):
2843
        """
2844
            assume account usage = 0
2845
            scenario:
2846
                upload object (length=50)
2847
                update object (length=100)
2848
                update object (length=150)
2849
                update object (length=70)
2850
                update object (length=80)
2851
                delete object
2852
                delete container history (3 earlier object versions)
2853
                delete container
2854
            account usage sequel: 50|150|300|370|450|450|150|0
2855
        """
2856
        if self.stop_execution:
2857
            return
2858

  
2859
        self.client.delete_object('container', 'object')
2860
        meta = self.client.retrieve_account_metadata()
2861
        self.usage = int(meta['x-account-bytes-used'])
2862
        print 'Current account usage: %d' % self.usage
2863
        self.assertEqual(self.usage, self.curr_acc_usage)
2864

  
2865
        # purge some container history
2866
        i = 3
2867
        self.client.delete_container('container', until=self.mtime[i])
2868
        meta = self.client.retrieve_account_metadata()
2869
        self.usage = int(meta['x-account-bytes-used'])
2870
        print 'Current account usage: %d' % self.usage
2871
        self.assertEqual(self.usage, self.curr_acc_usage - sum(self.l[:i]))
2872

  
2873
        self.client.delete_container('container')
2874
        meta = self.client.retrieve_account_metadata()
2875
        self.usage = int(meta['x-account-bytes-used'])
2876
        print 'Current account usage: %d' % self.usage
2877
        self.assertEqual(self.usage, self.initial_usage)
2878

  
2879
    def test_purge_container_delete_object(self):
2880
        """
2881
            assume account usage = 0
2882
            scenario:
2883
                upload object (length=50)
2884
                update object (length=100)
2885
                update object (length=150)
2886
                update object (length=70)
2887
                update object (length=80)
2888
                delete container history
2889
                delete object
2890
                delete container
2891
            account usage sequel: 50|150|300|370|450|150|150|0
2892
        """
2893
        if self.stop_execution:
2894
            return
2895

  
2896
        versions = self.client.retrieve_object_versionlist(
2897
            'container', 'object'
2898
        )['versions']
2899
        mtime = [i[1] for i in versions]
2900

  
2901
        # purge some container history
2902
        i = 3
2903
        until = int(mtime[i])
2904
        self.client.delete_container('container', until=until)
2905
        meta = self.client.retrieve_account_metadata()
2906
        self.usage = int(meta['x-account-bytes-used'])
2907
        print 'Current account usage: %d' % self.usage
2908
        self.assertEqual(self.usage, self.curr_acc_usage - sum(self.l[:i]))
2909

  
2910
        self.client.delete_object('container', 'object')
2911
        meta = self.client.retrieve_account_metadata()
2912
        self.usage = int(meta['x-account-bytes-used'])
2913
        print 'Current account usage: %d' % self.usage
2914
        self.assertEqual(self.usage, self.curr_acc_usage - sum(self.l[:i]))
2915

  
2916

  
2917
        self.client.delete_container('container')
2918
        meta = self.client.retrieve_account_metadata()
2919
        self.usage = int(meta['x-account-bytes-used'])
2920
        print 'Current account usage: %d' % self.usage
2921
        self.assertEqual(self.usage, self.initial_usage)
2922

  
2923
    def test_successive_purging(self):
2924
        """
2925
            assume account usage = 0
2926
            scenario:
2927
                upload object (length=50)
2928
                update object (length=100)
2929
                update object (length=150)
2930
                update object (length=70)
2931
                update object (length=80)
2932
                delete earlier 2 object history versions
2933
                delete next earlier object history version
2934
                delete rest object history and current version
2935
                delete object (404 status)
2936
                delete container
2937
            account usage sequel: 50|150|300|370|450|400|300|150|0|0
2938
        """
2939
        if self.stop_execution:
2940
            return
2941

  
2942
        versions = self.client.retrieve_object_versionlist(
2943
            'container', 'object'
2944
        )['versions']
2945
        mtime = [i[1] for i in versions]
2946

  
2947
        # purge some object history
2948
        i = 1
2949
        until = int(mtime[i])
2950
        self.client.delete_object('container', 'object', until=until)
2951
        meta = self.client.retrieve_account_metadata()
2952
        self.usage = int(meta['x-account-bytes-used'])
2953
        print 'Current account usage: %d' % self.usage
2954
        self.curr_acc_usage = self.curr_acc_usage - sum(self.l[:i])
2955
        self.assertEqual(self.usage, self.curr_acc_usage)
2956

  
2957
        # purge some container history
2958
        j = 2
2959
        until = int(mtime[j])
2960
        self.client.delete_container('container', until=until)
2961
        meta = self.client.retrieve_account_metadata()
2962
        self.usage = int(meta['x-account-bytes-used'])
2963
        print 'Current account usage: %d' % self.usage
2964
        self.curr_acc_usage = self.curr_acc_usage - sum(self.l[i:j])
2965
        self.assertEqual(self.usage, self.curr_acc_usage)
2966

  
2967
        # purge some more object history
2968
        k = 3
2969
        until = int(mtime[k])
2970
        self.client.delete_object('container', 'object', until=until)
2971
        meta = self.client.retrieve_account_metadata()
2972
        self.usage = int(meta['x-account-bytes-used'])
2973
        print 'Current account usage: %d' % self.usage
2974
        self.curr_acc_usage = self.curr_acc_usage - sum(self.l[j:k])
2975
        self.assertEqual(self.usage, self.curr_acc_usage)
2976

  
2977
        # purge some object history and current
2978
        t = datetime.datetime.utcnow()
2979
        now = int(_time.mktime(t.timetuple()))
2980
        self.client.delete_object('container', 'object', until=now)
2981
        meta = self.client.retrieve_account_metadata()
2982
        self.usage = int(meta['x-account-bytes-used'])
2983
        print 'Current account usage: %d' % self.usage
2984
        self.assertEqual(self.usage, self.initial_usage)
2985

  
2986
        # try to delete object and assert object is not found
2987
        self.assert_raises_fault(
2988
            404, self.client.delete_object, 'container', 'object'
2989
        )
2990

  
2991
        meta = self.client.retrieve_account_metadata()
2992
        self.usage = int(meta['x-account-bytes-used'])
2993
        print 'Current account usage: %d' % self.usage
2994
        self.assertEqual(self.usage, self.initial_usage)
2995

  
2996
        self.client.delete_container('container')
2997
        meta = self.client.retrieve_account_metadata()
2998
        self.usage = int(meta['x-account-bytes-used'])
2999
        print 'Current account usage: %d' % self.usage
3000
        self.assertEqual(self.usage, self.initial_usage)
3001

  
3002
    def test_delete_object_empty_container_content(self):
3003
        """
3004
            assume account usage = 0
3005
            scenario:
3006
                upload object (length=50)
3007
                update object (length=100)
3008
                update object (length=150)
3009
                update object (length=70)
3010
                update object (length=80)
3011
                delete object
3012
                delete container contents
3013
                delete container
3014
            account usage sequel: 50|150|300|370|450|450|450|0
3015
        """
3016
        if self.stop_execution:
3017
            return
3018

  
3019
        self.client.delete_object('container', 'object')
3020

  
3021
        meta = self.client.retrieve_account_metadata()
3022
        self.usage = int(meta['x-account-bytes-used'])
3023
        print 'Current account usage: %d' % self.usage
3024
        self.assertEqual(self.usage, self.curr_acc_usage)
3025

  
3026
        self.client.delete_container('container', delimiter='/')
3027
        meta = self.client.retrieve_account_metadata()
3028
        self.usage = int(meta['x-account-bytes-used'])
3029
        print 'Current account usage: %d' % self.usage
3030
        self.assertEqual(self.usage, self.curr_acc_usage)
3031

  
3032
        self.client.delete_container('container')
3033
        meta = self.client.retrieve_account_metadata()
3034
        self.usage = int(meta['x-account-bytes-used'])
3035
        print 'Current account usage: %d' % self.usage
3036
        self.assertEqual(self.usage, self.initial_usage)
3037

  
3038
    def test_delete_container_content(self):
3039
        """
3040
            assume account usage = 0
3041
            scenario:
3042
                upload object (length=50)
3043
                update object (length=100)
3044
                update object (length=150)
3045
                update object (length=70)
3046
                update object (length=80)
3047
                delete container contents
3048
                delete container
3049
            account usage sequel: 50|150|300|370|450|450|0
3050
        """
3051
        if self.stop_execution:
3052
            return
3053

  
3054
        self.client.delete_container('container', delimiter='/')
3055
        meta = self.client.retrieve_account_metadata()
3056
        self.usage = int(meta['x-account-bytes-used'])
3057
        print 'Current account usage: %d' % self.usage
3058
        self.assertEqual(self.usage, self.curr_acc_usage)
3059

  
3060
        self.client.delete_container('container')
3061
        meta = self.client.retrieve_account_metadata()
3062
        self.usage = int(meta['x-account-bytes-used'])
3063
        print 'Current account usage: %d' % self.usage
3064
        self.assertEqual(self.usage, self.initial_usage)
3065

  
3066
class TestUsageDebitVersioningNoneContainerPolicy(BaseTestCase):
3067
    """ Challenge debit version accounting
3068
       in a container with none versioning policy
3069

  
3070
       In case of unknown server version accounting policy or
3071
       free version accounting policy
3072
       enforce all test cases to return immediately
3073
    """
3074
    def setUp(self):
3075
        BaseTestCase.setUp(self)
3076

  
3077
        self.stop_execution = False
3078
        try:
3079
            from pithos.api.settings import BACKEND_FREE_VERSIONING
3080
        except ImportError:
3081
            print 'Unable to execute the test: unknown version accounting policy'
3082
            self.stop_execution = True
3083
            return
3084
        else:
3085
            if BACKEND_FREE_VERSIONING:
3086
                print 'Unable to execute the test: free version accounting policy'
3087
                self.stop_execution = True
3088
                return
3089

  
3090
        self.create_container('container')
3091
        meta = self.client.retrieve_account_metadata()
3092
        self.initial_usage = int(meta['x-account-bytes-used'])
3093
        print 'Current account usage: %d' % self.initial_usage
3094

  
3095
        l =  (50, 100, 150, 70, 80)
3096
        self.curr_acc_usage = self.initial_usage
3097
        for i, length in list(enumerate(l)):
3098
            self.upload_random_data('container', 'object', length=length)
3099
            meta = self.client.retrieve_account_metadata()
3100
            self.usage = int(meta['x-account-bytes-used'])
3101
            print 'Current account usage: %d' % self.usage
3102
            self.curr_acc_usage = self.initial_usage + length
3103
            self.assertEqual(self.usage, self.curr_acc_usage)
3104
            _time.sleep(1)
3105

  
3106
        versions = self.client.retrieve_object_versionlist(
3107
            'container', 'object'
3108
        )['versions']
3109
        self.mtime = [int(i[1]) for i in versions]
3110

  
3111
    def create_container(self, cname):
3112
        self.client.create_container(cname,
3113
                                     policies={'versioning':'none'})
3114

  
3115
    def test_delete_object_container(self):
3116
        """
3117
            assume account usage = 0
3118
            scenario:
3119
                upload object (length=50)
3120
                update object (length=100)
3121
                update object (length=150)
3122
                update object (length=70)
3123
                update object (length=80)
3124
                delete object
3125
                delete container
3126
            account usage sequel: 50|100|150|70|80|0|0
3127
        """
3128
        if self.stop_execution:
3129
            return
3130

  
3131
        self.client.delete_object('container', 'object')
3132
        meta = self.client.retrieve_account_metadata()
3133
        self.usage = int(meta['x-account-bytes-used'])
3134
        print 'Current account usage: %d' % self.usage
3135
        self.assertEqual(self.usage, self.initial_usage)
3136

  
3137
        self.client.delete_container('container')
3138
        meta = self.client.retrieve_account_metadata()
3139
        self.usage = int(meta['x-account-bytes-used'])
3140
        print 'Current account usage: %d' % self.usage
3141
        self.assertEqual(self.usage, self.initial_usage)
3142

  
3143
    def test_purge_delete_object(self):
3144
        """
3145
            assume account usage = 0
3146
            scenario:
3147
                upload object (length=50)
3148
                update object (length=100)
3149
                update object (length=150)
3150
                update object (length=70)
3151
                update object (length=80)
3152
                delete object history
3153
                delete object
3154
                delete container
3155
            account usage sequel: 50|100|150|70|80|80|0|0
3156
        """
3157
        if self.stop_execution:
3158
            return
3159

  
3160
        # purge some object history
3161
        self.client.delete_object('container', 'object', until=self.mtime[0])
3162
        meta = self.client.retrieve_account_metadata()
3163
        self.usage = int(meta['x-account-bytes-used'])
3164
        print 'Current account usage: %d' % self.usage
3165
        self.assertEqual(self.usage, self.curr_acc_usage)
3166

  
3167
        self.client.delete_object('container', 'object')
3168
        meta = self.client.retrieve_account_metadata()
3169
        self.usage = int(meta['x-account-bytes-used'])
3170
        print 'Current account usage: %d' % self.usage
3171
        self.assertEqual(self.usage, self.initial_usage)
3172

  
3173
        self.client.delete_container('container')
3174
        meta = self.client.retrieve_account_metadata()
3175
        self.usage = int(meta['x-account-bytes-used'])
3176
        print 'Current account usage: %d' % self.usage
3177
        self.assertEqual(self.usage, self.initial_usage)
3178

  
3179
    def test_delete_object_purge_container_history(self):
3180
        """
3181
            assume account usage = 0
3182
            scenario:
3183
                upload object (length=50)
3184
                update object (length=100)
3185
                update object (length=150)
3186
                update object (length=70)
3187
                update object (length=80)
3188
                delete object
3189
                delete container history
3190
                delete container
3191
            account usage sequel: 50|100|150|70|80|0|0|0
3192
        """
3193
        if self.stop_execution:
3194
            return
3195

  
3196
        self.client.delete_object('container', 'object')
3197
        meta = self.client.retrieve_account_metadata()
3198
        self.usage = int(meta['x-account-bytes-used'])
3199
        print 'Current account usage: %d' % self.usage
3200
        self.assertEqual(self.usage, self.initial_usage)
3201

  
3202
        # purge some container history
3203
        self.client.delete_container('container', until=self.mtime[0])
3204
        meta = self.client.retrieve_account_metadata()
3205
        self.usage = int(meta['x-account-bytes-used'])
3206
        print 'Current account usage: %d' % self.usage
3207
        self.assertEqual(self.usage, self.initial_usage)
3208

  
3209
        self.client.delete_container('container')
3210
        meta = self.client.retrieve_account_metadata()
3211
        self.usage = int(meta['x-account-bytes-used'])
3212
        print 'Current account usage: %d' % self.usage
3213
        self.assertEqual(self.usage, self.initial_usage)
3214

  
3215
    def test_purge_container_delete_object(self):
3216
        """
3217
            assume account usage = 0
3218
            scenario:
3219
                upload object (length=50)
3220
                update object (length=100)
3221
                update object (length=150)
3222
                update object (length=70)
3223
                update object (length=80)
3224
                delete container history
3225
                delete object
3226
                delete container
3227
            account usage sequel: 50|100|150|70|80|80|0|0
3228
        """
3229
        if self.stop_execution:
3230
            return
3231

  
3232
        # purge some container history
3233
        self.client.delete_container('container', until=self.mtime[0])
3234
        meta = self.client.retrieve_account_metadata()
3235
        self.usage = int(meta['x-account-bytes-used'])
3236
        print 'Current account usage: %d' % self.usage
3237
        self.assertEqual(self.usage, self.curr_acc_usage)
3238

  
3239
        self.client.delete_object('container', 'object')
3240
        meta = self.client.retrieve_account_metadata()
3241
        self.usage = int(meta['x-account-bytes-used'])
3242
        print 'Current account usage: %d' % self.usage
3243
        self.assertEqual(self.usage, self.initial_usage)
3244

  
3245
        self.client.delete_container('container')
3246
        meta = self.client.retrieve_account_metadata()
3247
        self.usage = int(meta['x-account-bytes-used'])
3248
        print 'Current account usage: %d' % self.usage
3249
        self.assertEqual(self.usage, self.initial_usage)
3250

  
3251
    def test_successive_purging(self):
3252
        """
3253
            assume account usage = 0
3254
            scenario:
3255
                upload object (length=50)
3256
                update object (length=100)
3257
                update object (length=150)
3258
                update object (length=70)
3259
                update object (length=80)
3260
                delete earlier 2 object history versions
3261
                delete next earlier object history version
3262
                delete rest object history and current version
3263
                delete object (404 status)
3264
                delete container
3265
            account usage sequel: 50|100|150|70|80|80|80|80|0|0|0
3266
        """
3267
        if self.stop_execution:
3268
            return
3269

  
3270
        # purge some object history
3271
        i = random.randrange(len(self.mtime))
3272
        self.client.delete_object('container', 'object', until=self.mtime[i])
3273
        meta = self.client.retrieve_account_metadata()
3274
        self.usage = int(meta['x-account-bytes-used'])
3275
        print 'Current account usage: %d' % self.usage
3276
        self.assertEqual(self.usage, self.curr_acc_usage)
3277

  
3278
        # purge some object history and current
3279
        t = datetime.datetime.utcnow()
3280
        now = int(_time.mktime(t.timetuple()))
3281
        self.client.delete_object('container', 'object', until=now)
3282
        meta = self.client.retrieve_account_metadata()
3283
        self.usage = int(meta['x-account-bytes-used'])
3284
        print 'Current account usage: %d' % self.usage
3285
        self.assertEqual(self.usage, self.initial_usage)
3286

  
3287
        # try to delete object and assert object is not found
3288
        self.assert_raises_fault(
3289
            404, self.client.delete_object, 'container', 'object'
3290
        )
3291

  
3292
        meta = self.client.retrieve_account_metadata()
3293
        self.usage = int(meta['x-account-bytes-used'])
3294
        print 'Current account usage: %d' % self.usage
3295
        self.assertEqual(self.usage, self.initial_usage)
3296

  
3297
        self.client.delete_container('container')
3298
        meta = self.client.retrieve_account_metadata()
3299
        self.usage = int(meta['x-account-bytes-used'])
3300
        print 'Current account usage: %d' % self.usage
3301
        self.assertEqual(self.usage, self.initial_usage)
3302

  
3303
    def test_delete_object_empty_container_content(self):
3304
        """
3305
            assume account usage = 0
3306
            scenario:
3307
                upload object (length=50)
3308
                update object (length=100)
3309
                update object (length=150)
3310
                update object (length=70)
3311
                update object (length=80)
3312
                delete object
3313
                delete container contents
3314
                delete container
3315
            account usage sequel: 50|100|150|70|80|0|0|0
3316
        """
3317
        if self.stop_execution:
3318
            return
3319

  
3320
        self.client.delete_object('container', 'object')
3321
        meta = self.client.retrieve_account_metadata()
3322
        self.usage = int(meta['x-account-bytes-used'])
3323
        print 'Current account usage: %d' % self.usage
3324
        self.assertEqual(self.usage, self.initial_usage)
3325

  
3326
        self.client.delete_container('container', delimiter='/')
3327
        meta = self.client.retrieve_account_metadata()
3328
        self.usage = int(meta['x-account-bytes-used'])
3329
        print 'Current account usage: %d' % self.usage
3330
        self.assertEqual(self.usage, self.initial_usage)
3331

  
3332
        self.client.delete_container('container')
3333
        meta = self.client.retrieve_account_metadata()
3334
        self.usage = int(meta['x-account-bytes-used'])
3335
        print 'Current account usage: %d' % self.usage
3336
        self.assertEqual(self.usage, self.initial_usage)
3337

  
3338
    def test_delete_container_content(self):
3339
        """
3340
            assume account usage = 0
3341
            scenario:
3342
                upload object (length=50)
3343
                update object (length=100)
3344
                update object (length=150)
3345
                update object (length=70)
3346
                update object (length=80)
3347
                delete container contents
3348
                delete container
3349
            account usage sequel: 50|100|150|70|80|0|0
3350
        """
3351
        if self.stop_execution:
3352
            return
3353

  
3354
        self.client.delete_container('container', delimiter='/')
3355
        meta = self.client.retrieve_account_metadata()
3356
        self.usage = int(meta['x-account-bytes-used'])
3357
        print 'Current account usage: %d' % self.usage
3358
        self.assertEqual(self.usage, self.initial_usage)
3359

  
3360
        self.client.delete_container('container')
3361
        meta = self.client.retrieve_account_metadata()
3362
        self.usage = int(meta['x-account-bytes-used'])
3363
        print 'Current account usage: %d' % self.usage
3364
        self.assertEqual(self.usage, self.initial_usage)
3365

  
2388 3366
class AssertUUidInvariant(object):
2389 3367
    def __init__(self, callable, *args, **kwargs):
2390 3368
        self.callable = callable
b/snf-pithos-backend/pithos/backends/lib/sqlalchemy/node.py
309 309
    def node_purge_children(self, parent, before=inf, cluster=0):
310 310
        """Delete all versions with the specified
311 311
           parent and cluster, and return
312
           the hashes and size of versions deleted.
312
           the hashes, the total size and the serials of versions deleted.
313 313
           Clears out nodes with no remaining versions.
314 314
        """
315 315
        #update statistics
......
317 317
                    self.nodes.c.parent == parent)
318 318
        where_clause = and_(self.versions.c.node.in_(c1),
319 319
                            self.versions.c.cluster == cluster)
320
        if before != inf:
321
            where_clause = and_(where_clause,
322
                                self.versions.c.mtime <= before)
320 323
        s = select([func.count(self.versions.c.serial),
321 324
                    func.sum(self.versions.c.size)])
322 325
        s = s.where(where_clause)
323
        if before != inf:
324
            s = s.where(self.versions.c.mtime <= before)
325 326
        r = self.conn.execute(s)
326 327
        row = r.fetchone()
327 328
        r.close()
328 329
        if not row:
329 330
            return (), 0, ()
330
        nr, size = row[0], -row[1] if row[1] else 0
331
        nr, size = row[0], row[1] if row[1] else 0
331 332
        mtime = time()
332
        self.statistics_update(parent, -nr, size, mtime, cluster)
333
        self.statistics_update_ancestors(parent, -nr, size, mtime, cluster)
333
        self.statistics_update(parent, -nr, -size, mtime, cluster)
334
        self.statistics_update_ancestors(parent, -nr, -size, mtime, cluster)
334 335

  
335 336
        s = select([self.versions.c.hash, self.versions.c.serial])
336 337
        s = s.where(where_clause)
......
373 374
                    func.sum(self.versions.c.size)])
374 375
        where_clause = and_(self.versions.c.node == node,
375 376
                            self.versions.c.cluster == cluster)
376
        s = s.where(where_clause)
377 377
        if before != inf:
378
            s = s.where(self.versions.c.mtime <= before)
378
            where_clause = and_(where_clause,
379
                                self.versions.c.mtime <= before)
380
        s = s.where(where_clause)
379 381
        r = self.conn.execute(s)
380 382
        row = r.fetchone()
381 383
        nr, size = row[0], row[1]
......
405 407
                   and_(self.nodes.c.node == node,
406 408
                        select([func.count(self.versions.c.serial)],
407 409
                               self.versions.c.node == self.nodes.c.node).as_scalar() == 0))
408
        r = self.conn.execute(s)
409
        nodes = r.fetchall()
410
        r.close()
410
        rp= self.conn.execute(s)
411
        nodes = [r[0] for r in rp.fetchall()]
412
        rp.close()
411 413
        if nodes:
412 414
            s = self.nodes.delete().where(self.nodes.c.node.in_(nodes))
413 415
            self.conn.execute(s).close()
b/snf-pithos-backend/pithos/backends/lib/sqlite/node.py
259 259
    def node_purge_children(self, parent, before=inf, cluster=0):
260 260
        """Delete all versions with the specified
261 261
           parent and cluster, and return
262
           the hashes and size of versions deleted.
262
           the hashes, the size and the serials of versions deleted.
263 263
           Clears out nodes with no remaining versions.
264 264
        """
265 265

  
......
311 311
    def node_purge(self, node, before=inf, cluster=0):
312 312
        """Delete all versions with the specified
313 313
           node and cluster, and return
314
           the hashes and size of versions deleted.
314
           the hashes, the size and the serials of versions deleted.
315 315
           Clears out the node if it has no remaining versions.
316 316
        """
317 317

  
b/snf-pithos-backend/pithos/backends/modular.py
538 538
            for h in hashes:
539 539
                self.store.map_delete(h)
540 540
            self.node.node_purge_children(node, until, CLUSTER_DELETED)
541
            self._report_size_change(user, account, -size,
542
                                     {'action':'container purge', 'path': path,
543
                                      'versions': ','.join(str(i) for i in serials)})
541
            if not self.free_versioning:
542
                self._report_size_change(
543
                    user, account, -size, {
544
                        'action':'container purge',
545
                        'path': path,
546
                        'versions': ','.join(str(i) for i in serials)
547
                    }
548
                )
544 549
            return
545 550

  
546 551
        if not delimiter:
......
552 557
                self.store.map_delete(h)
553 558
            self.node.node_purge_children(node, inf, CLUSTER_DELETED)
554 559
            self.node.node_remove(node)
555
            self._report_size_change(user, account, -size,
556
                                     {'action': 'container delete',
557
                                      'path': path,
558
                                      'versions': ','.join(str(i) for i in serials)})
560
            if not self.free_versioning:
561
                self._report_size_change(
562
                    user, account, -size, {
563
                        'action':'container purge',
564
                        'path': path,
565
                        'versions': ','.join(str(i) for i in serials)
566
                    }
567
                )
559 568
        else:
560 569
            # remove only contents
561 570
            src_names = self._list_objects_no_limit(user, account, container, prefix='', delimiter=None, virtual=False, domain=None, keys=[], shared=False, until=None, size_range=None, all_props=True, public=False)
......
995 1004
            serials += v
996 1005
            h, s, v = self.node.node_purge(node, until, CLUSTER_HISTORY)
997 1006
            hashes += h
998
            size += s
1007
            if not self.free_versioning:
1008
                size += s
999 1009
            serials += v
1000 1010
            for h in hashes:
1001 1011
                self.store.map_delete(h)
......
1004 1014
                props = self._get_version(node)
1005 1015
            except NameError:
1006 1016
                self.permissions.access_clear(path)
1007
            self._report_size_change(user, account, -size,
1008
                                    {'action': 'object purge', 'path': path,
1009
                                     'versions': ','.join(str(i) for i in serials)})
1017
            self._report_size_change(
1018
                user, account, -size, {
1019
                    'action': 'object purge',
1020
                    'path': path,
1021
                    'versions': ','.join(str(i) for i in serials)
1022
                }
1023
            )
1010 1024
            return
1011 1025

  
1012 1026
        path, node = self._lookup_object(account, container, name)

Also available in: Unified diff