Change menu ``Report`` to ``Usage``
[astakos] / snf-astakos-app / astakos / im / views.py
index eec7eec..708549b 100644 (file)
@@ -39,17 +39,13 @@ engine = inflect.engine()
 
 from urllib import quote
 from functools import wraps
-from datetime import datetime, timedelta
-from collections import defaultdict
+from datetime import datetime
 
 from django.contrib import messages
 from django.contrib.auth.decorators import login_required
-from django.contrib.auth.views import password_change
 from django.core.urlresolvers import reverse
 from django.db import transaction
-from django.db.models import Q
 from django.db.utils import IntegrityError
-from django.forms.fields import URLField
 from django.http import (HttpResponse, HttpResponseBadRequest,
                          HttpResponseForbidden, HttpResponseRedirect,
                          HttpResponseBadRequest, Http404)
@@ -57,51 +53,49 @@ from django.shortcuts import redirect
 from django.template import RequestContext, loader as template_loader
 from django.utils.http import urlencode
 from django.utils.translation import ugettext as _
-from django.views.generic.create_update import (create_object, delete_object,
+from django.views.generic.create_update import (delete_object,
                                                 get_model_and_form_class)
-from django.views.generic.list_detail import object_list, object_detail
-from django.http import HttpResponseBadRequest
+from django.views.generic.list_detail import object_list
 from django.core.xheaders import populate_xheaders
-
-from astakos.im.models import (AstakosUser, ApprovalTerms, AstakosGroup,
-                               Resource, EmailChange, GroupKind, Membership,
-                               AstakosGroupQuota, RESOURCE_SEPARATOR)
+from django.core.exceptions import ValidationError, PermissionDenied
+from django.template.loader import render_to_string
 from django.views.decorators.http import require_http_methods
+from django.db.models import Q
 
 from astakos.im.activation_backends import get_backend, SimpleBackend
-from astakos.im.util import get_context, prepare_response, set_cookie, get_query
+from astakos.im.models import (AstakosUser, ApprovalTerms, AstakosGroup,
+                               EmailChange, GroupKind, Membership,
+                               RESOURCE_SEPARATOR, AstakosUserAuthProvider)
+from astakos.im.util import get_context, prepare_response, get_query, restrict_next
 from astakos.im.forms import (LoginForm, InvitationForm, ProfileForm,
                               FeedbackForm, SignApprovalTermsForm,
-                              ExtendedPasswordChangeForm, EmailChangeForm,
+                              EmailChangeForm,
                               AstakosGroupCreationForm, AstakosGroupSearchForm,
                               AstakosGroupUpdateForm, AddGroupMembersForm,
-                              AstakosGroupSortForm, MembersSortForm,
+                              MembersSortForm, AstakosGroupSortForm,
                               TimelineForm, PickResourceForm,
                               AstakosGroupCreationSummaryForm)
 from astakos.im.functions import (send_feedback, SendMailError,
                                   logout as auth_logout,
                                   activate as activate_func,
-                                  switch_account_to_shibboleth,
+                                  send_activation as send_activation_func,
                                   send_group_creation_notification,
                                   SendNotificationError)
-from astakos.im.endpoints.quotaholder import timeline_charge
-from astakos.im.settings import (COOKIE_NAME, COOKIE_DOMAIN, LOGOUT_NEXT,
-                                 LOGGING_LEVEL, PAGINATE_BY)
-from astakos.im.tasks import request_billing
+from astakos.im.endpoints.qh import timeline_charge
+from astakos.im.settings import (COOKIE_DOMAIN, LOGOUT_NEXT,
+                                 LOGGING_LEVEL, PAGINATE_BY, RESOURCES_PRESENTATION_DATA, PAGINATE_BY_ALL)
+#from astakos.im.tasks import request_billing
 from astakos.im.api.callpoint import AstakosCallpoint
 
 import astakos.im.messages as astakos_messages
+from astakos.im import settings
+from astakos.im import auth_providers
 
 logger = logging.getLogger(__name__)
 
-
-DB_REPLACE_GROUP_SCHEME = """REPLACE(REPLACE("auth_group".name, 'http://', ''),
-                                     'https://', '')"""
-
 callpoint = AstakosCallpoint()
 
-def render_response(template, tab=None, status=200, reset_cookie=False,
-                    context_instance=None, **kwargs):
+def render_response(template, tab=None, status=200, context_instance=None, **kwargs):
     """
     Calls ``django.template.loader.render_to_string`` with an additional ``tab``
     keyword argument and returns an ``django.http.HttpResponse`` with the
@@ -113,10 +107,28 @@ def render_response(template, tab=None, status=200, reset_cookie=False,
     html = template_loader.render_to_string(
         template, kwargs, context_instance=context_instance)
     response = HttpResponse(html, status=status)
-    if reset_cookie:
-        set_cookie(response, context_instance['request'].user)
     return response
 
+def requires_auth_provider(provider_id, **perms):
+    """
+    """
+    def decorator(func, *args, **kwargs):
+        @wraps(func)
+        def wrapper(request, *args, **kwargs):
+            provider = auth_providers.get_provider(provider_id)
+
+            if not provider or not provider.is_active():
+                raise PermissionDenied
+
+            if provider:
+                for pkey, value in perms.iteritems():
+                    attr = 'is_available_for_%s' % pkey.lower()
+                    if getattr(provider, attr)() != value:
+                        raise PermissionDenied
+            return func(request, *args)
+        return wrapper
+    return decorator
+
 
 def requires_anonymous(func):
     """
@@ -151,7 +163,7 @@ def signed_terms_required(func):
 
 @require_http_methods(["GET", "POST"])
 @signed_terms_required
-def index(request, login_template_name='im/login.html', extra_context=None):
+def index(request, login_template_name='im/login.html', profile_template_name='im/profile.html', extra_context=None):
     """
     If there is logged on user renders the profile page otherwise renders login page.
 
@@ -173,12 +185,16 @@ def index(request, login_template_name='im/login.html', extra_context=None):
     im/profile.html or im/login.html or ``template_name`` keyword argument.
 
     """
+    extra_context = extra_context or {}
     template_name = login_template_name
     if request.user.is_authenticated():
-        return HttpResponseRedirect(reverse('edit_profile'))
-    return render_response(template_name,
-                           login_form=LoginForm(request=request),
-                           context_instance=get_context(request, extra_context))
+        return HttpResponseRedirect(reverse('astakos.im.views.edit_profile'))
+
+    return render_response(
+        template_name,
+        login_form = LoginForm(request=request),
+        context_instance = get_context(request, extra_context)
+    )
 
 
 @require_http_methods(["GET", "POST"])
@@ -216,8 +232,8 @@ def invite(request, template_name='im/invitations.html', extra_context=None):
     The view expectes the following settings are defined:
 
     * LOGIN_URL: login uri
-    * ASTAKOS_DEFAULT_CONTACT_EMAIL: service support email
     """
+    extra_context = extra_context or {}
     status = None
     message = None
     form = InvitationForm()
@@ -293,18 +309,29 @@ def edit_profile(request, template_name='im/profile.html', extra_context=None):
     * LOGIN_URL: login uri
     """
     extra_context = extra_context or {}
-    form = ProfileForm(instance=request.user)
+    form = ProfileForm(
+        instance=request.user,
+        session_key=request.session.session_key
+    )
     extra_context['next'] = request.GET.get('next')
-    reset_cookie = False
     if request.method == 'POST':
-        form = ProfileForm(request.POST, instance=request.user)
+        form = ProfileForm(
+            request.POST,
+            instance=request.user,
+            session_key=request.session.session_key
+        )
         if form.is_valid():
             try:
                 prev_token = request.user.auth_token
                 user = form.save()
-                reset_cookie = user.auth_token != prev_token
-                form = ProfileForm(instance=user)
-                next = request.POST.get('next')
+                form = ProfileForm(
+                    instance=user,
+                    session_key=request.session.session_key
+                )
+                next = restrict_next(
+                    request.POST.get('next'),
+                    domain=COOKIE_DOMAIN
+                )
                 if next:
                     return redirect(next)
                 msg = _(astakos_messages.PROFILE_UPDATED)
@@ -312,14 +339,21 @@ def edit_profile(request, template_name='im/profile.html', extra_context=None):
             except ValueError, ve:
                 messages.success(request, ve)
     elif request.method == "GET":
-        if not request.user.is_verified:
-            request.user.is_verified = True
-            request.user.save()
+        request.user.is_verified = True
+        request.user.save()
+
+    # existing providers
+    user_providers = request.user.get_active_auth_providers()
+
+    # providers that user can add
+    user_available_providers = request.user.get_available_auth_providers()
+
     return render_response(template_name,
-                           reset_cookie=reset_cookie,
-                           profile_form=form,
-                           context_instance=get_context(request,
-                                                        extra_context))
+                           profile_form = form,
+                           user_providers = user_providers,
+                           user_available_providers = user_available_providers,
+                           context_instance = get_context(request,
+                                                          extra_context))
 
 
 @transaction.commit_manually
@@ -359,14 +393,26 @@ def signup(request, template_name='im/signup.html', on_success='im/signup_comple
     im/signup.html or ``template_name`` keyword argument.
     im/signup_complete.html or ``on_success`` keyword argument.
     """
+    extra_context = extra_context or {}
     if request.user.is_authenticated():
         return HttpResponseRedirect(reverse('edit_profile'))
 
     provider = get_query(request).get('provider', 'local')
+    if not auth_providers.get_provider(provider).is_available_for_create():
+        raise PermissionDenied
+
+    id = get_query(request).get('id')
+    try:
+        instance = AstakosUser.objects.get(id=id) if id else None
+    except AstakosUser.DoesNotExist:
+        instance = None
+
+    third_party_token = request.REQUEST.get('third_party_token', None)
+
     try:
         if not backend:
             backend = get_backend(request)
-        form = backend.get_signup_form(provider)
+        form = backend.get_signup_form(provider, instance)
     except Exception, e:
         form = SimpleBackend(request).get_signup_form(provider)
         messages.error(request, e)
@@ -377,14 +423,18 @@ def signup(request, template_name='im/signup.html', on_success='im/signup_comple
                 result = backend.handle_activation(user)
                 status = messages.SUCCESS
                 message = result.message
-                user.save()
+
+                form.store_user(user, request)
+
                 if 'additional_email' in form.cleaned_data:
                     additional_email = form.cleaned_data['additional_email']
                     if additional_email != user.email:
                         user.additionalmail_set.create(email=additional_email)
                         msg = 'Additional email: %s saved for user %s.' % (
-                            additional_email, user.email)
-                        logger.log(LOGGING_LEVEL, msg)
+                            additional_email,
+                            user.email
+                        )
+                        logger._log(LOGGING_LEVEL, msg, [])
                 if user and user.is_active:
                     next = request.POST.get('next', '')
                     response = prepare_response(request, user, next=next)
@@ -392,19 +442,28 @@ def signup(request, template_name='im/signup.html', on_success='im/signup_comple
                     return response
                 messages.add_message(request, status, message)
                 transaction.commit()
-                return render_response(on_success,
-                                       context_instance=get_context(request, extra_context))
+                return render_response(
+                    on_success,
+                    context_instance=get_context(
+                        request,
+                        extra_context
+                    )
+                )
             except SendMailError, e:
+                logger.exception(e)
+                status = messages.ERROR
                 message = e.message
                 messages.error(request, message)
                 transaction.rollback()
             except BaseException, e:
+                logger.exception(e)
                 message = _(astakos_messages.GENERIC_ERROR)
                 messages.error(request, message)
                 logger.exception(e)
                 transaction.rollback()
     return render_response(template_name,
                            signup_form=form,
+                           third_party_token=third_party_token,
                            provider=provider,
                            context_instance=get_context(request, extra_context))
 
@@ -437,8 +496,8 @@ def feedback(request, template_name='im/feedback.html', email_template_name='im/
     **Settings:**
 
     * LOGIN_URL: login uri
-    * ASTAKOS_DEFAULT_CONTACT_EMAIL: List of feedback recipients
     """
+    extra_context = extra_context or {}
     if request.method == 'GET':
         form = FeedbackForm()
     if request.method == 'POST':
@@ -461,32 +520,31 @@ def feedback(request, template_name='im/feedback.html', email_template_name='im/
                            context_instance=get_context(request, extra_context))
 
 
-@require_http_methods(["GET", "POST"])
+@require_http_methods(["GET"])
 @signed_terms_required
 def logout(request, template='registration/logged_out.html', extra_context=None):
     """
-    Wraps `django.contrib.auth.logout` and delete the cookie.
+    Wraps `django.contrib.auth.logout`.
     """
+    extra_context = extra_context or {}
     response = HttpResponse()
     if request.user.is_authenticated():
         email = request.user.email
         auth_logout(request)
-        response.delete_cookie(COOKIE_NAME, path='/', domain=COOKIE_DOMAIN)
-        msg = 'Cookie deleted for %s' % email
-        logger.log(LOGGING_LEVEL, msg)
-    next = request.GET.get('next')
+    next = restrict_next(
+        request.GET.get('next'),
+        domain=COOKIE_DOMAIN
+    )
     if next:
         response['Location'] = next
         response.status_code = 302
-        return response
     elif LOGOUT_NEXT:
         response['Location'] = LOGOUT_NEXT
         response.status_code = 301
-        return response
-    messages.success(request, _(astakos_messages.LOGOUT_SUCCESS))
-    context = get_context(request, extra_context)
-    response.write(
-        template_loader.render_to_string(template, context_instance=context))
+    else:
+        messages.add_message(request, messages.SUCCESS, _(astakos_messages.LOGOUT_SUCCESS))
+        context = get_context(request, extra_context)
+        response.write(render_to_string(template, context_instance=context))
     return response
 
 
@@ -514,58 +572,27 @@ def activate(request, greeting_email_template_name='im/welcome_email.txt',
         return index(request)
 
     try:
-        local_user = AstakosUser.objects.get(
-            ~Q(id=user.id),
-            email=user.email,
-            is_active=True
-        )
-    except AstakosUser.DoesNotExist:
-        try:
-            activate_func(
-                user,
-                greeting_email_template_name,
-                helpdesk_email_template_name,
-                verify_email=True
-            )
-            response = prepare_response(request, user, next, renew=True)
-            transaction.commit()
-            return response
-        except SendMailError, e:
-            message = e.message
-            messages.error(request, message)
-            transaction.rollback()
-            return index(request)
-        except BaseException, e:
-            message = _(astakos_messages.GENERIC_ERROR)
-            messages.error(request, message)
-            logger.exception(e)
-            transaction.rollback()
-            return index(request)
-    else:
-        try:
-            user = switch_account_to_shibboleth(
-                user,
-                local_user,
-                greeting_email_template_name
-            )
-            response = prepare_response(request, user, next, renew=True)
-            transaction.commit()
-            return response
-        except SendMailError, e:
-            message = e.message
-            messages.error(request, message)
-            transaction.rollback()
-            return index(request)
-        except BaseException, e:
-            message = _(astakos_messages.GENERIC_ERROR)
-            messages.error(request, message)
-            logger.exception(e)
-            transaction.rollback()
-            return index(request)
+        activate_func(user, greeting_email_template_name, helpdesk_email_template_name, verify_email=True)
+        response = prepare_response(request, user, next, renew=True)
+        transaction.commit()
+        return response
+    except SendMailError, e:
+        message = e.message
+        messages.add_message(request, messages.ERROR, message)
+        transaction.rollback()
+        return index(request)
+    except BaseException, e:
+        status = messages.ERROR
+        message = _(astakos_messages.GENERIC_ERROR)
+        messages.add_message(request, messages.ERROR, message)
+        logger.exception(e)
+        transaction.rollback()
+        return index(request)
 
 
 @require_http_methods(["GET", "POST"])
 def approval_terms(request, term_id=None, template_name='im/approval_terms.html', extra_context=None):
+    extra_context = extra_context or {}
     term = None
     terms = None
     if not term_id:
@@ -586,7 +613,10 @@ def approval_terms(request, term_id=None, template_name='im/approval_terms.html'
     terms = f.read()
 
     if request.method == 'POST':
-        next = request.POST.get('next')
+        next = restrict_next(
+            request.POST.get('next'),
+            domain=COOKIE_DOMAIN
+        )
         if not next:
             next = reverse('index')
         form = SignApprovalTermsForm(request.POST, instance=request.user)
@@ -608,22 +638,15 @@ def approval_terms(request, term_id=None, template_name='im/approval_terms.html'
 
 
 @require_http_methods(["GET", "POST"])
-@signed_terms_required
-def change_password(request):
-    return password_change(request,
-                           post_change_redirect=reverse('edit_profile'),
-                           password_change_form=ExtendedPasswordChangeForm)
-
-
-@require_http_methods(["GET", "POST"])
-@signed_terms_required
 @login_required
+@signed_terms_required
 @transaction.commit_manually
 def change_email(request, activation_key=None,
                  email_template_name='registration/email_change_email.txt',
                  form_template_name='registration/email_change_form.html',
                  confirm_template_name='registration/email_change_done.html',
                  extra_context=None):
+    extra_context = extra_context or {}
     if activation_key:
         try:
             user = EmailChange.objects.change_email(activation_key)
@@ -661,100 +684,141 @@ def change_email(request, activation_key=None,
             msg = _(astakos_messages.EMAIL_CHANGE_REGISTERED)
             messages.success(request, msg)
             transaction.commit()
-    return render_response(form_template_name,
-                           form=form,
-                           context_instance=get_context(request,
-                                                        extra_context))
-
-
-
-resource_presentation = {
-       'compute': {
-            'help_text':'group compute help text',
-            'is_abbreviation':False,
-            'report_desc':''
-        },
-        'storage': {
-            'help_text':'group storage help text',
-            'is_abbreviation':False,
-            'report_desc':''
-        },
-        'pithos+.diskspace': {
-            'help_text':'resource pithos+.diskspace help text',
-            'is_abbreviation':False,
-            'report_desc':'Pithos+ Diskspace',
-            'placeholder':'eg. 10GB'
-        },
-        'cyclades.vm': {
-            'help_text':'resource cyclades.vm help text resource cyclades.vm help text resource cyclades.vm help text resource cyclades.vm help text',
-            'is_abbreviation':True,
-            'report_desc':'Virtual Machines',
-            'placeholder':'eg. 2'
-        },
-        'cyclades.disksize': {
-            'help_text':'resource cyclades.disksize help text',
-            'is_abbreviation':False,
-            'report_desc':'Disksize',
-            'placeholder':'eg. 5GB'
-        },
-        'cyclades.disk': {
-            'help_text':'resource cyclades.disk help text',
-            'is_abbreviation':False,
-            'report_desc':'Disk',
-            'placeholder':'eg. 5GB'    
-        },
-        'cyclades.ram': {
-            'help_text':'resource cyclades.ram help text',
-            'is_abbreviation':True,
-            'report_desc':'RAM',
-            'placeholder':'eg. 4GB'
-        },
-        'cyclades.cpu': {
-            'help_text':'resource cyclades.cpu help text',
-            'is_abbreviation':True,
-            'report_desc':'CPUs',
-            'placeholder':'eg. 1'
-        },
-        'cyclades.network.private': {
-            'help_text':'resource cyclades.network.private help text',
-            'is_abbreviation':False,
-            'report_desc':'Network',
-            'placeholder':'eg. 1'
-        }
-    }
+    return render_response(
+        form_template_name,
+        form=form,
+        context_instance=get_context(request, extra_context)
+    )
+
+
+def send_activation(request, user_id, template_name='im/login.html', extra_context=None):
+
+    if settings.MODERATION_ENABLED:
+        raise PermissionDenied
+
+    extra_context = extra_context or {}
+    try:
+        u = AstakosUser.objects.get(id=user_id)
+    except AstakosUser.DoesNotExist:
+        messages.error(request, _(astakos_messages.ACCOUNT_UNKNOWN))
+    else:
+        try:
+            send_activation_func(u)
+            msg = _(astakos_messages.ACTIVATION_SENT)
+            messages.success(request, msg)
+        except SendMailError, e:
+            messages.error(request, e)
+    return render_response(
+        template_name,
+        login_form = LoginForm(request=request),
+        context_instance = get_context(
+            request,
+            extra_context
+        )
+    )
+
+class ResourcePresentation():
+
+    def __init__(self, data):
+        self.data = data
+
+    def update_from_result(self, result):
+        if result.is_success:
+            for r in result.data:
+                rname = '%s%s%s' % (r.get('service'), RESOURCE_SEPARATOR, r.get('name'))
+                if not rname in self.data['resources']:
+                    self.data['resources'][rname] = {}
+
+                self.data['resources'][rname].update(r)
+                self.data['resources'][rname]['id'] = rname
+                group = r.get('group')
+                if not group in self.data['groups']:
+                    self.data['groups'][group] = {}
+
+                self.data['groups'][r.get('group')].update({'name': r.get('group')})
+
+    def test(self, quota_dict):
+        for k, v in quota_dict.iteritems():
+            rname = k
+            value = v
+            if not rname in self.data['resources']:
+                self.data['resources'][rname] = {}
+
+
+            self.data['resources'][rname]['value'] = value
+
+
+    def update_from_result_report(self, result):
+        if result.is_success:
+            for r in result.data:
+                rname = r.get('name')
+                if not rname in self.data['resources']:
+                    self.data['resources'][rname] = {}
+
+                self.data['resources'][rname].update(r)
+                self.data['resources'][rname]['id'] = rname
+                group = r.get('group')
+                if not group in self.data['groups']:
+                    self.data['groups'][group] = {}
+
+                self.data['groups'][r.get('group')].update({'name': r.get('group')})
+
+    def get_group_resources(self, group):
+        return dict(filter(lambda t: t[1].get('group') == group, self.data['resources'].iteritems()))
+
+    def get_groups_resources(self):
+        for g in self.data['groups']:
+            yield g, self.get_group_resources(g)
+
+    def get_quota(self, group_quotas):
+        for r, v in group_quotas.iteritems():
+            rname = str(r)
+            quota = self.data['resources'].get(rname)
+            quota['value'] = v
+            yield quota
+
+
+    def get_policies(self, policies_data):
+        for policy in policies_data:
+            rname = '%s%s%s' % (policy.get('service'), RESOURCE_SEPARATOR, policy.get('resource'))
+            policy.update(self.data['resources'].get(rname))
+            yield policy
+
+    def __repr__(self):
+        return self.data.__repr__()
+
+    def __iter__(self, *args, **kwargs):
+        return self.data.__iter__(*args, **kwargs)
+
+    def __getitem__(self, *args, **kwargs):
+        return self.data.__getitem__(*args, **kwargs)
+
+    def get(self, *args, **kwargs):
+        return self.data.get(*args, **kwargs)
+
+
 
 @require_http_methods(["GET", "POST"])
 @signed_terms_required
 @login_required
 def group_add(request, kind_name='default'):
+
     result = callpoint.list_resources()
-    resource_catalog = {'resources':defaultdict(defaultdict),
-                        'groups':defaultdict(list)}
-    if result.is_success:
-        for r in result.data:
-            service = r.get('service', '')
-            name = r.get('name', '')
-            group = r.get('group', '')
-            unit = r.get('unit', '')
-            fullname = '%s%s%s' % (service, RESOURCE_SEPARATOR, name)
-            resource_catalog['resources'][fullname] = dict(unit=unit)
-            resource_catalog['groups'][group].append(fullname)
-        
-        resource_catalog = dict(resource_catalog)
-        for k, v in resource_catalog.iteritems():
-            resource_catalog[k] = dict(v)
-    else:
+    resource_catalog = ResourcePresentation(RESOURCES_PRESENTATION_DATA)
+    resource_catalog.update_from_result(result)
+
+    if not result.is_success:
         messages.error(
             request,
             'Unable to retrieve system resources: %s' % result.reason
     )
-    
+
     try:
         kind = GroupKind.objects.get(name=kind_name)
     except:
         return HttpResponseBadRequest(_(astakos_messages.GROUPKIND_UNKNOWN))
-    
-    
+
+
 
     post_save_redirect = '/im/group/%(id)s/'
     context_processors = None
@@ -762,23 +826,27 @@ def group_add(request, kind_name='default'):
         model=None,
         form_class=AstakosGroupCreationForm
     )
-    
+
     if request.method == 'POST':
         form = form_class(request.POST, request.FILES)
         if form.is_valid():
+            policies = form.policies()
             return render_response(
                 template='im/astakosgroup_form_summary.html',
                 context_instance=get_context(request),
-                form = AstakosGroupCreationSummaryForm(form.cleaned_data),
-                policies = form.policies(),
-                resource_catalog=resource_catalog,
-                resource_presentation=resource_presentation
+                form=AstakosGroupCreationSummaryForm(form.cleaned_data),
+                policies=resource_catalog.get_policies(policies)
             )
     else:
         now = datetime.now()
         data = {
-            'kind': kind
+            'kind': kind,
         }
+        for group, resources in resource_catalog.get_groups_resources():
+            data['is_selected_%s' % group] = False
+            for resource in resources:
+                data['%s_uplimit' % resource] = ''
+
         form = form_class(data)
 
     # Create the template, context, response
@@ -791,7 +859,6 @@ def group_add(request, kind_name='default'):
         'form': form,
         'kind': kind,
         'resource_catalog':resource_catalog,
-        'resource_presentation':resource_presentation,
     }, context_processors)
     return HttpResponse(t.render(c))
 
@@ -812,7 +879,7 @@ def group_add_complete(request):
             msg = _(astakos_messages.OBJECT_CREATED) %\
                 {"verbose_name": model._meta.verbose_name}
             messages.success(request, msg, fail_silently=True)
-            
+
             # send notification
             try:
                 send_group_creation_notification(
@@ -828,14 +895,16 @@ def group_add_complete(request):
             post_save_redirect = '/im/group/%(id)s/'
             return HttpResponseRedirect(post_save_redirect % new_object)
         else:
-            msg = _(astakos_messages.OBJECT_CREATED_FAILED) %\
-                {"verbose_name": model._meta.verbose_name,
+            d = {"verbose_name": model._meta.verbose_name,
                  "reason":result.reason}
+            msg = _(astakos_messages.OBJECT_CREATED_FAILED) % d
             messages.error(request, msg, fail_silently=True)
     return render_response(
         template='im/astakosgroup_form_summary.html',
         context_instance=get_context(request),
-        form=form)
+        form=form,
+        policies=form.cleaned_data.get('policies')
+    )
 
 
 #@require_http_methods(["GET"])
@@ -844,9 +913,9 @@ def group_add_complete(request):
 @login_required
 def group_list(request):
     none = request.user.astakos_groups.none()
-    q = AstakosGroup.objects.raw("""
+    query = """
         SELECT auth_group.id,
-        %s AS groupname,
+        auth_group.name AS groupname,
         im_groupkind.name AS kindname,
         im_astakosgroup.*,
         owner.email AS groupowner,
@@ -856,7 +925,7 @@ def group_list(request):
         (SELECT CASE WHEN(
                     SELECT date_joined FROM im_membership
                     WHERE group_id = im_astakosgroup.group_ptr_id
-                    AND person_id = %s) IS NULL
+                    AND person_id = %(id)s) IS NULL
                     THEN 0 ELSE 1 END) AS membership_status
         FROM im_astakosgroup
         INNER JOIN im_membership ON (
@@ -867,43 +936,33 @@ def group_list(request):
             im_astakosuser_owner.astakosgroup_id = im_astakosgroup.group_ptr_id)
         LEFT JOIN auth_user as owner ON (
             im_astakosuser_owner.astakosuser_id = owner.id)
-        WHERE im_membership.person_id = %s
-        """ % (DB_REPLACE_GROUP_SCHEME, request.user.id, request.user.id))
-    q = list(q)
-#    d = defaultdict(list)
-#    
-#    for g in q:
-#        if request.user.email == g.groupowner:
-#            d['own'].append(g)
-#        else:
-#            d['other'].append(g)
-#    
-#    for g in q:
-#        d['all'].append(g)
-#        
-#        
-#        
-#    
-#    # validate sorting
-#    fields = ('own', 'other', 'all')
-#    for f in fields:
-#        v = globals()['%s_sorting' % f] = request.GET.get('%s_sorting' % f)
-#        if v:
-#            form = AstakosGroupSortForm({'sort_by': v})
-#            if not form.is_valid():
-#                globals()['%s_sorting' % f] = form.cleaned_data.get('sort_by')
-    return object_list(request, queryset=none,
-                       extra_context={'is_search': False,
-                                      'q': q,
-#                                      'mine': d['own'],
-#                                      'other': d['other'],
-#                                      'own_sorting': own_sorting,
-#                                      'other_sorting': other_sorting,
-                                      'sorting': request.GET.get('sorting'),
-#                                      'own_page': request.GET.get('own_page', 1),
-#                                      'other_page': request.GET.get('other_page', 1),
-#                                      'all_page': request.GET.get('all_page', 1)
-                                      })
+        WHERE im_membership.person_id = %(id)s
+        AND im_groupkind.name != 'default'
+        """ % request.user.__dict__
+
+    # validate sorting
+    sorting = 'groupname'
+    sort_form = AstakosGroupSortForm(request.GET)
+    if sort_form.is_valid():
+        sorting = sort_form.cleaned_data.get('sorting')
+    query = query+" ORDER BY %s ASC" %sorting
+    
+    q = AstakosGroup.objects.raw(query)
+    
+    # Create the template, context, response
+    template_name = "%s/%s_list.html" % (
+        q.model._meta.app_label,
+        q.model._meta.object_name.lower()
+    )
+    extra_context = dict(
+        is_search=False,
+        q=q,
+        sorting=sorting,
+        page=request.GET.get('page', 1)
+    )
+    return render_response(template_name,
+                           context_instance=get_context(request, extra_context)
+    )
 
 
 @require_http_methods(["GET", "POST"])
@@ -966,16 +1025,28 @@ def group_detail(request, group_id):
     }, context_processors)
 
     # validate sorting
-    sorting = request.GET.get('sorting')
-    if sorting:
-        form = MembersSortForm({'sort_by': sorting})
-        if form.is_valid():
-            sorting = form.cleaned_data.get('sort_by')
+    sorting = 'person__email'
+    form = MembersSortForm(request.GET)
+    if form.is_valid():
+        sorting = form.cleaned_data.get('sorting')
+    
+    result = callpoint.list_resources()
+    resource_catalog = ResourcePresentation(RESOURCES_PRESENTATION_DATA)
+    resource_catalog.update_from_result(result)
+
+
+    if not result.is_success:
+        messages.error(
+            request,
+            'Unable to retrieve system resources: %s' % result.reason
+    )
 
     extra_context = {'update_form': update_form,
                      'addmembers_form': addmembers_form,
                      'page': request.GET.get('page', 1),
-                     'sorting': sorting}
+                     'sorting': sorting,
+                     'resource_catalog':resource_catalog,
+                     'quota':resource_catalog.get_quota(obj.quota)}
     for key, value in extra_context.items():
         if callable(value):
             c[key] = value()
@@ -992,19 +1063,21 @@ def group_detail(request, group_id):
 @login_required
 def group_search(request, extra_context=None, **kwargs):
     q = request.GET.get('q')
-    sorting = request.GET.get('sorting')
     if request.method == 'GET':
         form = AstakosGroupSearchForm({'q': q} if q else None)
     else:
         form = AstakosGroupSearchForm(get_query(request))
         if form.is_valid():
             q = form.cleaned_data['q'].strip()
+    
+    sorting = 'groupname'
     if q:
         queryset = AstakosGroup.objects.select_related()
+        queryset = queryset.filter(~Q(kind__name='default'))
         queryset = queryset.filter(name__contains=q)
         queryset = queryset.filter(approval_date__isnull=False)
         queryset = queryset.extra(select={
-                                  'groupname': DB_REPLACE_GROUP_SCHEME,
+                                  'groupname': "auth_group.name",
                                   'kindname': "im_groupkind.name",
                                   'approved_members_num': """
                     SELECT COUNT(*) FROM im_membership
@@ -1025,16 +1098,26 @@ def group_search(request, extra_context=None, **kwargs):
                     SELECT id FROM im_astakosuser_owner
                     WHERE astakosgroup_id = im_astakosgroup.group_ptr_id
                     AND astakosuser_id = %s)
-                    THEN 1 ELSE 0 END""" % request.user.id})
-        if sorting:
-            # TODO check sorting value
-            queryset = queryset.order_by(sorting)
+                    THEN 1 ELSE 0 END""" % request.user.id,
+                    'is_owner': """SELECT CASE WHEN EXISTS(
+                        SELECT id FROM im_astakosuser_owner
+                        WHERE astakosgroup_id = im_astakosgroup.group_ptr_id
+                        AND astakosuser_id = %s)
+                        THEN 1 ELSE 0 END""" % request.user.id,
+                    })
+        
+        # validate sorting
+        sort_form = AstakosGroupSortForm(request.GET)
+        if sort_form.is_valid():
+            sorting = sort_form.cleaned_data.get('sorting')
+        queryset = queryset.order_by(sorting)
+
     else:
         queryset = AstakosGroup.objects.none()
     return object_list(
         request,
         queryset,
-        paginate_by=PAGINATE_BY,
+        paginate_by=PAGINATE_BY_ALL,
         page=request.GET.get('page') or 1,
         template_name='im/astakosgroup_list.html',
         extra_context=dict(form=form,
@@ -1048,9 +1131,10 @@ def group_search(request, extra_context=None, **kwargs):
 @login_required
 def group_all(request, extra_context=None, **kwargs):
     q = AstakosGroup.objects.select_related()
+    q = q.filter(~Q(kind__name='default'))
     q = q.filter(approval_date__isnull=False)
     q = q.extra(select={
-                'groupname': DB_REPLACE_GROUP_SCHEME,
+                'groupname': "auth_group.name",
                 'kindname': "im_groupkind.name",
                 'approved_members_num': """
                     SELECT COUNT(*) FROM im_membership
@@ -1065,15 +1149,24 @@ def group_all(request, extra_context=None, **kwargs):
                     SELECT date_joined FROM im_membership
                     WHERE group_id = im_astakosgroup.group_ptr_id
                     AND person_id = %s)
-                    THEN 1 ELSE 0 END""" % request.user.id})
-    sorting = request.GET.get('sorting')
-    if sorting:
-        # TODO check sorting value
-        q = q.order_by(sorting)
+                    THEN 1 ELSE 0 END""" % request.user.id,
+                 'is_owner': """SELECT CASE WHEN EXISTS(
+                        SELECT id FROM im_astakosuser_owner
+                        WHERE astakosgroup_id = im_astakosgroup.group_ptr_id
+                        AND astakosuser_id = %s)
+                        THEN 1 ELSE 0 END""" % request.user.id,   })
+    
+    # validate sorting
+    sorting = 'groupname'
+    sort_form = AstakosGroupSortForm(request.GET)
+    if sort_form.is_valid():
+        sorting = sort_form.cleaned_data.get('sorting')
+    q = q.order_by(sorting)
+    
     return object_list(
         request,
         q,
-        paginate_by=PAGINATE_BY,
+        paginate_by=PAGINATE_BY_ALL,
         page=request.GET.get('page') or 1,
         template_name='im/astakosgroup_list.html',
         extra_context=dict(form=AstakosGroupSearchForm(),
@@ -1111,7 +1204,7 @@ def group_leave(request, group_id):
             group__id=group_id,
             person=request.user)
     except Membership.DoesNotExist:
-        return HttpResponseBadRequest(_(astakos_messages.NOT_A_MEMBER))
+        return HttpResponseBadRequest(_(astakos_messages.NOT_MEMBER))
     if request.user in m.group.owner.all():
         return HttpResponseForbidden(_(astakos_messages.OWNER_CANNOT_LEAVE_GROUP))
     return delete_object(
@@ -1169,7 +1262,7 @@ def disapprove_member(request, membership):
     try:
         membership.disapprove()
         realname = membership.person.realname
-        msg = MEMBER_REMOVED % realname
+        msg = astakos_messages.MEMBER_REMOVED % locals()
         messages.success(request, msg)
     except BaseException, e:
         logger.exception(e)
@@ -1181,33 +1274,15 @@ def disapprove_member(request, membership):
 @require_http_methods(["POST", "GET"])
 @signed_terms_required
 @login_required
-def resource_list(request):
-#     if request.method == 'POST':
-#         form = PickResourceForm(request.POST)
-#         if form.is_valid():
-#             r = form.cleaned_data.get('resource')
-#             if r:
-#                 groups = request.user.membership_set.only('group').filter(
-#                     date_joined__isnull=False)
-#                 groups = [g.group_id for g in groups]
-#                 q = AstakosGroupQuota.objects.select_related().filter(
-#                     resource=r, group__in=groups)
-#     else:
-#         form = PickResourceForm()
-#         q = AstakosGroupQuota.objects.none()
-#
-#     return object_list(request, q,
-#                        template_name='im/astakosuserquota_list.html',
-#                        extra_context={'form': form, 'data':data})
-
+def resource_usage(request):
     def with_class(entry):
         entry['load_class'] = 'red'
         max_value = float(entry['maxValue'])
         curr_value = float(entry['currValue'])
         if max_value > 0 :
-           entry['ratio'] = (curr_value / max_value) * 100
-        else: 
-           entry['ratio'] = 0 
+            entry['ratio'] = (curr_value / max_value) * 100
+        else:
+            entry['ratio'] = 0
         if entry['ratio'] < 66:
             entry['load_class'] = 'yellow'
         if entry['ratio'] < 33:
@@ -1218,17 +1293,20 @@ def resource_list(request):
         entry['plural'] = engine.plural(entry.get('name'))
         return entry
 
-    result = callpoint.get_user_status(request.user.id)
+    result = callpoint.get_user_usage(request.user.id)
     if result.is_success:
         backenddata = map(with_class, result.data)
         data = map(pluralize, result.data)
     else:
         data = None
         messages.error(request, result.reason)
-    return render_response('im/resource_list.html',
+    resource_catalog = ResourcePresentation(RESOURCES_PRESENTATION_DATA)
+    resource_catalog.update_from_result_report(result)
+    return render_response('im/resource_usage.html',
                            data=data,
-                           resource_presentation=resource_presentation,
-                           context_instance=get_context(request))
+                           context_instance=get_context(request),
+                           resource_catalog=resource_catalog,
+                           result=result)
 
 
 def group_create_list(request):
@@ -1238,69 +1316,66 @@ def group_create_list(request):
         context_instance=get_context(request),)
 
 
-#@require_http_methods(["GET"])
-@require_http_methods(["POST", "GET"])
-@signed_terms_required
-@login_required
-def billing(request):
-
-    today = datetime.today()
-    month_last_day = calendar.monthrange(today.year, today.month)[1]
-    data['resources'] = map(with_class, data['resources'])
-    start = request.POST.get('datefrom', None)
-    if start:
-        today = datetime.fromtimestamp(int(start))
-        month_last_day = calendar.monthrange(today.year, today.month)[1]
-
-    start = datetime(today.year, today.month, 1).strftime("%s")
-    end = datetime(today.year, today.month, month_last_day).strftime("%s")
-    r = request_billing.apply(args=('pgerakios@grnet.gr',
-                                    int(start) * 1000,
-                                    int(end) * 1000))
-    data = {}
-
-    try:
-        status, data = r.result
-        data = _clear_billing_data(data)
-        if status != 200:
-            messages.error(request, _(astakos_messages.BILLING_ERROR) % status)
-    except:
-        messages.error(request, r.result)
-
-    print type(start)
-
-    return render_response(
-        template='im/billing.html',
-        context_instance=get_context(request),
-        data=data,
-        zerodate=datetime(month=1, year=1970, day=1),
-        today=today,
-        start=int(start),
-        month_last_day=month_last_day)
-
-
-def _clear_billing_data(data):
-
-    # remove addcredits entries
-    def isnotcredit(e):
-        return e['serviceName'] != "addcredits"
+##@require_http_methods(["GET"])
+#@require_http_methods(["POST", "GET"])
+#@signed_terms_required
+#@login_required
+#def billing(request):
+#
+#    today = datetime.today()
+#    month_last_day = calendar.monthrange(today.year, today.month)[1]
+#    start = request.POST.get('datefrom', None)
+#    if start:
+#        today = datetime.fromtimestamp(int(start))
+#        month_last_day = calendar.monthrange(today.year, today.month)[1]
+#
+#    start = datetime(today.year, today.month, 1).strftime("%s")
+#    end = datetime(today.year, today.month, month_last_day).strftime("%s")
+#    r = request_billing.apply(args=('pgerakios@grnet.gr',
+#                                    int(start) * 1000,
+#                                    int(end) * 1000))
+#    data = {}
+#
+#    try:
+#        status, data = r.result
+#        data = _clear_billing_data(data)
+#        if status != 200:
+#            messages.error(request, _(astakos_messages.BILLING_ERROR) % status)
+#    except:
+#        messages.error(request, r.result)
+#
+#    return render_response(
+#        template='im/billing.html',
+#        context_instance=get_context(request),
+#        data=data,
+#        zerodate=datetime(month=1, year=1970, day=1),
+#        today=today,
+#        start=int(start),
+#        month_last_day=month_last_day)
 
-    # separate services
-    def servicefilter(service_name):
-        service = service_name
 
-        def fltr(e):
-            return e['serviceName'] == service
-        return fltr
+#def _clear_billing_data(data):
+#
+#    # remove addcredits entries
+#    def isnotcredit(e):
+#        return e['serviceName'] != "addcredits"
+#
+#    # separate services
+#    def servicefilter(service_name):
+#        service = service_name
+#
+#        def fltr(e):
+#            return e['serviceName'] == service
+#        return fltr
+#
+#    data['bill_nocredits'] = filter(isnotcredit, data['bill'])
+#    data['bill_vmtime'] = filter(servicefilter('vmtime'), data['bill'])
+#    data['bill_diskspace'] = filter(servicefilter('diskspace'), data['bill'])
+#    data['bill_addcredits'] = filter(servicefilter('addcredits'), data['bill'])
+#
+#    return data
 
-    data['bill_nocredits'] = filter(isnotcredit, data['bill'])
-    data['bill_vmtime'] = filter(servicefilter('vmtime'), data['bill'])
-    data['bill_diskspace'] = filter(servicefilter('diskspace'), data['bill'])
-    data['bill_addcredits'] = filter(servicefilter('addcredits'), data['bill'])
 
-    return data
-     
-     
 #@require_http_methods(["GET"])
 @require_http_methods(["POST", "GET"])
 @signed_terms_required
@@ -1330,3 +1405,27 @@ def timeline(request):
                            timeline_header=timeline_header,
                            timeline_body=timeline_body)
     return data
+
+
+# TODO: action only on POST and user should confirm the removal
+@require_http_methods(["GET", "POST"])
+@login_required
+@signed_terms_required
+def remove_auth_provider(request, pk):
+    try:
+        provider = request.user.auth_providers.get(pk=pk)
+    except AstakosUserAuthProvider.DoesNotExist:
+        raise Http404
+
+    if provider.can_remove():
+        provider.delete()
+        return HttpResponseRedirect(reverse('edit_profile'))
+    else:
+        raise PermissionDenied
+
+
+def how_it_works(request):
+    return render_response(
+        template='im/how_it_works.html',
+        context_instance=get_context(request),)
+