staLUN: EMC LUN statistics

# -*- coding: utf-8 -*- vim:encoding=utf-8:

# vim: tabstop=4:shiftwidth=4:softtabstop=4:expandtab

#

#Copyright © 2013 Greek Research and Technology Network (GRNET S.A.)

#

#Developed by Leonidas Poulopoulos (leopoul-at-noc-dot-grnet-dot-gr) and

#George Kargiotakis (kargig-at-noc-dot-grnet-dot-gr), GRNET NOC

#

#Permission to use, copy, modify, and/or distribute this software for any

#purpose with or without fee is hereby granted, provided that the above

#copyright notice and this permission notice appear in all copies.

#

#THE SOFTWARE IS PROVIDED "AS IS" AND ISC DISCLAIMS ALL WARRANTIES WITH REGARD

#TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF MERCHANTABILITY AND

#FITNESS. IN NO EVENT SHALL ISC BE LIABLE FOR ANY SPECIAL, DIRECT, INDIRECT, OR

#CONSEQUENTIAL DAMAGES OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE,

#DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS

#ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS

#SOFTWARE.

from core.models import *

from graphs.models import *

from django.core.urlresolvers import reverse

from django.shortcuts import render_to_response,get_object_or_404

from django.http import HttpResponse,HttpResponseRedirect,Http404

from django.conf import settings

from django.views.decorators.cache import cache_page

from django.views.decorators.cache import never_cache

from django.core.cache import cache

from django.template import RequestContext

from django.db.models import Q

import json

import datetime

import time

import rrdtool

import tempfile

import os

import re

import random

import pprint

from gevent.pool import Pool

from gevent.timeout import Timeout

from operator import itemgetter

from util.analyze import analyze

from core.forms import *

def lun_tags():

    tagslist = cache.get('luntags')

    if tagslist is None:

        tags = []

        luns = Lun.objects.all()

        for lun in luns:

            for i in lun.tags.all():

                if i:

                    tags.extend(i.name for i in lun.tags.all())

        tags = list(set(tags))

        tags.sort()

        tagslist = []

        for tag in tags:

            tag_dict = {}

            tag_dict['text'] = tag

            tag_dict['id'] = tag

            tagslist.append(tag_dict)

        cache.set('luntags',tagslist, 300)

    return tagslist

def index(request):

    start='-30h'

    end='-8m'

    q_tags = None

    q_period = None

    q_zeroavg = False

    q_diskio = False

    q_requests = False

    q_sortbystd = False

    q_groupby = 'clear'

    q_topfive = False

    selected_tags = []

    tags = lun_tags()

    all_tags=[]

    all_tags.extend([i['text'] for i in tags])

    try:

        q_tags = request.GET['tags']

    except:

        pass

    try:

        q_period = request.GET['period']

        if q_period == '2hours':

            start = "-2h"

            end = '-8m'

        if q_period == 'daily':

            start = "-30h"

            end = '-8m'

        if q_period == 'weekly':

            start = "-7d"

            end = '-8m'

        if q_period == 'monthly':

            start = "-30d"

            end = '-8m'

        if q_period == 'yearly':

            start = "-365d"

            end = '-8m'

    except:

        q_period = 'daily'

    try:

        q_zeroavg = request.GET['zeroavg']

        q_zeroavg = True

    except:

        pass

    if q_tags:

        #This should be tags

        selected_tags = q_tags.split(',')

    try:

        q_diskio = request.GET['diskio']

    except:

        pass

    try:

        q_requests = request.GET['requests']

    except:

        pass

    if not q_diskio and not q_requests:

        q_diskio = True

        q_requests = True

    try:

        q_groupby = request.GET['groupby']

    except:

        pass

    try:

        q_sortbystd = request.GET['sortbystd']

    except:

        pass

    try:

        q_topfive = request.GET['topfive']

        q_sortbystd = True

    except:

        pass

    luns = Lun.objects.all()

    if q_tags:

        lunsfiltered = luns.filter(tags__name__in=selected_tags).distinct()

    if q_tags and 'uncategorized' in q_tags:

        lunsfiltered = luns.filter(Q(tags__name__in=selected_tags)|~Q(tags__name__in=all_tags)).distinct()

    if len(selected_tags) == 1 and 'uncategorized' in selected_tags:

        lunsfiltered = luns.exclude(tags__name__in=all_tags).distinct()

    if q_diskio:

        graphs = Graph.objects.filter(type='diskio')

    if q_requests:

        graphs = Graph.objects.filter(type='requests')

    if q_diskio and q_requests:

        graphs = Graph.objects.all()

    nl = cache.get('lun:stats')

    if nl is None:

        nl = analyze(luns)

    urls = []

    for i in nl:

        lun = luns.get(pk=i['lunpk'])

        if q_tags:

            try:

                lun = lunsfiltered.get(pk=i['lunpk'])

            except Lun.DoesNotExist:

                continue

        try:

            graph = graphs.get(pk=i['graph'])

        except Graph.DoesNotExist:

            continue

        if q_zeroavg == False:

            if int(i['avg']) == 0:

                continue

        graph_dict = {}

        if q_period == 'daily':

            graph.url = reverse("get-thumbnail",  kwargs={'graph_id': graph.pk, 'start':start, 'end':end})

        else:

            graph.url = reverse("get-thumbnail-non-static",  kwargs={'graph_id': graph.pk, 'start':start, 'end':end})

        graph_dict['url'] = graph.url

        graph_dict['graph'] = graph

        graph_dict['lun'] = lun

        graph_dict['dev'] = i['dev']

        urls.append(graph_dict)

    if q_groupby == 'lun':

        if q_sortbystd:

            urls = sorted(urls, key=lambda k: (k['lun'].name.lower(), -k['dev']))

        else:

            urls = sorted(urls, key=lambda k: (k['lun'].name.lower()))

    elif q_groupby == 'type':

        if q_sortbystd:

            urls = sorted(urls, key=lambda k: (k['graph'].type.lower(), -k['dev']))

        else:

            urls = sorted(urls, key=lambda k: (k['graph'].type.lower()))

    else:

        if q_sortbystd:

            urls = sorted(urls, key=lambda k: (-k['dev']))

        else:

            urls = sorted(urls, key=lambda k:  k['graph'].pk)

    tags.append({'text':'uncategorized', 'id':'uncategorized'})

    if q_topfive:

        urls = urls[:5]

    saved = Query.objects.all()

    ret = {

           'urls' :urls,

           'tags' :tags,

           'saved' : saved,

           'selected_tags' : selected_tags,

           'q_period' : q_period,

           'q_zeroavg' : q_zeroavg,

           'q_diskio' : q_diskio,

           'q_requests' : q_requests,

           'q_groupby' : q_groupby,

           'q_sortbystd' :q_sortbystd,

           'q_topfive': q_topfive,

           }

    return render_to_response('index.html', ret,

                context_instance=RequestContext(request))

def editsearch(request):

    if request.method == 'POST':

        form = queryForm(request.POST)

        if form.is_valid():

            form.save()

            res = {'result':'success'}

            return HttpResponse(json.dumps(res), mimetype='application/json')

        else:

            return render_to_response('querysave.html',

                {

                    'form': form,

                }, 

                context_instance=RequestContext(request)

                )

    elif request.method == 'GET':

        form = queryForm()

        return render_to_response('querysave.html', {

                'form': form }, 

                context_instance=RequestContext(request)

            )

    else:

        return HttpResponseRedirect(reverse('index'))

def deletesearch(request, search_id):

    if request.method == 'POST':

        searchpk = request.POST['searchpk']

        delsearch = Query.objects.get(pk=searchpk).delete()

        res = {'result':'success'}

        return HttpResponse(json.dumps(res), mimetype='application/json')

    elif request.method == 'GET':

        return render_to_response('querydelete.html', {

                'delquery': Query.objects.get(pk=search_id) }, 

                context_instance=RequestContext(request)

            )

    else:

        return HttpResponseRedirect(reverse('index'))

def taggedluns(request):

    tags = lun_tags()

    return HttpResponse(json.dumps(tags), mimetype="application/json")

def graphjson(request, graph_id):

    ret = {"graph_pk": graph_id}

    return render_to_response('graph.html', ret,  

                context_instance=RequestContext(request))

def graphperiods(request, graph_id):

    graph =  Graph.objects.get(pk=graph_id)

    lun = Lun.objects.get(pk=int(graph.tags.filter(name__icontains="lun:")[0].name.replace('lun:','')))

    return render_to_response('graphperiods.html', {'graph': graph, 'lun': lun},

                context_instance=RequestContext(request))

def getpngdata(path,static):

    file = open(path,'r')

    data = file.read()

    file.close()

    if static == False:

        os.remove(path)

    response = HttpResponse(mimetype='image/png')

    response.write(data)

    return response

def drawpng(request, graph_id, static=True, lazy=False, legend=False, start='-30h', end='-8m', jsonexport=False):

    graph = get_object_or_404(Graph,pk=graph_id)

    if static == True:

        lazy = True

        path = '%s/%s%s%s.png' % (settings.RRD_PNG_STATICPATH, graph_id, start, end)

        try:

            statinfo = os.stat(path)

            if time.time() - statinfo.st_mtime < 300 :

                return getpngdata(path,static)

        except OSError:

            pass

    else:

        path = tempfile.NamedTemporaryFile().name

    '''Removing unicode from strings while constructing kwargs'''

    kwargs = { 'start': str(start), 'end': str(end), 'lazy': lazy, 'legend': legend, 'jsonexport':jsonexport }

    result = graph.create_graph(path, **kwargs)

    if result == None:

        raise Http404

    if jsonexport:

        return HttpResponse(json.dumps(result), mimetype="application/json")

    return getpngdata(path, static)

def thumb(request,graph_id=None,legend=True,lazy=True,static=True, start='-30h', end='-8m',):

    graph = get_object_or_404(Graph,pk=graph_id)

    if static == True:

        lazy = True

        path = '%s/thumb-%s%s%s.png' % (settings.RRD_PNG_STATICPATH, graph_id, start, end)

        try:

            statinfo = os.stat(path)

            if time.time() - statinfo.st_mtime < 300 :

                return getpngdata(path,static)

        except OSError:

            pass

    else:

        path = tempfile.NamedTemporaryFile().name

    '''Removing unicode from strings while constructing kwargs'''

    kwargs = { 'start': str(start), 'end': str(end), 'lazy': lazy, 'legend': legend, 'thumb': True, }

    result=graph.create_graph(path,**kwargs)

    return getpngdata(path,static)