staLUN: EMC LUN statistics

settings.py

*.pyc

rrds/

*.db

# -*- coding: utf-8 -*- vim:encoding=utf-8:

# vim: tabstop=4:shiftwidth=4:softtabstop=4:expandtab

from core.models import *

from django.contrib import admin

admin.site.register(Lun)

# -*- coding: utf-8 -*- vim:encoding=utf-8:

# vim: tabstop=4:shiftwidth=4:softtabstop=4:expandtab

from django.conf import settings

from core.models import *

from graphs.models import *

from datetime import datetime, timedelta

import time

import sys

def expireoldgraphs(node, type):

    if node.snmpmanagednode.perm_fail == 1:

        graphs = Graph.objects.filter(host=node)

        graphs.delete()

        return

    graphs = Graph.objects.filter(host=node, type=type,

                                 date__lt=datetime.now()-timedelta(settings.RG_EXPIRE_DAYS))

    graphs.delete()

def dbreg(type, rrdpath, lun, **kwargs):

    try:

        rrdfile = RrdFile.objects.get(path=rrdpath)

    except RrdFile.DoesNotExist, e:

        rrdfile = RrdFile(path=rrdpath)

        rrdfile.save()

    except RrdFile.MultipleObjectsReturned, e:

        #This should never happen

        sys.stderr.write('Multiple RrdFiles in database. This should never happen\n')

        sys.stderr.write('RRD File: %s\n' % rrdpath)

        sys.stderr.write('Refusing to proceed. Fix the problem\n')

        sys.exit()

    if 'tags' in kwargs:

        tags = kwargs['tags']

    else:

        tags = []

    #Making the assumption that an RRDfile will never contain data sources not in the database

    #Also since the rrdfiles are created by mrtg datasource names are hardcoded.

    if rrdfile.datasource_set.count() == 0:

        ds0 = DataSource(name='ds0')

        ds0.rrdfile=rrdfile

        ds0.save()

        ds0.tags.clear()

        for tag in tags:

            ds0.tags.add(tag) 

        # ds0.content_object = content

        ds1 = DataSource(name='ds1')

        ds1.rrdfile=rrdfile

        ds1.save()

        ds1.tags.clear()

        for tag in tags:

            ds1.tags.add(tag) 

        # ds1.content_object = content

        datasources=[ds0,ds1]

    else:

        datasources=rrdfile.datasource_set.all()

        for tempds in datasources:

            tempds.tags.clear()

            for tag in tags:

                tempds.tags.add(tag) 

    datasourcespk = [ds.pk for ds in datasources]

    graphs = Graph.objects.filter(type=type,tags__name='lun:%s'%lun.pk,datasources__pk__in=datasourcespk).distinct()

    if len(graphs) == 1:

        graph = graphs[0]

    if len(graphs) < 1:

        graph = Graph(type=type)

        graph.save()

        graph.tags.add('lun:%s'%lun.pk)

        if 'description' in kwargs:

            graph.tags.add('graphdescription:%s %s'%(lun.name, kwargs['description']))

    elif len(graphs) > 1:

        sys.stderr.write('Found more than one possible graph. Using oldest one as an approximation: %s\n' % e)

        graph = Graph.objects.filter(type=type,tags__name='lun:%s'%lun.pk,datasources__pk__in=datasourcespk).distinct().order_by('-date')[0]

    # Setting various attributes through kwargs

    for i in ('pc95','units','logarithmic','label','description'):

        if i in kwargs:

            setattr(graph,i,kwargs[i])

    graph.save()

    for datasource in datasources:

        try:

            grds = GraphDataSource.objects.get(graph=graph,datasource=datasource)

        except GraphDataSource.DoesNotExist, e:

            grds = GraphDataSource(graph=graph,datasource=datasource)

        # Some defaults

        grds.args = ''

        grds.function = 'LINE1'

        grds.color = '#0000bbbb'

        # Hardcoded default. Max Datasources in a graph.

        for n in range(0,1024):

            for i in ( 'function', 'color', 'args', 'legend', 'stackorder' ):

                if datasource.name == 'ds%s' % n and 'ds%s%s' % ( n, i ) in kwargs:

                    setattr(grds,i,kwargs['ds%s%s' % ( n, i)])

        grds.save()

    try:

        f = open(rrdpath)

    except IOError:

        necreation_epoch = int(time.time())

        f = rrdtool.create(str(rrdpath), str("--start"), str(1349965501),  [str('DS:ds0:COUNTER:300:U:U'), str('DS:ds1:COUNTER:300:U:U')], str('RRA:AVERAGE:0.5:1:288'), str('RRA:AVERAGE:0.5:3:672'), str('RRA:AVERAGE:0.5:12:744'), str('RRA:AVERAGE:0.5:72:1460'))

# -*- coding: utf-8 -*- vim:encoding=utf-8:

# vim: tabstop=4:shiftwidth=4:softtabstop=4:expandtab

from __dbfuncs__ import *

from core.models import *

from django.conf import settings

import rrdtool

import time

import datetime

def diskio_genconfig(lun):

    devfilename = "lun_%s" % lun.pk

    seriesfilename = 'blocksiostats'

    target = "%s_%s" % (devfilename, seriesfilename)

    rrdpath = settings.RRDFILEDIR + '/%s.rrd' % target

    dbreg('diskio', rrdpath, lun, description='Disk IO' , label=target,

            ds0legend='Blocks Read', ds1legend='Blocks Write',

            ds0function='AREA', ds1function='LINE1',

            ds0color='#00bb00bb', ds1color='#0000bbbb',

            pc95=False, units='blocks', tags=['lun:%s'%lun.pk, 'ds0:blocksread', "ds1:blockswrite", "type:blocksiostats"])

def requests_genconfig(lun):

    devfilename = "lun_%s" % lun.pk

    seriesfilename = 'requeststats'

    target = "%s_%s" % (devfilename, seriesfilename)

    rrdpath = settings.RRDFILEDIR + '/%s.rrd' % target

    dbreg('requests', rrdpath, lun, description='Requests IO' , label=target,

            ds0legend='Read Requests', ds1legend='Write Requests',

            ds0function='AREA', ds1function='LINE1',

            ds0color='#00bb00bb', ds1color='#0000bbbb',

            pc95=False, units='reuests', tags=['lun:%s'%lun.pk, 'ds0:readrequests', "ds1:writerequests", "type:requeststats"])

# -*- coing: utf-8 -*- vim:encoding=utf-8:

# vim: tabstop=4:shiftwidth=4:softtabstop=4:expandtab

# Copyright 2013 Leonidas Poulopoulos

#

# Licensed under the Apache License, Version 2.0 (the "License");

# you may not use this file except in compliance with the License.

# You may obtain a copy of the License at

#

#    http://www.apache.org/licenses/LICENSE-2.0

#

# Unless required by applicable law or agreed to in writing, software

# distributed under the License is distributed on an "AS IS" BASIS,

# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

# See the License for the specific language governing permissions and

# limitations under the License.

from django.core.management.base import BaseCommand, CommandError

from django.conf import settings

from core.models import *

import csv

import time

import random

class Command(BaseCommand):

    help = "Populates the lun series measurements from a csv file" \

            "It assumes  that the meassurement file is formatted like:\n" \

            "<unix_timestamp>\n"\

            "LUNID, METRIC_A, METRIC_B, METRIC_C, METRIC_D\n"\

            "1, 0, 2, 2, 1\n"\

            "2, 0, 0, 0, 0\n" \

            "...\n" \

            "N, 1, 2, 3, 1 \n" \

    args = ""

    label = ''

    # HAve that in mind:

    # '''{'lun_id': 12, stats'diskio':['blocksread':189202196, 'blockswrite':279338720], 'requests':[readreq':121123, 'writereq': 2121234], 'timestamp':1412341234}'''

    def handle(self, *args, **options):

        f = open(settings.CSV_FILE, "w")

        out = str(int(time.time())) + ",\n"

        lun_ids = [10, 20, 21, 22, 12, 42, 32, 31, 144, 11, 233, 321134, 324, 4234, 2343, 631, 3456, 2341, 635, 152345, 12345123, 123496, 65431]

        header = ["Blocks Read","Blocks Write","Read Reqs","Write Reqs"]

        out = out + "LUN," +",".join(header) + "," + "\n"

        for (i,k) in enumerate(lun_ids):

            out = out + str(lun_ids[i]) + ","

            for j in header:

                out = out + str(random.randint(8000, 9000)) + ","

            out  = out + "\n"

        f.write(out)

        f.close()

    def handle_json(self, *args, **options):

        f = open(settings.CSV_FILE, "w")

        out = str(int(time.time())) + "\n"

        lun_ids = [10, 20, 21, 22, 12, 42, 32, 31, 144, 11, 233, 321134, 324, 4234, 2343, 631, 3456, 2341, 635, 152345, 12345123, 123496, 65431]

        out = '{"luns": ['

        for id in lun_ids:

            out = out + '{"lun_id":%s, "diskio":{"blocksread":%s, "blockswrite":%s}, "requests":{"readreq":%s, "writereq": %s}, "timestamp":%s}' % (id, random.randint(100, 10000), random.randint(100, 10000), random.randint(100, 10000), random.randint(100, 10000), int(time.time()))

            out = out + ","

        out = out.rstrip(',')

        out = out + ']}'

        f.write(out)

        f.close()

# -*- coing: utf-8 -*- vim:encoding=utf-8:

# vim: tabstop=4:shiftwidth=4:softtabstop=4:expandtab

# Copyright 2013 Leonidas Poulopoulos

#

# Licensed under the Apache License, Version 2.0 (the "License");

# you may not use this file except in compliance with the License.

# You may obtain a copy of the License at

#

#    http://www.apache.org/licenses/LICENSE-2.0

#

# Unless required by applicable law or agreed to in writing, software

# distributed under the License is distributed on an "AS IS" BASIS,

# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

# See the License for the specific language governing permissions and

# limitations under the License.

from django.core.management.base import BaseCommand, CommandError

from django.conf import settings

from core.models import *

from graphs.models import *

import csv

import time

import json

from __genrgconfig__ import *

from __dbfuncs__ import *

# for file listing

from stat import S_ISREG, ST_CTIME, ST_MODE

import os, sys, time, datetime

class Command(BaseCommand):

    help = "Populates the lun series measurements from a csv file" \

            "It assumes  that the meassurement file is formatted like:\n" \

            "<unix_timestamp>\n"\

            "LUNID, METRIC_A, METRIC_B, METRIC_C, METRIC_D\n"\

            "1, 0, 2, 2, 1\n"\

            "2, 0, 0, 0, 0\n" \

            "...\n" \

            "N, 1, 2, 3, 1 \n" \

    args = ""

    label = ''

    def handle(self, *args, **options):

        for statsfile in self.get_files_listing():

            self.getgraph(statsfile)

        print "Finished!"

    def get_files_listing(self):

        # path to the directory (relative or absolute)

        dirpath = '/srv/emc-stats/incoming'

        # get all entries in the directory w/ stats

        entries = (os.path.join(dirpath, fn) for fn in os.listdir(dirpath))

        entries = ((os.stat(path), path) for path in entries)

        # leave only regular files, insert creation date

        entries = ((stat[ST_CTIME], path)

                   for stat, path in entries if S_ISREG(stat[ST_MODE]))

        #NOTE: on Windows `ST_CTIME` is a creation date 

        #  but on Unix it could be something else

        #NOTE: use `ST_MTIME` to sort by a modification date

        statfiles = []

        for cdate, path in sorted(entries):

            statfiles.append("%s/%s"%('/srv/emc-stats/incoming',os.path.basename(path)))

        return statfiles

    def getgraph(self, statsfile):

        # We assume that the following is received

        # 1380622204

        # LUN,Blocks Read,Blocks Write,Read Reqs,Write Reqs,

        # 10,7346,8149,5311,9510,

        # 20,4581,3879,2894,9419,

        f = open(statsfile)

        #print statsfile

        # get first line. This is the timestamp

        timestamp = "%s" %f.readline().replace('\n', '')

        reader = csv.reader(self.remove_last_char(f))

        timestamp = self.round_to_5(int(timestamp))

        out = {}

        luns = []

        for i,serieslist in enumerate(reader):

            #print serieslist

            lundict = {}

            if i == 0:

                #those are headers, ignore (for the time)

                continue

            lundict['lun_name'] = serieslist[0] 

            lundict['timestamp'] = timestamp

            lundict['blocksread'] = serieslist[1]

            lundict['blockswrite'] = serieslist[2]

            lundict['readreq'] = serieslist[3]

            lundict['writereq'] = serieslist[4]

            lun, created = Lun.objects.get_or_create(name="%s"%lundict['lun_name'])

            if created:

                types = settings.LUN_GRAPH_TYPES

                for type in types:

                    self.dbupdate(lun, type)

            lundict['lun_id'] = lun.pk

            luns.append(lundict)

        self.update_graphs(luns)

        f.close()

        print "Done with %s" %statsfile

    def update_graphs(self, lunslist):

        for lundict in lunslist:

            self.update_graph(lundict)

    def update_graph(self, lundict):

        # Try to automate once proof of concept is done

        #print "updating Lun %s"%(lundict['lun_name'])

        dss_block = DataSource.objects.values('rrdfile').filter(tags__name="lun:%s"%lundict['lun_id']).filter(tags__name="type:blocksiostats").distinct()

        # This should return a single element list

        if len(dss_block) != 1:

            print "ERROR: It appears that there are %s blocksiostats datasources for lun %s" %(len(dss_block), lundict['lun_name'])

        else:

            rrdfile = dss_block[0]['rrdfile']

            try:

                rrdtool.update(

                    str('%s'%(RrdFile.objects.get(pk=rrdfile).path)), 

                    str('--template'), str('ds0:ds1'),

                    str(lundict['timestamp']+":" +lundict['blocksread']+":"+lundict['blockswrite']))

            except Exception as e:

                #print e

                pass

        dss_req = DataSource.objects.values('rrdfile').filter(tags__name="lun:%s"%lundict['lun_id']).filter(tags__name="type:requeststats").distinct()  

        # This should return a single element list

        if len(dss_req) != 1:

            print "ERROR: It appears that there are %s requeststats datasources for lun %s" %(len(dss_req), lundict['lun_name'])

        else:

            rrdfile = dss_req[0]['rrdfile']

            try:

                rrdtool.update(

                    str('%s'%(RrdFile.objects.get(pk=rrdfile).path)), 

                    str('--template'), str('ds0:ds1'),

                    str(lundict['timestamp']+":" +lundict['readreq']+":"+lundict['writereq']))

            except Exception as e:

                #print e

                pass

    def round_to_5(self, timestamp):

        tm = datetime.datetime.fromtimestamp(timestamp)

        tm = tm - datetime.timedelta(minutes=tm.minute % 5,

                             seconds=tm.second,

                             microseconds=tm.microsecond)

        return tm.strftime("%s")

    def remove_last_char(self, fileobj):

        for line in fileobj:

            yield line.strip()[:-1]

    def dbupdate(self, device, type):

        # Ok. A couple of hacks here. Instead of a if then elif then elif.... for each type

        # we get first from the settings dictionnary the confdir and then the function used 

        # for this type from the globals() dictionary

        try:

#            dir = getattr(settings,'RG_%s_CONFDIR' % type.upper())

#            self.checkdir(dir)

            func = globals()['%s_genconfig' % type]

            func(device)

        except AttributeError as e:

            print "%s is not going to be configured because %s" % (type, e)

            return

from django.db import models

# Create your models here.

from taggit.managers import TaggableManager

class Lun(models.Model):

    name = models.CharField(max_length=80, null=True, blank=True)

    tags = TaggableManager()

    def __unicode__(self):

        return 'ID: %s, Name: %s' % (self.pk, self.name)

    def getGraphs(self):

        lungraphs = Graph.objects.filter(tags__name='lun:%s'%self.pk)

        return lungraphs 

    def graphCounters(self, dataset):

        '''dataset format\n'''\

        '''{'lun_id': 12, 'diskio':['blocksread':189202196, 'blockswrite':279338720], 'requests':[readreq':121123, 'writereq': 2121234], 'timestamp':1412341234}'''

        if not dataset:

            return "Empty data for lun %s" %(self.name)

        reps = {'/':'_', '#':'_','-':'_', ' ':'_'}

        pmpoints_list = res

        for pmpoints_meassurement in res:

            period_epoch = pmpoints_meassurement['timestamp']

            for serie in self.series.all():

                counterLink = DeviceSeries.objects.get(device=self, series__name=serie.name)

                try:

                    rrdfilename = counterLink.getGraphUrl()

                except Exception as e:

                    print e

                    continue

                if rrdfilename:

                    c_value = pmpoints_meassurement[serie.name]

                    try:

                        rrdtool.update(str('%s'%(rrdfilename)), str(period_epoch+":" + c_value))

                        print "done updating"

                    except Exception as e:

                        print e

                        continue

                else:

                    print "No rrd file found"

                    pass 

from graphs.models import *

"""

This file demonstrates two different styles of tests (one doctest and one

unittest). These will both pass when you run "manage.py test".

Replace these with more appropriate tests for your application.

"""

from django.test import TestCase

class SimpleTest(TestCase):

    def test_basic_addition(self):

        """

        Tests that 1 + 1 always equals 2.

        """

        self.failUnlessEqual(1 + 1, 2)

__test__ = {"doctest": """

Another way to test that 1 + 1 is equal to 2.

>>> 1 + 1 == 2

True

"""}

# -*- coding: utf-8 -*- vim:encoding=utf-8:

# vim: tabstop=4:shiftwidth=4:softtabstop=4:expandtab

from core.models import *

from graphs.models import *

from django.core.urlresolvers import reverse

from django.shortcuts import render_to_response,get_object_or_404

from django.http import HttpResponse,HttpResponseRedirect,Http404

from django.conf import settings

from django.views.decorators.cache import cache_page

from django.views.decorators.cache import never_cache

from django.core.cache import cache

from django.template import RequestContext

import json

import datetime

import time

import rrdtool

import tempfile

import os

import re

import random

import pprint

from gevent.pool import Pool

from gevent.timeout import Timeout

from util.analyze import analyze

def index(request):

    luns = Lun.objects.all()

    graphs = Graph.objects.all()

    nl = cache.get('lun:stats')

    if nl is None:

        nl = analyze(luns)

    urls = []

    for i in nl:

        lun = luns.get(pk=i['lunpk'])

        graph = graphs.get(pk=i['graph'])

        if int(i['avg']) == 0:

            continue 

        #for graph in lungraphs:

        graph_dict = {}

        graph.url = reverse(thumb, args=(graph.pk,))

        graph_dict['url'] = graph.url

        graph_dict['graph'] = graph

        graph_dict['lun'] = lun.name

        urls.append(graph_dict)

    ret = {

           'urls' :urls

           }

    return render_to_response('media.html', ret,

                context_instance=RequestContext(request))

def graphjson(request, graph_id):

    ret = {"graph_pk": graph_id}

    return render_to_response('graph.html', ret,  

                context_instance=RequestContext(request))

def graphperiods(request, graph_id):

    graph =  Graph.objects.get(pk=graph_id)

    return render_to_response('graphperiods.html', {'graph': graph, },

                context_instance=RequestContext(request))

def get_all_json(lazy=False, legend=False, start='-2d', end='-8m'):

    graphs = Graph.objects.all()

    kwargs = { 'start': str(start), 'end': str(end), 'lazy': lazy, 'legend': legend, 'jsonexport':True }

    results = []

    for graph in graphs:

        results.extend(graph.create_graph(path=None, **kwargs))

#     p = Pool(40)

#     

#     def _get_graphs_json(graph):

#         t = Timeout(5)

#         t.start()

#         try:

#             results.extend(graph.create_graph(path=None, **kwargs))

#         except (Timeout):

#             pass

#         finally:

#             t.cancel()

#     p.imap(_get_graphs_json, graphs)

    return results

def getpngdata(path,static):

    file = open(path,'r')

    data = file.read()

    file.close()

    if static == False:

        os.remove(path)

    response = HttpResponse(mimetype='image/png')

    response.write(data)

    return response

def drawpng(request, graph_id, static=True, lazy=False, legend=False, start='-30h', end='-8m', jsonexport=False):

    graph = get_object_or_404(Graph,pk=graph_id)

    if static == True:

        lazy = True

        path = '%s/%s%s%s.png' % (settings.RRD_PNG_STATICPATH, graph_id, start, end)

        try:

            statinfo = os.stat(path)

            if time.time() - statinfo.st_mtime < 300 :

                return getpngdata(path,static)

        except OSError:

            pass

    else:

        path = tempfile.NamedTemporaryFile().name

    '''Removing unicode from strings while constructing kwargs'''

    kwargs = { 'start': str(start), 'end': str(end), 'lazy': lazy, 'legend': legend, 'jsonexport':jsonexport }

    result = graph.create_graph(path, **kwargs)

    if result == None:

        raise Http404

    if jsonexport:

        return HttpResponse(json.dumps(result), mimetype="application/json")

    return getpngdata(path, static)

def thumb(request,graph_id,legend=True,lazy=True,static=True):

    start='-1d'

    end='-300'

    graph = get_object_or_404(Graph,pk=graph_id)

    path = '%s/thumb-%s.png' % (settings.RRD_PNG_STATICPATH, graph_id)

    try:

        statinfo = os.stat(path)

        if time.time() - statinfo.st_mtime < 300 :

            return getpngdata(path,static)

    except OSError:

        pass

    '''Removing unicode from strings while constructing kwargs'''

    kwargs = { 'start': str(start), 'end': str(end), 'lazy': lazy, 'legend': legend, 'thumb': True, }

    result=graph.create_graph(path,**kwargs)

    return getpngdata(path,static)

# -*- coding: utf-8 -*- vim:encoding=utf-8:

# vim: tabstop=4:shiftwidth=4:softtabstop=4:expandtab

from graphs.models import *

from django.contrib import admin

admin.site.register(Graph)

admin.site.register(GraphLabel)

admin.site.register(RrdFile)

admin.site.register(DataSource)

admin.site.register(GraphDataSource)

# -*- coding: utf-8 -*- vim:encoding=utf-8:

# vim: tabstop=4:shiftwidth=4:softtabstop=4:expandtab

from django.conf import settings

from core.models import *

from graphs.models import *

from datetime import datetime, timedelta

import time

import sys

def expireoldgraphs(node, type):

    if node.snmpmanagednode.perm_fail == 1:

        graphs = Graph.objects.filter(host=node)

        graphs.delete()

        return

    graphs = Graph.objects.filter(host=node, type=type,

                                 date__lt=datetime.now()-timedelta(settings.RG_EXPIRE_DAYS))

    graphs.delete()

def dbreg(type, rrdpath, lun, **kwargs):

    try:

        rrdfile = RrdFile.objects.get(path=rrdpath)

    except RrdFile.DoesNotExist, e:

        rrdfile = RrdFile(path=rrdpath)

        rrdfile.save()

    except RrdFile.MultipleObjectsReturned, e:

        #This should never happen

        sys.stderr.write('Multiple RrdFiles in database. This should never happen\n')

        sys.stderr.write('RRD File: %s\n' % rrdpath)

        sys.stderr.write('Refusing to proceed. Fix the problem\n')

        sys.exit()

    if 'tags' in kwargs:

        tags = kwargs['tags']

    else:

        tags = []

    #Making the assumption that an RRDfile will never contain data sources not in the database

    #Also since the rrdfiles are created by mrtg datasource names are hardcoded.

    if rrdfile.datasource_set.count() == 0:

        ds0 = DataSource(name='ds0')

        ds0.rrdfile=rrdfile

        ds0.save()

        ds0.tags.clear()

        for tag in tags:

            if 'ds1:' in tag:

                continue

            ds0.tags.add(tag)

        # ds0.content_object = content

        ds1 = DataSource(name='ds1')

        ds1.rrdfile=rrdfile

        ds1.save()

        ds1.tags.clear()

        for tag in tags:

            if 'ds0:' in tag:

                continue

            ds1.tags.add(tag) 

        # ds1.content_object = content

        datasources=[ds0,ds1]

    else:

        datasources=rrdfile.datasource_set.all()

        for tempds in datasources:

            tempds.tags.clear()

            for tag in tags:

                if 'ds1:' in tag and tempds.name == 'ds0':

                    continue

                if 'ds0:' in tag and tempds.name == 'ds1':

                    continue

                tempds.tags.add(tag) 

    datasourcespk = [ds.pk for ds in datasources]

    graphs = Graph.objects.filter(type=type,tags__name='lun:%s'%lun.pk,datasources__pk__in=datasourcespk).distinct()

    if len(graphs) == 1:

        graph = graphs[0]

    if len(graphs) < 1:

        graph = Graph(type=type)

        graph.save()

        graph.tags.add('lun:%s'%lun.pk)

        if 'description' in kwargs:

            graph.tags.add('graphdescription:%s %s'%(lun.name, kwargs['description']))

    elif len(graphs) > 1:

        sys.stderr.write('Found more than one possible graph. Using oldest one as an approximation: %s\n' % e)

        graph = Graph.objects.filter(type=type,tags__name='lun:%s'%lun.pk,datasources__pk__in=datasourcespk).distinct().order_by('-date')[0]

    # Setting various attributes through kwargs

    for i in ('pc95','units','logarithmic','label','description'):

        if i in kwargs:

            setattr(graph,i,kwargs[i])

    graph.save()

    for datasource in datasources:

        try:

            grds = GraphDataSource.objects.get(graph=graph,datasource=datasource)

        except GraphDataSource.DoesNotExist, e:

            grds = GraphDataSource(graph=graph,datasource=datasource)

        # Some defaults

        grds.args = ''

        grds.function = 'LINE1'

        grds.color = '#0000bbbb'

        # Hardcoded default. Max Datasources in a graph.

        for n in range(0,1024):

            for i in ( 'function', 'color', 'args', 'legend', 'stackorder' ):

                if datasource.name == 'ds%s' % n and 'ds%s%s' % ( n, i ) in kwargs:

                    setattr(grds,i,kwargs['ds%s%s' % ( n, i)])

        grds.save()

    try:

        f = open(rrdpath)

    except IOError:

        necreation_epoch = int(time.time())

        f = rrdtool.create(str(rrdpath), str("--start"), str(1349965501),  [str('DS:ds0:COUNTER:300:U:U'), str('DS:ds1:COUNTER:300:U:U')], str('RRA:AVERAGE:0.5:1:288'), str('RRA:AVERAGE:0.5:3:672'), str('RRA:AVERAGE:0.5:12:744'), str('RRA:AVERAGE:0.5:72:1460'))

# -*- coding: utf-8 -*- vim:encoding=utf-8:

# vim: tabstop=4:shiftwidth=4:softtabstop=4:expandtab

from __dbfuncs__ import *

from core.models import *

from django.conf import settings

import rrdtool

import time

import datetime

def diskio_genconfig(lun):

    devfilename = "lun_%s" % lun.pk

    seriesfilename = 'blocksiostats'

    target = "%s_%s" % (devfilename, seriesfilename)

    rrdpath = settings.RRDFILEDIR + '/%s.rrd' % target

    dbreg('diskio', rrdpath, lun, description='Disk IO' , label=target,

            ds0legend='Blocks Read', ds1legend='Blocks Write',

            ds0function='AREA', ds1function='LINE1',

            ds0color='#00bb00bb', ds1color='#0000bbbb',

            pc95=False, units='blocks', tags=['lun:%s'%lun.pk, 'ds0:blocksread', "ds1:blockswrite", "type:blocksiostats"])

def requests_genconfig(lun):

    devfilename = "lun_%s" % lun.pk

    seriesfilename = 'requeststats'

    target = "%s_%s" % (devfilename, seriesfilename)

    rrdpath = settings.RRDFILEDIR + '/%s.rrd' % target

    dbreg('requests', rrdpath, lun, description='Requests IO' , label=target,

            ds0legend='Read Requests', ds1legend='Write Requests',

            ds0function='AREA', ds1function='LINE1',

            ds0color='#00bb00bb', ds1color='#0000bbbb',

            pc95=False, units='reuests', tags=['lun:%s'%lun.pk, 'ds0:readrequests', "ds1:writerequests", "type:requeststats"])

# -*- coding: utf-8 -*- vim:encoding=utf-8:

# vim: tabstop=4:shiftwidth=4:softtabstop=4:expandtab

from django.core.management.base import BaseCommand, CommandError

from django.conf import settings

from core.models import *

from util.analyze import analyze

class Command(BaseCommand):

    def handle(self, *args, **options):

        luns = Lun.objects.all()

        analyze(luns)

        print "Done"

        return

# -*- coding: utf-8 -*- vim:encoding=utf-8:

# vim: tabstop=4:shiftwidth=4:softtabstop=4:expandtab

from django.core.management.base import BaseCommand, CommandError

from django.conf import settings

import sys

import os

class Command(BaseCommand):

#    help = "Creates the configuration for rg project. Will output mrtg, collectd and munin configuration files as well as populate accordingly the rg database."

#    args = "[[device]]"

#    label = "Domain of devices to be run for(e.g. example.com) and type of configuration run(or all if you wish everything). An optional device name can also be given to run the program for a single node. Valid types are" + ", ".join(settings.RG_CREATE_TYPES) 

    def handle(self, *args, **options):

        dir = os.listdir(settings.RRDFILEDIR)

        for fname in dir:

            os.remove("%s/%s"%(settings.RRDFILEDIR,fname))

            print "Removed %s"%fname

        print "done"

# -*- coding: utf-8 -*- vim:encoding=utf-8:

# vim: tabstop=4:shiftwidth=4:softtabstop=4:expandtab

from django.core.management.base import BaseCommand, CommandError

from django.conf import settings

from core.models import *

from datetime import datetime, timedelta

import sys

import os

from __genrgconfig__ import *

from __dbfuncs__ import *

class Command(BaseCommand):

    def handle(self, *args, **options):

        luns = Lun.objects.all()

        types = settings.LUN_GRAPH_TYPES

        for lun in luns:

            for type in types:

                self.dbupdate(lun, type)

            print 'Node: %s ...done' % lun.pk

    def dbupdate(self, device, type):

        # Ok. A couple of hacks here. Instead of a if then elif then elif.... for each type

        # we get first from the settings dictionnary the confdir and then the function used 

        # for this type from the globals() dictionary

        try:

#            dir = getattr(settings,'RG_%s_CONFDIR' % type.upper())

#            self.checkdir(dir)

            func = globals()['%s_genconfig' % type]

            func(device)

            print "Done with device"

        except AttributeError as e:

            print "%s is not going to be configured because %s" % (type, e)

            return

# -*- coding: utf-8 -*- vim:encoding=utf-8:

# vim: tabstop=4:shiftwidth=4:softtabstop=4:expandtab

from django.db import models

from django.conf import settings

from django.utils.encoding import iri_to_uri

from django.contrib.contenttypes.models import ContentType

from django.contrib.contenttypes import generic

from django.core.urlresolvers import reverse

from taggit.managers import TaggableManager

from util.rrd2json import rrd2json

import rrdtool, json

from core.models import Lun

from numpy import *

class RrdFile(models.Model):

    RRDPATH = settings.RRDFILEDIR

    path = models.FilePathField(path=RRDPATH,match='.*\.rrd$', recursive=True,max_length=255)

    def __unicode__(self):

        return 'RRD file: %s' % (self.path)

class DataSource(models.Model):

    name = models.CharField(max_length=30)

    rrdfile = models.ForeignKey(RrdFile)

    tags = TaggableManager()

    def __unicode__(self):

        return 'Data Source: %s, RRD file: %s' % (self.name,self.rrdfile.path)

    def get_plain_graphs(self):

        graphs = self.graph_set.filter(aggrfunction=None)

        return graphs

class Graph(models.Model):

    AGGREGATE_FUNCTIONS = (

        ( u'+', u'Addition' ),

        ( u'-', u'Substraction' ),

        ( u'*', u'Multiplication' ),

        ( u'/', u'Division' ),

        ( u'%', u'Modulo' ),

        ( u'ADDNAN', u'Nan-aware Addition'),

    )

    datasources = models.ManyToManyField(DataSource,through='GraphDataSource')

    date = models.DateTimeField(auto_now=True)

    type = models.CharField(max_length=35)

    label = models.CharField(max_length=60, null=True)

    units = models.CharField(max_length=30)

    description = models.CharField(max_length=255, null=True)

    pc95 = models.BooleanField(verbose_name="95th Percentile Creation",default=False)

    logarithmic = models.BooleanField(verbose_name="Logarithmic Y-Axis Scaling",default=False)

    aggrfunction = models.CharField(max_length=10, null=True, blank=True, choices=AGGREGATE_FUNCTIONS,

                                     verbose_name='Aggregate Function')

    aggrlabel = models.CharField(max_length=10, null=True,blank=True, verbose_name='Aggregate Function Label' )

    aggrgfunction = models.CharField(max_length=10, null=True, blank=True, verbose_name='Aggregate Graphing Function')

    aggrcolor = models.CharField(max_length=9, blank=True)

    tags = TaggableManager()

    class Meta:

        ordering=('label',)

    def __unicode__(self):

        return 'Tags: %s, type: %s, label: %s, description: %s' % (self.tags.all(), self.type,

                                                        self.label, self.description)

    def create_graph(self, path, **kwargs):

        BOTTOMLOGO = settings.RRD_BOTTOMLOGO

        start = kwargs['start']

        end = kwargs['end']

        lazy = kwargs['lazy']

        legend = kwargs['legend']

        args = list()

        tempgraphargs = list()

        args.append(path)

        if lazy == True:

            args.append('--lazy')

        legend = True

        if legend == False:

            args.append('--no-legend')

        args.append('-s %s ' % start )

        args.append('-e %s ' % end )

        args.append('-W %s' % ( BOTTOMLOGO ))

        tags = [t.name for t in self.tags.all() if 'graphdescription' in t.name]

        if len(tags) > 0:

            description = ",".join([tname.split(':')[1] for tname in tags])

            args.append('-t %s' % (description[:60]))

#         if self.description != None:

#             args.append('-t %s' % (self.description[:60]))

#         else:

#             args.append('-t %s ' % self.type)

        args.append('-v %s' % ( self.units ))

        args.append('--slope-mode')

        if 'thumb' in kwargs:

            args.append('-h 160')

            args.append('-w 360')

#             args.append('-j')

#        # Logarithmic care

#        if self.logarithmic == True:

#            args.append('--logarithmic')

#            args.append('--units=si')

#        else:

#            args.append('-l %s' % ( '0' ))

        dsnames=list()

        if 'thumb' not in kwargs:

            args.append('-h 180')

            args.append('-w 470')

        args.append("-c")

        args.append("FONT#333333")

        args.append("-c")

        args.append("AXIS#333333")

        args.append("-c")

        args.append("ARROW#3D2828")

        args.append("-n")

        args.append("DEFAULT:0:Arial")

        graphdatasources = self.graphdatasource_set.all()

        legends = [grds.legend for grds in graphdatasources]

        if self.aggrlabel != None:

            legends.append(self.aggrlabel)

        maxl = len(max(legends, key=len))

        if self.pc95:

            pc95cdef = 'CDEF:pc95bits='

        if self.aggrfunction:

            aggrfcdef= 'CDEF:aggrf='

        i=0

        for grds in graphdatasources:

            name = grds.datasource.name

            dsnames.append(name)

            # Internal variable for usage with rrdtool DEFs,CDEFs. The d at the start is dummy and is

            # used because rrdtool seems to want its CDEFs,DEFs starting with a letter in some cases

            grname = 'd%s' % grds.pk

            filename = grds.datasource.rrdfile.path

            args.append('DEF:%s=%s:%s:AVERAGE' % ( grname, filename, name ))

            if grds.graphable:

                padding = ' ' * (maxl - len(grds.legend))

                if grds.legend != '':

                    tempstack = '%s:%s%s%s:%s%s\:\g' % ( grds.function, grname, grds.args, grds.color, grds.legend, padding)

                else:

                    tempstack = '%s:%s%s%s:%s' % ( grds.function, grname, grds.args, grds.color, grds.legend)

                if grds.stackorder != 0:

                    tempstack = tempstack + ':STACK'

                tempgraphargs.append(tempstack)

                if grds.legend != '':

                    tempgraphargs.append('%s:%s%s:%s:%s' % ( 'GPRINT', grname, grds.args, 'MAX', '\tMax\: %4.2lf%s\g' ))

                    tempgraphargs.append('%s:%s%s:%s:%s' % ( 'GPRINT', grname, grds.args, 'MIN', '\tMin\: %4.2lf%s\g' ))

                    tempgraphargs.append('%s:%s%s:%s:%s' % ( 'GPRINT', grname, grds.args, 'AVERAGE', '\tAvg\: \t%4.2lf%s\g' ))

                    tempgraphargs.append('%s:%s%s:%s:%s' % ( 'GPRINT', grname, grds.args, 'LAST', '\tLast\: %4.2lf%s\\n' ))

            if self.pc95:

                pc95cdef = pc95cdef + grname  + grds.args + ','

            if self.aggrfunction:

                aggrfcdef = '%s%s%s,' % ( aggrfcdef, grname, grds.args )

                if i > 0:

                    aggrfcdef = aggrfcdef + self.aggrfunction + ','

            i = i + 1

        if self.pc95:

            pc95cdef = pc95cdef + 'MAX'

            args.append(pc95cdef)

            args.append('VDEF:pc95=pc95bits,95,PERCENT')

        if not self.aggrfunction:

            args.extend(tempgraphargs)

        if self.aggrfunction:

            padding = ' ' * (maxl - len(self.aggrlabel))

            args.append('%s:aggrf%s:%s%s\:\g' % (self.aggrgfunction, self.aggrcolor, self.aggrlabel, padding))

            args.append('GPRINT:aggrf:MAX:\tMax\: %4.2lf%s\g')

            args.append('GPRINT:aggrf:MIN:\tMin\: %4.2lf%s\g')

            args.append('GPRINT:aggrf:AVERAGE:\tAvg\: %4.2lf%s\g')

            args.append('GPRINT:aggrf:LAST:\tLast\: %4.2lf%s\\n')

        if self.pc95:

            args.append('COMMENT: \\n')

            args.append('HRULE:pc95#ff0000a0:95th Percentile\::dashes')

            args.append('GPRINT:pc95:%4.2lf%s\\n')

        # This is for rrdtool which hates unicode

        args=[str(val) for val in args]

        try:

            return rrdtool.graphv(*args)

        except Exception as e:

            print e

            return None

    def analyze_graph(self, start="-2d", end='-10m', extended=False):

        graphdatasources = self.graphdatasource_set.all()

        for grds in graphdatasources:

            filename = grds.datasource.rrdfile.path

            graph_data = rrdtool.fetch(str('%s'%filename), str('AVERAGE'), str('-s'), str("%s"%start), str('-e'), str('%s'%end))

            json_ret = []

            tojsondict = rrd2json(graph_data)

            graphdss = self.datasources.all()

            if tojsondict:

                for k,v in tojsondict.items():

                    tojson = {}

                    na = array(map(lambda x:x[1], v))

                    tojson['label']= ":".join([t.name for t in graphdss.get(name=k).tags.all()])

                    if extended:

                        tojson['data'] = v

                    tojson['graph'] = self.pk

                    tojson['lunpk'] = Lun.objects.get(pk=int(self.tags.get(name__contains='lun').name.replace('lun:',''))).pk

                    #tojson['ds'] =  graphdss.get(name=k).tags.get(name__startswith='ds').name

                    #tojson['avg'] =  reduce(lambda x, y: x+y, map(lambda x:x[1], v)) / len(v)

                    tojson['avg'] =  na.mean()

                    tojson['dev'] =  na.std()

                    json_ret.append(tojson)

        jret = [sorted(json_ret, key=lambda k: k['dev'], reverse=True)[0]]

        return jret

    @models.permalink

    def get_absolute_url(self):

        return ('core.views.drawpng', [str(self.id)])

    @models.permalink

    def get_draw_url(self):

        return ('core.views.drawpng', [str(self.id)])

    @models.permalink

    def get_thumb_url(self):

        return ('core.views.thumb', [str(self.id)])

    @models.permalink

    def get_hour_url(self):

        return ('core.views.drawpng', [str(self.id),'-1h','-300'])

    @models.permalink

    def get_day_url(self):

        return ('core.views.drawpng', [str(self.id),'-2d','-30m'])

    @models.permalink

    def get_week_url(self):

        return ('core.views.drawpng', [str(self.id),'-1w','-30m'])

    @models.permalink

    def get_month_url(self):

        return ('core.views.drawpng', [str(self.id),'-1m','-30m'])

    @models.permalink

    def get_year_url(self):

        return ('core.views.drawpng', [str(self.id),'-1y','-30m'])

class GraphDataSource(models.Model):

    GRAPHFUNCTIONS = (

        ( u'PRINT', u'PRINT' ),

        ( u'GPRINT', u'GPRINT' ),

        ( u'VRULE', u'VRULE' ),

        ( u'HRULE', u'HRULE' ),

        ( u'LINE',  u'LINE' ),

        ( u'AREA', u'AREA' ),

        ( u'TICK', u'TICK' ),

        ( u'SHIFT', u'SHIFT' ),

        ( u'PRINT', u'PRINT' ),

        ( u'', u'' ),

    )

    graph = models.ForeignKey(Graph)

    datasource = models.ForeignKey(DataSource)

    graphable = models.BooleanField(default=True)

    legend = models.CharField(max_length=35, null=True, blank=True)

    function = models.CharField(max_length=5, null=True, choices=GRAPHFUNCTIONS )

    color = models.CharField(max_length=9, blank=True)

    args = models.CharField(max_length=255, null=True)

    stackorder = models.PositiveIntegerField(default=0)

    class Meta:

        ordering = ('stackorder',)

    def __unicode__(self):

        return 'Graph: %s, %s, Graphing Variables: %s:%s%s:%s' % (self.graph.__unicode__(), self.datasource.__unicode__(), self.function, self.args, self.color, self.legend)

class GraphLabel(models.Model):

    graph = models.ManyToManyField(Graph)

    key = models.CharField(max_length=30,db_index=True)

    data = models.CharField(max_length=255)

    def __unicode__(self):

        return 'Key: %s, data: %s' % (self.key,self.data)

"""

This file demonstrates two different styles of tests (one doctest and one

unittest). These will both pass when you run "manage.py test".

Replace these with more appropriate tests for your application.

"""

from django.test import TestCase

class SimpleTest(TestCase):

    def test_basic_addition(self):

        """

        Tests that 1 + 1 always equals 2.

        """

        self.failUnlessEqual(1 + 1, 2)

__test__ = {"doctest": """

Another way to test that 1 + 1 is equal to 2.

>>> 1 + 1 == 2

True

"""}

# Create your views here.

#!/usr/bin/python

from django.core.management import execute_manager

try:

    import settings # Assumed to be in the same directory.

except ImportError:

    import sys

    sys.stderr.write("Error: Can't find the file 'settings.py' in the directory containing %r. It appears you've customized things.\nYou'll have to run django-admin.py, passing it your settings module.\n(If the file settings.py does indeed exist, it's causing an ImportError somehow.)\n" % __file__)

    sys.exit(1)

if __name__ == "__main__":

    execute_manager(settings)

# Django settings for lunstats project.

DEBUG = True

TEMPLATE_DEBUG = DEBUG

import os

here = lambda x: os.path.join(os.path.abspath(os.path.dirname(__file__)), x)

ADMINS = (

    # ('Your Name', 'your_email@domain.com'),

)

MANAGERS = ADMINS

DATABASES = {

    'default': {

        'ENGINE': 'django.db.backends.', # Add 'postgresql_psycopg2', 'postgresql', 'mysql', 'sqlite3' or 'oracle'.

        'NAME': ''                      # Or path to database file if using sqlite3.

        'USER': '',                      # Not used with sqlite3.

        'PASSWORD': '',                  # Not used with sqlite3.

        'HOST': '',                      # Set to empty string for localhost. Not used with sqlite3.

        'PORT': '',                      # Set to empty string for default. Not used with sqlite3.

    }

}

# Hosts/domain names that are valid for this site; required if DEBUG is False

# See https://docs.djangoproject.com/en/{{ docs_version }}/ref/settings/#allowed-hosts

ALLOWED_HOSTS = []

# Local time zone for this installation. Choices can be found here:

# http://en.wikipedia.org/wiki/List_of_tz_zones_by_name

# although not all choices may be available on all operating systems.

# On Unix systems, a value of None will cause Django to use the same

# timezone as the operating system.

# If running in a Windows environment this must be set to the same as your

# system time zone.

TIME_ZONE = 'Europe/Athens'

# Language code for this installation. All choices can be found here:

# http://www.i18nguy.com/unicode/language-identifiers.html

LANGUAGE_CODE = 'en-us'

SITE_ID = 1

# If you set this to False, Django will make some optimizations so as not

# to load the internationalization machinery.

USE_I18N = True

# If you set this to False, Django will not format dates, numbers and

# calendars according to the current locale

USE_L10N = True

# Absolute path to the directory that holds media.

# Example: "/home/media/media.lawrence.com/"

MEDIA_ROOT = ''

# URL that handles the media served from MEDIA_ROOT. Make sure to use a

# trailing slash if there is a path component (optional in other cases).

# Examples: "http://media.lawrence.com", "http://example.com/media/"

MEDIA_URL = ''

STATIC_URL = ''

# URL prefix for admin media -- CSS, JavaScript and images. Make sure to use a

# trailing slash.

# Examples: "http://foo.com/media/", "/media/".

ADMIN_MEDIA_PREFIX = '/media/'

# Make this unique, and don't share it with anybody.

SECRET_KEY = '#8(lzp_r(u4r6(+_&do44v-)8zrnl=272tob+#vvd378s8%kit'

# List of callables that know how to import templates from various sources.

TEMPLATE_LOADERS = (

    'django.template.loaders.filesystem.Loader',

    'django.template.loaders.app_directories.Loader',

#     'django.template.loaders.eggs.Loader',

)

MIDDLEWARE_CLASSES = (

    'django.middleware.common.CommonMiddleware',

    'django.contrib.sessions.middleware.SessionMiddleware',

    'django.middleware.csrf.CsrfViewMiddleware',

    'django.contrib.auth.middleware.AuthenticationMiddleware',

    'django.contrib.messages.middleware.MessageMiddleware',

)

ROOT_URLCONF = 'lunstats.urls'

TEMPLATE_DIRS = (

    here('templates'),

    # Put strings here, like "/home/html/django_templates" or "C:/www/django/templates".

    # Always use forward slashes, even on Windows.

    # Don't forget to use absolute paths, not relative paths.

)

INSTALLED_APPS = (

    'django.contrib.auth',

    'django.contrib.contenttypes',

    'django.contrib.sessions',

    'django.contrib.sites',

    'django.contrib.messages',

    'core',

    'graphs',

    'taggit',

    'django_extensions',

    'south',

    # Uncomment the next line to enable the admin:

    'django.contrib.admin',

    # Uncomment the next line to enable admin documentation:

    # 'django.contrib.admindocs',

)

CACHE_BACKEND = 'memcached://127.0.0.1:11211/?timeout=50000'

RRDFILEDIR = here('rrds')

RRD_PNG_STATICPATH = here('rrd_pngs')

RRD_BOTTOMLOGO = u''

LUN_COUNTERS = (

    ( u'diskio', u'Disk IO' ),

    ( u'requests', u'Requests' ),

    )

LUN_COUNTER_SERIES = {

                      'diskio':['blocksread', 'blockswrite'], 

                      'requests':['writereq', 'readreq']

                      }

LUN_GRAPH_TYPES = [type[0] for type in LUN_COUNTERS]

.isotope,

.isotope .isotope-item {

  /* change duration value to whatever you like */

  -webkit-transition-duration: 0.8s;

     -moz-transition-duration: 0.8s;

      -ms-transition-duration: 0.8s;

       -o-transition-duration: 0.8s;

          transition-duration: 0.8s;

}

.isotope {

  -webkit-transition-property: height, width;

     -moz-transition-property: height, width;

      -ms-transition-property: height, width;

       -o-transition-property: height, width;