staLUN: EMC LUN statistics

# -*- coing: utf-8 -*- vim:encoding=utf-8:

# vim: tabstop=4:shiftwidth=4:softtabstop=4:expandtab

# Copyright 2013 Leonidas Poulopoulos

#

# Licensed under the Apache License, Version 2.0 (the "License");

# you may not use this file except in compliance with the License.

# You may obtain a copy of the License at

#

#    http://www.apache.org/licenses/LICENSE-2.0

#

# Unless required by applicable law or agreed to in writing, software

# distributed under the License is distributed on an "AS IS" BASIS,

# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

# See the License for the specific language governing permissions and

# limitations under the License.

from django.core.management.base import BaseCommand, CommandError

from django.conf import settings

from core.models import *

from graphs.models import *

import csv

import time

import json

from __genrgconfig__ import *

from __dbfuncs__ import *

# for file listing

from stat import S_ISREG, ST_CTIME, ST_MODE

import os, sys, time, datetime

class Command(BaseCommand):

    help = "Populates the lun series measurements from a csv file" \

            "It assumes  that the meassurement file is formatted like:\n" \

            "<unix_timestamp>\n"\

            "LUNID, METRIC_A, METRIC_B, METRIC_C, METRIC_D\n"\

            "1, 0, 2, 2, 1\n"\

            "2, 0, 0, 0, 0\n" \

            "...\n" \

            "N, 1, 2, 3, 1 \n" \

    args = ""

    label = ''

    def handle(self, *args, **options):

        for statsfile in self.get_files_listing():

            self.getgraph(statsfile)

        print "Finished!"

    def get_files_listing(self):

        # path to the directory (relative or absolute)

        dirpath = '/srv/emc-stats/incoming'

        # get all entries in the directory w/ stats

        entries = (os.path.join(dirpath, fn) for fn in os.listdir(dirpath))

        entries = ((os.stat(path), path) for path in entries)

        # leave only regular files, insert creation date

        entries = ((stat[ST_CTIME], path)

                   for stat, path in entries if S_ISREG(stat[ST_MODE]))

        #NOTE: on Windows `ST_CTIME` is a creation date 

        #  but on Unix it could be something else

        #NOTE: use `ST_MTIME` to sort by a modification date

        statfiles = []

        for cdate, path in sorted(entries):

            statfiles.append("%s/%s"%('/srv/emc-stats/incoming',os.path.basename(path)))

        return statfiles

    def getgraph(self, statsfile):

        # We assume that the following is received

        # 1380622204

        # LUN,Blocks Read,Blocks Write,Read Reqs,Write Reqs,

        # 10,7346,8149,5311,9510,

        # 20,4581,3879,2894,9419,

        f = open(statsfile)

        #print statsfile

        # get first line. This is the timestamp

        timestamp = "%s" %f.readline().replace('\n', '')

        reader = csv.reader(self.remove_last_char(f))

        timestamp = self.round_to_5(int(timestamp))

        out = {}

        luns = []

        for i,serieslist in enumerate(reader):

            #print serieslist

            lundict = {}

            if i == 0:

                #those are headers, ignore (for the time)

                continue

            lundict['lun_name'] = serieslist[0] 

            lundict['timestamp'] = timestamp

            lundict['blocksread'] = serieslist[1]

            lundict['blockswrite'] = serieslist[2]

            lundict['readreq'] = serieslist[3]

            lundict['writereq'] = serieslist[4]

            lun, created = Lun.objects.get_or_create(name="%s"%lundict['lun_name'])

            if created:

                types = settings.LUN_GRAPH_TYPES

                for type in types:

                    self.dbupdate(lun, type)

            lundict['lun_id'] = lun.pk

            luns.append(lundict)

        self.update_graphs(luns)

        f.close()

        print "Done with %s" %statsfile

    def update_graphs(self, lunslist):

        for lundict in lunslist:

            self.update_graph(lundict)

    def update_graph(self, lundict):

        # Try to automate once proof of concept is done

        #print "updating Lun %s"%(lundict['lun_name'])

        dss_block = DataSource.objects.values('rrdfile').filter(tags__name="lun:%s"%lundict['lun_id']).filter(tags__name="type:blocksiostats").distinct()

        # This should return a single element list

        if len(dss_block) != 1:

            print "ERROR: It appears that there are %s blocksiostats datasources for lun %s" %(len(dss_block), lundict['lun_name'])

        else:

            rrdfile = dss_block[0]['rrdfile']

            try:

                rrdtool.update(

                    str('%s'%(RrdFile.objects.get(pk=rrdfile).path)), 

                    str('--template'), str('ds0:ds1'),

                    str(lundict['timestamp']+":" +lundict['blocksread']+":"+lundict['blockswrite']))

            except Exception as e:

                #print e

                pass

        dss_req = DataSource.objects.values('rrdfile').filter(tags__name="lun:%s"%lundict['lun_id']).filter(tags__name="type:requeststats").distinct()  

        # This should return a single element list

        if len(dss_req) != 1:

            print "ERROR: It appears that there are %s requeststats datasources for lun %s" %(len(dss_req), lundict['lun_name'])

        else:

            rrdfile = dss_req[0]['rrdfile']

            try:

                rrdtool.update(

                    str('%s'%(RrdFile.objects.get(pk=rrdfile).path)), 

                    str('--template'), str('ds0:ds1'),

                    str(lundict['timestamp']+":" +lundict['readreq']+":"+lundict['writereq']))

            except Exception as e:

                #print e

                pass

    def round_to_5(self, timestamp):

        tm = datetime.datetime.fromtimestamp(timestamp)

        tm = tm - datetime.timedelta(minutes=tm.minute % 5,

                             seconds=tm.second,

                             microseconds=tm.microsecond)

        return tm.strftime("%s")

    def remove_last_char(self, fileobj):

        for line in fileobj:

            yield line.strip()[:-1]

    def dbupdate(self, device, type):

        # Ok. A couple of hacks here. Instead of a if then elif then elif.... for each type

        # we get first from the settings dictionnary the confdir and then the function used 

        # for this type from the globals() dictionary

        try:

#            dir = getattr(settings,'RG_%s_CONFDIR' % type.upper())

#            self.checkdir(dir)

            func = globals()['%s_genconfig' % type]

            func(device)

        except AttributeError as e:

            print "%s is not going to be configured because %s" % (type, e)

            return