Statistics
| Branch: | Tag: | Revision:

root / snf-cyclades-app / synnefo / admin / stats.py @ 23808592

History | View | Annotate | Download (9.6 kB)

1
# Copyright 2013 GRNET S.A. All rights reserved.
2
#
3
# Redistribution and use in source and binary forms, with or
4
# without modification, are permitted provided that the following
5
# conditions are met:
6
#
7
#   1. Redistributions of source code must retain the above
8
#      copyright notice, this list of conditions and the following
9
#      disclaimer.
10
#
11
#   2. Redistributions in binary form must reproduce the above
12
#      copyright notice, this list of conditions and the following
13
#      disclaimer in the documentation and/or other materials
14
#      provided with the distribution.
15
#
16
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
17
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
18
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
19
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
20
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
23
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
24
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
26
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
27
# POSSIBILITY OF SUCH DAMAGE.
28
#
29
# The views and conclusions contained in the software and
30
# documentation are those of the authors and should not be
31
# interpreted as representing official policies, either expressed
32
# or implied, of GRNET S.A.
33

    
34

    
35
import itertools
36
import operator
37
import datetime
38

    
39
from collections import defaultdict  # , OrderedDict
40
from copy import copy
41
from django.conf import settings
42
from django.db.models import Count, Sum
43

    
44
from snf_django.lib.astakos import UserCache
45
from synnefo.db.models import VirtualMachine, Network, Backend
46
from synnefo.plankton.backend import PlanktonBackend
47
from synnefo.logic import backend as backend_mod
48

    
49

    
50
def get_cyclades_stats(backend=None, clusters=True, servers=True,
51
                       resources=True, networks=True, images=True):
52
    stats = {"datetime": datetime.datetime.now().strftime("%c")}
53
    if clusters:
54
        stats["clusters"] = get_cluster_stats(backend=backend)
55
    if servers:
56
        stats["servers"] = get_servers_stats(backend=backend)
57
    if resources:
58
        stats["resources"] = get_resources_stats(backend=backend)
59
    if networks:
60
        stats["networks"] = get_networks_stats()
61
    if images:
62
        stats["images"] = get_images_stats(backend=None)
63
    return stats
64

    
65

    
66
def get_cluster_stats(backend):
67
    total = Backend.objects.all()
68
    stats = {"total": total.count(),
69
             "drained": total.filter(drained=True).count(),
70
             "offline": total.filter(offline=True).count()}
71
    return stats
72

    
73

    
74
def _get_total_servers(backend=None):
75
    total_servers = VirtualMachine.objects.all()
76
    if backend is not None:
77
        total_servers = total_servers.filter(backend=backend)
78
    return total_servers
79

    
80

    
81
def get_servers_stats(backend=None):
82
    total_servers = _get_total_servers(backend=backend)
83
    per_state = total_servers.values("operstate")\
84
                             .annotate(count=Count("operstate"))
85
    stats = {"total": 0}
86
    [stats.setdefault(s[0], 0) for s in VirtualMachine.OPER_STATES]
87
    for x in per_state:
88
        stats[x["operstate"]] = x["count"]
89
        stats["total"] += x["count"]
90
    return stats
91

    
92

    
93
def get_resources_stats(backend=None):
94
    total_servers = _get_total_servers(backend=backend)
95
    active_servers = total_servers.filter(deleted=False)
96

    
97
    allocated = {}
98
    server_count = {}
99
    for res in ["cpu", "ram", "disk", "disk_template"]:
100
        server_count[res] = {}
101
        allocated[res] = 0
102
        val = "flavor__%s" % res
103
        results = active_servers.values(val).annotate(count=Count(val))
104
        for result in results:
105
            server_count[res][result[val]] = result["count"]
106
            if res != "disk_template":
107
                prod = (result["count"] * int(result[val]))
108
                if res == "disk":
109
                    prod = prod << 10
110
                allocated[res] += prod
111

    
112
    resources_stats = get_backend_stats(backend=backend)
113
    for res in ["cpu", "ram", "disk", "disk_template"]:
114
        if res not in resources_stats:
115
            resources_stats[res] = {}
116
        resources_stats[res]["servers"] = server_count[res]
117
        resources_stats[res]["allocated"] = allocated[res]
118

    
119
    return resources_stats
120

    
121

    
122
def get_images_stats(backend=None):
123
    total_servers = _get_total_servers(backend=backend)
124
    active_servers = total_servers.filter(deleted=False)
125

    
126
    active_servers_images = active_servers.values("imageid", "userid")\
127
                                          .annotate(number=Count("imageid"))
128

    
129
    image_cache = ImageCache()
130
    image_stats = defaultdict(int)
131
    for result in active_servers_images:
132
        imageid = image_cache.get_image(result["imageid"], result["userid"])
133
        image_stats[imageid] += result["number"]
134
    return dict(image_stats)
135

    
136

    
137
def get_networks_stats():
138
    total_networks = Network.objects.all()
139
    stats = {"public_ips": get_ip_stats(),
140
             "total": 0}
141
    per_state = total_networks.values("state")\
142
                              .annotate(count=Count("state"))
143
    [stats.setdefault(s[0], 0) for s in Network.OPER_STATES]
144
    for x in per_state:
145
        stats[x["state"]] = x["count"]
146
        stats["total"] += x["count"]
147
    return stats
148

    
149

    
150
def group_by_resource(objects, resource):
151
    stats = {}
152
    key = operator.attrgetter("flavor."+resource)
153
    grouped = itertools.groupby(sorted(objects, key=key), key)
154
    for val, group in grouped:
155
        stats[val] = len(list(group))
156
    return stats
157

    
158

    
159
def get_ip_stats():
160
    total, free = 0, 0,
161
    for network in Network.objects.filter(public=True, deleted=False):
162
        try:
163
            net_total, net_free = network.ip_count()
164
        except AttributeError:
165
            # TODO: Check that this works..
166
            pool = network.get_pool(locked=False)
167
            net_total = pool.pool_size
168
            net_free = pool.count_available()
169
        if not network.drained:
170
            total += net_total
171
            free += net_free
172
    return {"total": total,
173
            "free": free}
174

    
175

    
176
def get_backend_stats(backend=None):
177
    if backend is None:
178
        backends = Backend.objects.filter(offline=False)
179
    else:
180
        if backend.offline:
181
            return {}
182
        backends = [backend]
183
    [backend_mod.update_backend_resources(b) for b in backends]
184
    resources = {}
185
    for attr in ("dfree", "dtotal", "mfree", "mtotal", "ctotal"):
186
        resources[attr] = 0
187
        for b in backends:
188
            resources[attr] += getattr(b, attr)
189

    
190
    return {"disk": {"free": resources["dfree"], "total": resources["dtotal"]},
191
            "ram": {"free": resources["mfree"], "total": resources["mtotal"]},
192
            "cpu": {"free": resources["ctotal"], "total": resources["ctotal"]},
193
            "disk_template": {"free": 0, "total": 0}}
194

    
195

    
196
class ImageCache(object):
197
    def __init__(self):
198
        self.images = {}
199
        usercache = UserCache(settings.ASTAKOS_AUTH_URL,
200
                              settings.CYCLADES_SERVICE_TOKEN)
201
        self.system_user_uuid = \
202
            usercache.get_uuid(settings.SYSTEM_IMAGES_OWNER)
203

    
204
    def get_image(self, imageid, userid):
205
        if not imageid in self.images:
206
            try:
207
                with PlanktonBackend(userid) as ib:
208
                    image = ib.get_image(imageid)
209
                properties = image.get("properties")
210
                os = properties.get("os",
211
                                    properties.get("osfamily", "unknown"))
212
                owner = image["owner"]
213
                owner = "system" if image["owner"] == self.system_user_uuid\
214
                        else "user"
215
                self.images[imageid] = owner + ":" + os
216
            except Exception:
217
                self.images[imageid] = "unknown:unknown"
218

    
219
        return self.images[imageid]
220

    
221

    
222
def get_public_stats():
223
    # VirtualMachines
224
    vm_objects = VirtualMachine.objects
225
    servers = vm_objects.values("deleted", "operstate")\
226
                        .annotate(count=Count("id"),
227
                                  cpu=Sum("flavor__cpu"),
228
                                  ram=Sum("flavor__ram"),
229
                                  disk=Sum("flavor__disk"))
230
    zero_stats = {"count": 0, "cpu": 0, "ram": 0, "disk": 0}
231
    server_stats = {}
232
    for state in VirtualMachine.RSAPI_STATE_FROM_OPER_STATE.values():
233
        server_stats[state] = copy(zero_stats)
234

    
235
    for stats in servers:
236
        deleted = stats.pop("deleted")
237
        operstate = stats.pop("operstate")
238
        state = VirtualMachine.RSAPI_STATE_FROM_OPER_STATE.get(operstate)
239
        if deleted:
240
            for key in zero_stats.keys():
241
                server_stats["DELETED"][key] += stats.get(key, 0)
242
        elif state:
243
            for key in zero_stats.keys():
244
                server_stats[state][key] += stats.get(key, 0)
245

    
246
    #Networks
247
    net_objects = Network.objects
248
    networks = net_objects.values("deleted", "state")\
249
                          .annotate(count=Count("id"))
250
    zero_stats = {"count": 0}
251
    network_stats = {}
252
    for state in Network.RSAPI_STATE_FROM_OPER_STATE.values():
253
        network_stats[state] = copy(zero_stats)
254

    
255
    for stats in networks:
256
        deleted = stats.pop("deleted")
257
        state = stats.pop("state")
258
        state = Network.RSAPI_STATE_FROM_OPER_STATE.get(state)
259
        if deleted:
260
            for key in zero_stats.keys():
261
                network_stats["DELETED"][key] += stats.get(key, 0)
262
        elif state:
263
            for key in zero_stats.keys():
264
                network_stats[state][key] += stats.get(key, 0)
265

    
266
    statistics = {"servers": server_stats,
267
                  "networks": network_stats}
268
    return statistics