root / snf-cyclades-app / synnefo / admin / stats.py @ 19b2c29d
History | View | Annotate | Download (9.6 kB)
1 |
# Copyright 2013 GRNET S.A. All rights reserved.
|
---|---|
2 |
#
|
3 |
# Redistribution and use in source and binary forms, with or
|
4 |
# without modification, are permitted provided that the following
|
5 |
# conditions are met:
|
6 |
#
|
7 |
# 1. Redistributions of source code must retain the above
|
8 |
# copyright notice, this list of conditions and the following
|
9 |
# disclaimer.
|
10 |
#
|
11 |
# 2. Redistributions in binary form must reproduce the above
|
12 |
# copyright notice, this list of conditions and the following
|
13 |
# disclaimer in the documentation and/or other materials
|
14 |
# provided with the distribution.
|
15 |
#
|
16 |
# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS
|
17 |
# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
|
18 |
# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
|
19 |
# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR
|
20 |
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
|
21 |
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
|
22 |
# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF
|
23 |
# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
|
24 |
# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
|
25 |
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN
|
26 |
# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
|
27 |
# POSSIBILITY OF SUCH DAMAGE.
|
28 |
#
|
29 |
# The views and conclusions contained in the software and
|
30 |
# documentation are those of the authors and should not be
|
31 |
# interpreted as representing official policies, either expressed
|
32 |
# or implied, of GRNET S.A.
|
33 |
|
34 |
|
35 |
import itertools |
36 |
import operator |
37 |
import datetime |
38 |
|
39 |
from collections import defaultdict # , OrderedDict |
40 |
from copy import copy |
41 |
from django.conf import settings |
42 |
from django.db.models import Count, Sum |
43 |
|
44 |
from snf_django.lib.astakos import UserCache |
45 |
from synnefo.db.models import VirtualMachine, Network, Backend |
46 |
from synnefo.plankton.utils import image_backend |
47 |
from synnefo.logic import backend as backend_mod |
48 |
|
49 |
|
50 |
def get_cyclades_stats(backend=None, clusters=True, servers=True, |
51 |
resources=True, networks=True, images=True): |
52 |
stats = {"datetime": datetime.datetime.now().strftime("%c")} |
53 |
if clusters:
|
54 |
stats["clusters"] = get_cluster_stats(backend=backend)
|
55 |
if servers:
|
56 |
stats["servers"] = get_servers_stats(backend=backend)
|
57 |
if resources:
|
58 |
stats["resources"] = get_resources_stats(backend=backend)
|
59 |
if networks:
|
60 |
stats["networks"] = get_networks_stats()
|
61 |
if images:
|
62 |
stats["images"] = get_images_stats(backend=None) |
63 |
return stats
|
64 |
|
65 |
|
66 |
def get_cluster_stats(backend): |
67 |
total = Backend.objects.all() |
68 |
stats = {"total": total.count(),
|
69 |
"drained": total.filter(drained=True).count(), |
70 |
"offline": total.filter(offline=True).count()} |
71 |
return stats
|
72 |
|
73 |
|
74 |
def _get_total_servers(backend=None): |
75 |
total_servers = VirtualMachine.objects.all() |
76 |
if backend is not None: |
77 |
total_servers = total_servers.filter(backend=backend) |
78 |
return total_servers
|
79 |
|
80 |
|
81 |
def get_servers_stats(backend=None): |
82 |
total_servers = _get_total_servers(backend=backend) |
83 |
per_state = total_servers.values("operstate")\
|
84 |
.annotate(count=Count("operstate"))
|
85 |
stats = {"total": 0} |
86 |
[stats.setdefault(s[0], 0) for s in VirtualMachine.OPER_STATES] |
87 |
for x in per_state: |
88 |
stats[x["operstate"]] = x["count"] |
89 |
stats["total"] += x["count"] |
90 |
return stats
|
91 |
|
92 |
|
93 |
def get_resources_stats(backend=None): |
94 |
total_servers = _get_total_servers(backend=backend) |
95 |
active_servers = total_servers.filter(deleted=False)
|
96 |
|
97 |
allocated = {} |
98 |
server_count = {} |
99 |
for res in ["cpu", "ram", "disk", "disk_template"]: |
100 |
server_count[res] = {} |
101 |
allocated[res] = 0
|
102 |
val = "flavor__%s" % res
|
103 |
results = active_servers.values(val).annotate(count=Count(val)) |
104 |
for result in results: |
105 |
server_count[res][result[val]] = result["count"]
|
106 |
if res != "disk_template": |
107 |
prod = (result["count"] * int(result[val])) |
108 |
if res == "disk": |
109 |
prod = prod << 10
|
110 |
allocated[res] += prod |
111 |
|
112 |
resources_stats = get_backend_stats(backend=backend) |
113 |
for res in ["cpu", "ram", "disk", "disk_template"]: |
114 |
if res not in resources_stats: |
115 |
resources_stats[res] = {} |
116 |
resources_stats[res]["servers"] = server_count[res]
|
117 |
resources_stats[res]["allocated"] = allocated[res]
|
118 |
|
119 |
return resources_stats
|
120 |
|
121 |
|
122 |
def get_images_stats(backend=None): |
123 |
total_servers = _get_total_servers(backend=backend) |
124 |
active_servers = total_servers.filter(deleted=False)
|
125 |
|
126 |
active_servers_images = active_servers.values("imageid", "userid")\ |
127 |
.annotate(number=Count("imageid"))
|
128 |
|
129 |
image_cache = ImageCache() |
130 |
image_stats = defaultdict(int)
|
131 |
for result in active_servers_images: |
132 |
imageid = image_cache.get_image(result["imageid"], result["userid"]) |
133 |
image_stats[imageid] += result["number"]
|
134 |
return dict(image_stats) |
135 |
|
136 |
|
137 |
def get_networks_stats(): |
138 |
total_networks = Network.objects.all() |
139 |
stats = {"public_ips": get_ip_stats(),
|
140 |
"total": 0} |
141 |
per_state = total_networks.values("state")\
|
142 |
.annotate(count=Count("state"))
|
143 |
[stats.setdefault(s[0], 0) for s in Network.OPER_STATES] |
144 |
for x in per_state: |
145 |
stats[x["state"]] = x["count"] |
146 |
stats["total"] += x["count"] |
147 |
return stats
|
148 |
|
149 |
|
150 |
def group_by_resource(objects, resource): |
151 |
stats = {} |
152 |
key = operator.attrgetter("flavor."+resource)
|
153 |
grouped = itertools.groupby(sorted(objects, key=key), key)
|
154 |
for val, group in grouped: |
155 |
stats[val] = len(list(group)) |
156 |
return stats
|
157 |
|
158 |
|
159 |
def get_ip_stats(): |
160 |
total, free = 0, 0, |
161 |
for network in Network.objects.filter(public=True, deleted=False): |
162 |
try:
|
163 |
net_total, net_free = network.ip_count() |
164 |
except AttributeError: |
165 |
# TODO: Check that this works..
|
166 |
pool = network.get_pool(locked=False)
|
167 |
net_total = pool.pool_size |
168 |
net_free = pool.count_available() |
169 |
if not network.drained: |
170 |
total += net_total |
171 |
free += net_free |
172 |
return {"total": total, |
173 |
"free": free}
|
174 |
|
175 |
|
176 |
def get_backend_stats(backend=None): |
177 |
if backend is None: |
178 |
backends = Backend.objects.filter(offline=False)
|
179 |
else:
|
180 |
if backend.offline:
|
181 |
return {}
|
182 |
backends = [backend] |
183 |
[backend_mod.update_backend_resources(b) for b in backends] |
184 |
resources = {} |
185 |
for attr in ("dfree", "dtotal", "mfree", "mtotal", "ctotal"): |
186 |
resources[attr] = 0
|
187 |
for b in backends: |
188 |
resources[attr] += getattr(b, attr)
|
189 |
|
190 |
return {"disk": {"free": resources["dfree"], "total": resources["dtotal"]}, |
191 |
"ram": {"free": resources["mfree"], "total": resources["mtotal"]}, |
192 |
"cpu": {"free": resources["ctotal"], "total": resources["ctotal"]}, |
193 |
"disk_template": {"free": 0, "total": 0}} |
194 |
|
195 |
|
196 |
class ImageCache(object): |
197 |
def __init__(self): |
198 |
self.images = {}
|
199 |
usercache = UserCache(settings.ASTAKOS_AUTH_URL, |
200 |
settings.CYCLADES_SERVICE_TOKEN) |
201 |
self.system_user_uuid = \
|
202 |
usercache.get_uuid(settings.SYSTEM_IMAGES_OWNER) |
203 |
|
204 |
def get_image(self, imageid, userid): |
205 |
if not imageid in self.images: |
206 |
try:
|
207 |
with image_backend(userid) as ib: |
208 |
image = ib.get_image(imageid) |
209 |
properties = image.get("properties")
|
210 |
os = properties.get("os",
|
211 |
properties.get("osfamily", "unknown")) |
212 |
owner = image["owner"]
|
213 |
owner = "system" if image["owner"] == self.system_user_uuid\ |
214 |
else "user" |
215 |
self.images[imageid] = owner + ":" + os |
216 |
except Exception: |
217 |
self.images[imageid] = "unknown:unknown" |
218 |
|
219 |
return self.images[imageid] |
220 |
|
221 |
|
222 |
def get_public_stats(): |
223 |
# VirtualMachines
|
224 |
vm_objects = VirtualMachine.objects |
225 |
servers = vm_objects.values("deleted", "operstate")\ |
226 |
.annotate(count=Count("id"),
|
227 |
cpu=Sum("flavor__cpu"),
|
228 |
ram=Sum("flavor__ram"),
|
229 |
disk=Sum("flavor__disk"))
|
230 |
zero_stats = {"count": 0, "cpu": 0, "ram": 0, "disk": 0} |
231 |
server_stats = {} |
232 |
for state in VirtualMachine.RSAPI_STATE_FROM_OPER_STATE.values(): |
233 |
server_stats[state] = copy(zero_stats) |
234 |
|
235 |
for stats in servers: |
236 |
deleted = stats.pop("deleted")
|
237 |
operstate = stats.pop("operstate")
|
238 |
state = VirtualMachine.RSAPI_STATE_FROM_OPER_STATE.get(operstate) |
239 |
if deleted:
|
240 |
for key in zero_stats.keys(): |
241 |
server_stats["DELETED"][key] += stats.get(key, 0) |
242 |
elif state:
|
243 |
for key in zero_stats.keys(): |
244 |
server_stats[state][key] += stats.get(key, 0)
|
245 |
|
246 |
#Networks
|
247 |
net_objects = Network.objects |
248 |
networks = net_objects.values("deleted", "state")\ |
249 |
.annotate(count=Count("id"))
|
250 |
zero_stats = {"count": 0} |
251 |
network_stats = {} |
252 |
for state in Network.RSAPI_STATE_FROM_OPER_STATE.values(): |
253 |
network_stats[state] = copy(zero_stats) |
254 |
|
255 |
for stats in networks: |
256 |
deleted = stats.pop("deleted")
|
257 |
state = stats.pop("state")
|
258 |
state = Network.RSAPI_STATE_FROM_OPER_STATE.get(state) |
259 |
if deleted:
|
260 |
for key in zero_stats.keys(): |
261 |
network_stats["DELETED"][key] += stats.get(key, 0) |
262 |
elif state:
|
263 |
for key in zero_stats.keys(): |
264 |
network_stats[state][key] += stats.get(key, 0)
|
265 |
|
266 |
statistics = {"servers": server_stats,
|
267 |
"networks": network_stats}
|
268 |
return statistics
|