Revision 2e2fc330
| b/snf-cyclades-app/synnefo/quotas/management/commands/reconcile-resources-cyclades.py | ||
|---|---|---|
| 35 | 35 |
from django.core.management.base import BaseCommand |
| 36 | 36 |
from optparse import make_option |
| 37 | 37 |
|
| 38 |
|
|
| 39 | 38 |
from synnefo import quotas |
| 40 |
from synnefo.quotas.util import (get_db_holdings, get_quotaholder_holdings, |
|
| 41 |
transform_quotas) |
|
| 39 |
from synnefo.quotas import util |
|
| 42 | 40 |
from snf_django.management.utils import pprint_table |
| 41 |
from snf_django.utils import reconcile |
|
| 43 | 42 |
|
| 44 | 43 |
|
| 45 | 44 |
class Command(BaseCommand): |
| ... | ... | |
| 53 | 52 |
make_option("--userid", dest="userid",
|
| 54 | 53 |
default=None, |
| 55 | 54 |
help="Reconcile resources only for this user"), |
| 55 |
make_option("--project",
|
|
| 56 |
help="Reconcile resources only for this project"), |
|
| 56 | 57 |
make_option("--fix", dest="fix",
|
| 57 | 58 |
default=False, |
| 58 | 59 |
action="store_true", |
| ... | ... | |
| 67 | 68 |
def handle(self, *args, **options): |
| 68 | 69 |
write = self.stderr.write |
| 69 | 70 |
userid = options['userid'] |
| 71 |
project = options["project"] |
|
| 70 | 72 |
|
| 71 | 73 |
# Get holdings from Cyclades DB |
| 72 |
db_holdings = get_db_holdings(userid) |
|
| 73 |
# Get holdings from QuotaHolder |
|
| 74 |
qh_holdings = get_quotaholder_holdings(userid) |
|
| 75 |
|
|
| 76 |
users = set(db_holdings.keys()) |
|
| 77 |
users.update(qh_holdings.keys()) |
|
| 78 |
# Remove 'None' user |
|
| 79 |
users.discard(None) |
|
| 80 |
|
|
| 81 |
if userid and userid not in users: |
|
| 82 |
write("User '%s' does not exist in Quotaholder!", userid)
|
|
| 83 |
return |
|
| 84 |
|
|
| 85 |
pending_exists = False |
|
| 86 |
unknown_user_exists = False |
|
| 87 |
unsynced = [] |
|
| 88 |
for user in users: |
|
| 89 |
db = db_holdings.get(user, {})
|
|
| 90 |
try: |
|
| 91 |
qh_all = qh_holdings[user] |
|
| 92 |
except KeyError: |
|
| 93 |
write("User '%s' does not exist in Quotaholder!\n" %
|
|
| 94 |
user) |
|
| 95 |
unknown_user_exists = True |
|
| 96 |
continue |
|
| 74 |
db_holdings = util.get_db_holdings(userid, project) |
|
| 75 |
db_project_holdings = util.get_db_project_holdings(project) |
|
| 97 | 76 |
|
| 98 |
# Assuming only one source |
|
| 99 |
qh = qh_all.get(quotas.DEFAULT_SOURCE, {})
|
|
| 100 |
qh = transform_quotas(qh) |
|
| 101 |
for resource in quotas.RESOURCES: |
|
| 102 |
db_value = db.pop(resource, 0) |
|
| 103 |
try: |
|
| 104 |
qh_value, _, qh_pending = qh[resource] |
|
| 105 |
except KeyError: |
|
| 106 |
write("Resource '%s' does not exist in Quotaholder"
|
|
| 107 |
" for user '%s'!\n" % (resource, user)) |
|
| 108 |
continue |
|
| 109 |
if qh_pending: |
|
| 110 |
write("Pending commission. User '%s', resource '%s'.\n" %
|
|
| 111 |
(user, resource)) |
|
| 112 |
pending_exists = True |
|
| 113 |
continue |
|
| 114 |
if db_value != qh_value: |
|
| 115 |
data = (user, resource, db_value, qh_value) |
|
| 116 |
unsynced.append(data) |
|
| 117 |
|
|
| 118 |
headers = ("User", "Resource", "Database", "Quotaholder")
|
|
| 77 |
# Get holdings from QuotaHolder |
|
| 78 |
qh_holdings = util.get_qh_users_holdings( |
|
| 79 |
[userid] if userid is not None else None) |
|
| 80 |
qh_project_holdings = util.get_qh_project_holdings( |
|
| 81 |
[project] if project is not None else None) |
|
| 82 |
|
|
| 83 |
unsynced_users, users_pending, users_unknown =\ |
|
| 84 |
reconcile.check_users(self.stderr, quotas.RESOURCES, |
|
| 85 |
db_holdings, qh_holdings) |
|
| 86 |
|
|
| 87 |
unsynced_projects, projects_pending, projects_unknown =\ |
|
| 88 |
reconcile.check_projects(self.stderr, quotas.RESOURCES, |
|
| 89 |
db_project_holdings, qh_project_holdings) |
|
| 90 |
pending_exists = users_pending or projects_pending |
|
| 91 |
unknown_exists = users_unknown or projects_unknown |
|
| 92 |
|
|
| 93 |
headers = ("Type", "Holder", "Source", "Resource",
|
|
| 94 |
"Database", "Quotaholder") |
|
| 95 |
unsynced = unsynced_users + unsynced_projects |
|
| 119 | 96 |
if unsynced: |
| 120 | 97 |
pprint_table(self.stdout, unsynced, headers) |
| 121 | 98 |
if options["fix"]: |
| 122 | 99 |
qh = quotas.Quotaholder.get() |
| 123 |
request = {}
|
|
| 124 |
request["force"] = options["force"]
|
|
| 125 |
request["auto_accept"] = True
|
|
| 126 |
request["name"] = \
|
|
| 127 |
("client: reconcile-resources-cyclades, time: %s"
|
|
| 128 |
% datetime.now())
|
|
| 129 |
request["provisions"] = map(create_provision, unsynced)
|
|
| 100 |
force = options["force"]
|
|
| 101 |
name = ("client: reconcile-resources-cyclades, time: %s"
|
|
| 102 |
% datetime.now())
|
|
| 103 |
user_provisions = reconcile.create_user_provisions(
|
|
| 104 |
unsynced_users)
|
|
| 105 |
project_provisions = reconcile.create_project_provisions(
|
|
| 106 |
unsynced_projects)
|
|
| 130 | 107 |
try: |
| 131 |
qh.issue_commission(request) |
|
| 108 |
qh.issue_commission_generic( |
|
| 109 |
user_provisions, project_provisions, |
|
| 110 |
name=name, force=force, |
|
| 111 |
auto_accept=True) |
|
| 132 | 112 |
except quotas.errors.QuotaLimit: |
| 133 | 113 |
write("Reconciling failed because a limit has been "
|
| 134 | 114 |
"reached. Use --force to ignore the check.\n") |
| ... | ... | |
| 138 | 118 |
if pending_exists: |
| 139 | 119 |
write("Found pending commissions. Run 'snf-manage"
|
| 140 | 120 |
" reconcile-commissions-cyclades'\n") |
| 141 |
elif not (unsynced or unknown_user_exists):
|
|
| 121 |
elif not (unsynced or unknown_exists): |
|
| 142 | 122 |
write("Everything in sync.\n")
|
| 143 |
|
|
| 144 |
|
|
| 145 |
def create_provision(provision_info): |
|
| 146 |
user, resource, db_value, qh_value = provision_info |
|
| 147 |
return {"holder": user,
|
|
| 148 |
"source": quotas.DEFAULT_SOURCE, |
|
| 149 |
"resource": resource, |
|
| 150 |
"quantity": db_value - qh_value} |
|
| b/snf-cyclades-app/synnefo/quotas/util.py | ||
|---|---|---|
| 35 | 35 |
|
| 36 | 36 |
from synnefo.db.models import VirtualMachine, Network, IPAddress |
| 37 | 37 |
from synnefo.quotas import Quotaholder |
| 38 |
from synnefo.util.keypath import set_path |
|
| 38 | 39 |
|
| 39 | 40 |
|
| 40 |
def get_db_holdings(user=None): |
|
| 41 |
MiB = 2 ** 20 |
|
| 42 |
GiB = 2 ** 30 |
|
| 43 |
|
|
| 44 |
|
|
| 45 |
def get_db_holdings(user=None, project=None): |
|
| 41 | 46 |
"""Get holdings from Cyclades DB.""" |
| 42 | 47 |
holdings = {}
|
| 43 | 48 |
|
| ... | ... | |
| 50 | 55 |
networks = networks.filter(userid=user) |
| 51 | 56 |
floating_ips = floating_ips.filter(userid=user) |
| 52 | 57 |
|
| 58 |
if project is not None: |
|
| 59 |
vms = vms.filter(project=project) |
|
| 60 |
networks = networks.filter(project=project) |
|
| 61 |
floating_ips = floating_ips.filter(project=project) |
|
| 62 |
|
|
| 53 | 63 |
# Get resources related with VMs |
| 54 |
vm_resources = vms.values("userid")\
|
|
| 55 |
.annotate(num=Count("id"),
|
|
| 56 |
total_ram=Sum("flavor__ram"),
|
|
| 57 |
total_cpu=Sum("flavor__cpu"),
|
|
| 58 |
disk=Sum("flavor__disk"))
|
|
| 59 |
vm_active_resources = \ |
|
| 60 |
vms.values("userid")\
|
|
| 61 |
.filter(Q(operstate="STARTED") | Q(operstate="BUILD") | |
|
| 62 |
Q(operstate="ERROR"))\ |
|
| 63 |
.annotate(ram=Sum("flavor__ram"),
|
|
| 64 |
cpu=Sum("flavor__cpu"))
|
|
| 64 |
vm_resources = vms.values("userid", "project")\
|
|
| 65 |
.annotate(num=Count("id"),
|
|
| 66 |
total_ram=Sum("flavor__ram"),
|
|
| 67 |
total_cpu=Sum("flavor__cpu"),
|
|
| 68 |
disk=Sum("flavor__disk"))
|
|
| 65 | 69 |
|
| 66 | 70 |
for vm_res in vm_resources.iterator(): |
| 67 | 71 |
user = vm_res['userid'] |
| 72 |
project = vm_res['project'] |
|
| 68 | 73 |
res = {"cyclades.vm": vm_res["num"],
|
| 69 | 74 |
"cyclades.total_cpu": vm_res["total_cpu"], |
| 70 |
"cyclades.disk": 1073741824 * vm_res["disk"], |
|
| 71 |
"cyclades.total_ram": 1048576 * vm_res["total_ram"]} |
|
| 72 |
holdings[user] = res |
|
| 75 |
"cyclades.disk": vm_res["disk"] * GiB, |
|
| 76 |
"cyclades.total_ram": vm_res["total_ram"] * MiB} |
|
| 77 |
set_path(holdings, [user, project], res, createpath=True) |
|
| 78 |
|
|
| 79 |
vm_active_resources = vms.values("userid", "project")\
|
|
| 80 |
.filter(Q(operstate="STARTED") | Q(operstate="BUILD") | |
|
| 81 |
Q(operstate="ERROR"))\ |
|
| 82 |
.annotate(ram=Sum("flavor__ram"),
|
|
| 83 |
cpu=Sum("flavor__cpu"))
|
|
| 73 | 84 |
|
| 74 | 85 |
for vm_res in vm_active_resources.iterator(): |
| 75 | 86 |
user = vm_res['userid'] |
| 76 |
holdings[user]["cyclades.cpu"] = vm_res["cpu"] |
|
| 77 |
holdings[user]["cyclades.ram"] = 1048576 * vm_res["ram"] |
|
| 87 |
project = vm_res['project'] |
|
| 88 |
set_path(holdings, [user, project, "cyclades.cpu"], vm_res["cpu"], |
|
| 89 |
createpath=True) |
|
| 90 |
set_path(holdings, [user, project, "cyclades.ram"], |
|
| 91 |
vm_res["ram"] * MiB, createpath=True) |
|
| 78 | 92 |
|
| 79 | 93 |
# Get resources related with networks |
| 80 |
net_resources = networks.values("userid")\
|
|
| 94 |
net_resources = networks.values("userid", "project")\
|
|
| 81 | 95 |
.annotate(num=Count("id"))
|
| 96 |
|
|
| 82 | 97 |
for net_res in net_resources.iterator(): |
| 83 | 98 |
user = net_res['userid'] |
| 84 |
holdings.setdefault(user, {})
|
|
| 85 |
holdings[user]["cyclades.network.private"] = net_res["num"] |
|
| 99 |
if user is None: |
|
| 100 |
continue |
|
| 101 |
project = net_res['project'] |
|
| 102 |
set_path(holdings, [user, project, "cyclades.network.private"], |
|
| 103 |
net_res["num"], createpath=True) |
|
| 86 | 104 |
|
| 87 |
floating_ips_resources = floating_ips.values("userid")\
|
|
| 105 |
floating_ips_resources = floating_ips.values("userid", "project")\
|
|
| 88 | 106 |
.annotate(num=Count("id"))
|
| 107 |
|
|
| 89 | 108 |
for floating_ip_res in floating_ips_resources.iterator(): |
| 90 | 109 |
user = floating_ip_res["userid"] |
| 91 |
holdings.setdefault(user, {})
|
|
| 92 |
holdings[user]["cyclades.floating_ip"] = floating_ip_res["num"] |
|
| 110 |
project = floating_ip_res["project"] |
|
| 111 |
set_path(holdings, [user, project, "cyclades.floating_ip"], |
|
| 112 |
floating_ip_res["num"], createpath=True) |
|
| 113 |
|
|
| 114 |
return holdings |
|
| 115 |
|
|
| 116 |
|
|
| 117 |
def get_db_project_holdings(project=None): |
|
| 118 |
"""Get holdings from Cyclades DB.""" |
|
| 119 |
holdings = {}
|
|
| 120 |
|
|
| 121 |
vms = VirtualMachine.objects.filter(deleted=False) |
|
| 122 |
networks = Network.objects.filter(deleted=False) |
|
| 123 |
floating_ips = IPAddress.objects.filter(deleted=False, floating_ip=True) |
|
| 124 |
|
|
| 125 |
if project is not None: |
|
| 126 |
vms = vms.filter(project=project) |
|
| 127 |
networks = networks.filter(project=project) |
|
| 128 |
floating_ips = floating_ips.filter(project=project) |
|
| 129 |
|
|
| 130 |
# Get resources related with VMs |
|
| 131 |
vm_resources = vms.values("project")\
|
|
| 132 |
.annotate(num=Count("id"),
|
|
| 133 |
total_ram=Sum("flavor__ram"),
|
|
| 134 |
total_cpu=Sum("flavor__cpu"),
|
|
| 135 |
disk=Sum("flavor__disk"))
|
|
| 136 |
|
|
| 137 |
for vm_res in vm_resources.iterator(): |
|
| 138 |
project = vm_res['project'] |
|
| 139 |
res = {"cyclades.vm": vm_res["num"],
|
|
| 140 |
"cyclades.total_cpu": vm_res["total_cpu"], |
|
| 141 |
"cyclades.disk": vm_res["disk"] * GiB, |
|
| 142 |
"cyclades.total_ram": vm_res["total_ram"] * MiB} |
|
| 143 |
set_path(holdings, [project], res, createpath=True) |
|
| 144 |
|
|
| 145 |
vm_active_resources = vms.values("project")\
|
|
| 146 |
.filter(Q(operstate="STARTED") | Q(operstate="BUILD") | |
|
| 147 |
Q(operstate="ERROR"))\ |
|
| 148 |
.annotate(ram=Sum("flavor__ram"),
|
|
| 149 |
cpu=Sum("flavor__cpu"))
|
|
| 150 |
|
|
| 151 |
for vm_res in vm_active_resources.iterator(): |
|
| 152 |
project = vm_res['project'] |
|
| 153 |
set_path(holdings, [project, "cyclades.cpu"], vm_res["cpu"], |
|
| 154 |
createpath=True) |
|
| 155 |
set_path(holdings, [project, "cyclades.ram"], |
|
| 156 |
vm_res["ram"] * MiB, createpath=True) |
|
| 157 |
|
|
| 158 |
# Get resources related with networks |
|
| 159 |
net_resources = networks.values("project").annotate(num=Count("id"))
|
|
| 160 |
|
|
| 161 |
for net_res in net_resources.iterator(): |
|
| 162 |
project = net_res['project'] |
|
| 163 |
if project is None: |
|
| 164 |
continue |
|
| 165 |
set_path(holdings, [project, "cyclades.network.private"], |
|
| 166 |
net_res["num"], createpath=True) |
|
| 167 |
|
|
| 168 |
floating_ips_resources = floating_ips.values("project")\
|
|
| 169 |
.annotate(num=Count("id"))
|
|
| 170 |
|
|
| 171 |
for floating_ip_res in floating_ips_resources.iterator(): |
|
| 172 |
project = floating_ip_res["project"] |
|
| 173 |
set_path(holdings, [project, "cyclades.floating_ip"], |
|
| 174 |
floating_ip_res["num"], createpath=True) |
|
| 93 | 175 |
|
| 94 | 176 |
return holdings |
| 95 | 177 |
|
Also available in: Unified diff