From: Antony Chazapis Date: Thu, 13 Oct 2011 13:41:47 +0000 (+0300) Subject: Merge policy into node. Needs database reset, or the following commands: X-Git-Tag: pithos/v0.7.8.2~4 X-Git-Url: https://code.grnet.gr/git/pithos/commitdiff_plain/5e7485da33afbbd4274adb49335b1c29765d2d2f Merge policy into node. Needs database reset, or the following commands: alter table policy rename to pre_policy; create table if not exists policy ( node integer, key text, value text, primary key (node, key) foreign key (node) references nodes(node) on update cascade on delete cascade ); insert into policy select n.node, p.key, p.value from nodes n, pre_policy p where n.path = p.path; drop table pre_policy; Refs #1165 Refs #1167 --- diff --git a/docs/source/backends.rst b/docs/source/backends.rst index bf14110..5542e54 100644 --- a/docs/source/backends.rst +++ b/docs/source/backends.rst @@ -33,14 +33,6 @@ Node :members: :undoc-members: -Policy -~~~~~~ - -.. automodule:: pithos.backends.lib.sqlite.policy - :show-inheritance: - :members: - :undoc-members: - Permissions ~~~~~~~~~~~ diff --git a/pithos/backends/lib/sqlalchemy/__init__.py b/pithos/backends/lib/sqlalchemy/__init__.py index b880ba6..0c88a09 100644 --- a/pithos/backends/lib/sqlalchemy/__init__.py +++ b/pithos/backends/lib/sqlalchemy/__init__.py @@ -1,7 +1,39 @@ +# Copyright 2011 GRNET S.A. All rights reserved. +# +# Redistribution and use in source and binary forms, with or +# without modification, are permitted provided that the following +# conditions are met: +# +# 1. Redistributions of source code must retain the above +# copyright notice, this list of conditions and the following +# disclaimer. +# +# 2. Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials +# provided with the distribution. +# +# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS +# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF +# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED +# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN +# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# +# The views and conclusions contained in the software and +# documentation are those of the authors and should not be +# interpreted as representing official policies, either expressed +# or implied, of GRNET S.A. + from dbwrapper import DBWrapper from node import Node from permissions import Permissions -from policy import Policy -__all__ = ["DBWrapper", "Node", "Permissions", "Policy"] +__all__ = ["DBWrapper", "Node", "Permissions"] diff --git a/pithos/backends/lib/sqlalchemy/node.py b/pithos/backends/lib/sqlalchemy/node.py index d24a0b4..3ca5eef 100644 --- a/pithos/backends/lib/sqlalchemy/node.py +++ b/pithos/backends/lib/sqlalchemy/node.py @@ -119,6 +119,17 @@ class Node(DBWorker): # place an index on path Index('idx_nodes_path', self.nodes.c.path) + #create policy table + columns=[] + columns.append(Column('node', Integer, + ForeignKey('nodes.node', + ondelete='CASCADE', + onupdate='CASCADE'), + primary_key=True)) + columns.append(Column('key', String(255), primary_key=True)) + columns.append(Column('value', String(255))) + self.policies = Table('policy', metadata, *columns, mysql_engine='InnoDB') + #create statistics table columns=[] columns.append(Column('node', Integer, @@ -366,6 +377,28 @@ class Node(DBWorker): self.conn.execute(s).close() return True + def policy_get(self, node): + s = select([self.policies.c.key, self.policies.c.value], + self.policies.c.node==node) + r = self.conn.execute(s) + d = dict(r.fetchall()) + r.close() + return d + + def policy_set(self, node, policy): + #insert or replace + for k, v in policy.iteritems(): + s = self.policies.update().where(and_(self.policies.c.node == node, + self.policies.c.key == k)) + s = s.values(value = v) + rp = self.conn.execute(s) + rp.close() + if rp.rowcount == 0: + s = self.policies.insert() + values = {'node':node, 'key':k, 'value':v} + r = self.conn.execute(s, values) + r.close() + def statistics_get(self, node, cluster=0): """Return population, total size and last mtime for all versions under node that belong to the cluster. diff --git a/pithos/backends/lib/sqlalchemy/policy.py b/pithos/backends/lib/sqlalchemy/policy.py deleted file mode 100644 index f8d54d1..0000000 --- a/pithos/backends/lib/sqlalchemy/policy.py +++ /dev/null @@ -1,78 +0,0 @@ -# Copyright 2011 GRNET S.A. All rights reserved. -# -# Redistribution and use in source and binary forms, with or -# without modification, are permitted provided that the following -# conditions are met: -# -# 1. Redistributions of source code must retain the above -# copyright notice, this list of conditions and the following -# disclaimer. -# -# 2. Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following -# disclaimer in the documentation and/or other materials -# provided with the distribution. -# -# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS -# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF -# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED -# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT -# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN -# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -# POSSIBILITY OF SUCH DAMAGE. -# -# The views and conclusions contained in the software and -# documentation are those of the authors and should not be -# interpreted as representing official policies, either expressed -# or implied, of GRNET S.A. - -from sqlalchemy import Table, Column, String, MetaData -from sqlalchemy.sql import select -from sqlalchemy.sql import and_ -from dbworker import DBWorker - - -class Policy(DBWorker): - """Paths can be assigned key, value pairs, representing policy.""" - - def __init__(self, **params): - DBWorker.__init__(self, **params) - metadata = MetaData() - columns=[] - columns.append(Column('path', String(2048), index=True)) - columns.append(Column('key', String(255))) - columns.append(Column('value', String(255))) - self.policies = Table('policy', metadata, *columns, mysql_engine='InnoDB') - metadata.create_all(self.engine) - - def policy_set(self, path, policy): - #insert or replace - for k, v in policy.iteritems(): - s = self.policies.update().where(and_(self.policies.c.path == path, - self.policies.c.key == k)) - s = s.values(value = v) - rp = self.conn.execute(s) - rp.close() - if rp.rowcount == 0: - s = self.policies.insert() - values = {'path':path, 'key':k, 'value':v} - r = self.conn.execute(s, values) - r.close() - - def policy_unset(self, path): - s = self.policies.delete().where(self.policies.c.path==path) - r = self.conn.execute(s) - r.close() - - def policy_get(self, path): - s = select([self.policies.c.key, self.policies.c.value], - self.policies.c.path==self.policies.c.path==path) - r = self.conn.execute(s) - d = dict(r.fetchall()) - r.close() - return d diff --git a/pithos/backends/lib/sqlite/__init__.py b/pithos/backends/lib/sqlite/__init__.py index c1b5149..0c88a09 100644 --- a/pithos/backends/lib/sqlite/__init__.py +++ b/pithos/backends/lib/sqlite/__init__.py @@ -34,7 +34,6 @@ from dbwrapper import DBWrapper from node import Node from permissions import Permissions -from policy import Policy -__all__ = ["DBWrapper", "Node", "Permissions", "Policy"] +__all__ = ["DBWrapper", "Node", "Permissions"] diff --git a/pithos/backends/lib/sqlite/node.py b/pithos/backends/lib/sqlite/node.py index 0e5cfc0..1fe66c8 100644 --- a/pithos/backends/lib/sqlite/node.py +++ b/pithos/backends/lib/sqlite/node.py @@ -111,10 +111,20 @@ class Node(DBWorker): foreign key (parent) references nodes(node) on update cascade - on delete cascade )""") + on delete cascade ) """) execute(""" create unique index if not exists idx_nodes_path on nodes(path) """) + execute(""" create table if not exists policy + ( node integer, + key text, + value text, + primary key (node, key) + foreign key (node) + references nodes(node) + on update cascade + on delete cascade ) """) + execute(""" create table if not exists statistics ( node integer, population integer not null default 0, @@ -125,7 +135,7 @@ class Node(DBWorker): foreign key (node) references nodes(node) on update cascade - on delete cascade )""") + on delete cascade ) """) execute(""" create table if not exists versions ( serial integer primary key, @@ -323,6 +333,15 @@ class Node(DBWorker): self.execute(q, (node,)) return True + def policy_get(self, node): + q = "select key, value from policy where node = ?" + self.execute(q, (node,)) + return dict(self.fetchall()) + + def policy_set(self, node, policy): + q = "insert or replace into policy (node, key, value) values (?, ?, ?)" + self.executemany(q, ((node, k, v) for k, v in policy.iteritems())) + def statistics_get(self, node, cluster=0): """Return population, total size and last mtime for all versions under node that belong to the cluster. diff --git a/pithos/backends/lib/sqlite/policy.py b/pithos/backends/lib/sqlite/policy.py deleted file mode 100644 index 0f00ea6..0000000 --- a/pithos/backends/lib/sqlite/policy.py +++ /dev/null @@ -1,61 +0,0 @@ -# Copyright 2011 GRNET S.A. All rights reserved. -# -# Redistribution and use in source and binary forms, with or -# without modification, are permitted provided that the following -# conditions are met: -# -# 1. Redistributions of source code must retain the above -# copyright notice, this list of conditions and the following -# disclaimer. -# -# 2. Redistributions in binary form must reproduce the above -# copyright notice, this list of conditions and the following -# disclaimer in the documentation and/or other materials -# provided with the distribution. -# -# THIS SOFTWARE IS PROVIDED BY GRNET S.A. ``AS IS'' AND ANY EXPRESS -# OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED -# WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR -# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL GRNET S.A OR -# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF -# USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED -# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT -# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN -# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE -# POSSIBILITY OF SUCH DAMAGE. -# -# The views and conclusions contained in the software and -# documentation are those of the authors and should not be -# interpreted as representing official policies, either expressed -# or implied, of GRNET S.A. - -from dbworker import DBWorker - - -class Policy(DBWorker): - """Paths can be assigned key, value pairs, representing policy.""" - - def __init__(self, **params): - DBWorker.__init__(self, **params) - execute = self.execute - - execute(""" create table if not exists policy - ( path text, - key text, - value text, - primary key (path, key) ) """) - - def policy_set(self, path, policy): - q = "insert or replace into policy (path, key, value) values (?, ?, ?)" - self.executemany(q, ((path, k, v) for k, v in policy.iteritems())) - - def policy_unset(self, path): - q = "delete from policy where path = ?" - self.execute(q, (path,)) - - def policy_get(self, path): - q = "select key, value from policy where path = ?" - self.execute(q, (path,)) - return dict(self.fetchall()) diff --git a/pithos/backends/modular.py b/pithos/backends/modular.py index 6deeafc..3b574da 100644 --- a/pithos/backends/modular.py +++ b/pithos/backends/modular.py @@ -132,7 +132,6 @@ class ModularBackend(BaseBackend): self.permissions = self.mod.permissions.Permissions(**params) for x in ['READ', 'WRITE']: setattr(self, x, getattr(self.mod.permissions, x)) - self.policy = self.mod.policy.Policy(**params) self.node = self.mod.node.Node(**params) for x in ['ROOTNODE', 'SERIAL', 'HASH', 'SIZE', 'MTIME', 'MUSER', 'CLUSTER']: setattr(self, x, getattr(self.mod.node, x)) @@ -313,8 +312,8 @@ class ModularBackend(BaseBackend): if container not in self._allowed_containers(user, account): raise NotAllowedError return {} - path = self._lookup_container(account, container)[0] - return self.policy.policy_get(path) + path, node = self._lookup_container(account, container) + return self.node.policy_get(node) @backend_method def update_container_policy(self, user, account, container, policy, replace=False): @@ -323,13 +322,13 @@ class ModularBackend(BaseBackend): logger.debug("update_container_policy: %s %s %s %s", account, container, policy, replace) if user != account: raise NotAllowedError - path = self._lookup_container(account, container)[0] + path, node = self._lookup_container(account, container) self._check_policy(policy) if replace: for k, v in self.default_policy.iteritems(): if k not in policy: policy[k] = v - self.policy.policy_set(path, policy) + self.node.policy_set(node, policy) @backend_method def put_container(self, user, account, container, policy=None): @@ -347,11 +346,11 @@ class ModularBackend(BaseBackend): if policy: self._check_policy(policy) path = '/'.join((account, container)) - self._put_path(user, self._lookup_account(account, True)[1], path) + node = self._put_path(user, self._lookup_account(account, True)[1], path) for k, v in self.default_policy.iteritems(): if k not in policy: policy[k] = v - self.policy.policy_set(path, policy) + self.node.policy_set(node, policy) @backend_method def delete_container(self, user, account, container, until=None): @@ -372,7 +371,6 @@ class ModularBackend(BaseBackend): self.node.node_purge_children(node, inf, CLUSTER_HISTORY) self.node.node_purge_children(node, inf, CLUSTER_DELETED) self.node.node_remove(node) - self.policy.policy_unset(path) @backend_method def list_objects(self, user, account, container, prefix='', delimiter=None, marker=None, limit=10000, virtual=True, keys=[], shared=False, until=None):