From 7ebd4904b977d29c97447b53fbd718bccfa39969 Mon Sep 17 00:00:00 2001 From: Ryan McNair Date: Sat, 30 Jan 2016 16:24:32 +0000 Subject: [PATCH] Split out NestedQuotas into a separate driver Fixes the following issues with NestedQuotas: * Requires conf setting change to use nested quota driver * Enforces default child quota value with volume creation * Disables the use of -1 to be set for child quotas * Adds an admin only API command which can be used to validate the current setup for nested quotas, and can update existing allocated quotas in the DB which have been incorrectly set by previous use of child limits with -1 There will be follow-up patches with the following improvements: * make -1 limits functional for child projects * cache the Keystone project heirarchies to improve efficiency Note: ideally validation of nested quotas would occur in the setup of the nested quota driver, but doing the validation requires a view of ALL projects present in Keystone, so unless we require Keystone change to allow "cinder" service user to be able to list/get projects, we need the admin-only API for validation that should be called by cloud-admin. DocImpact Change-Id: Ibbd6f47c370d8f10c08cba358574b55e3059dcd1 Closes-Bug: #1531502 Partial-Bug: #1537189 Related-Bug: #1535878 --- cinder/api/contrib/quotas.py | 222 +++--- cinder/exception.py | 5 + cinder/quota.py | 225 ++++-- cinder/quota_utils.py | 129 +++- cinder/tests/unit/api/contrib/test_quotas.py | 678 +++++++++++------- cinder/tests/unit/test_quota.py | 405 +++++++---- cinder/tests/unit/test_quota_utils.py | 110 +++ etc/cinder/policy.json | 1 + ...-nested-quota-driver-e9493f478d2b8be5.yaml | 12 + 9 files changed, 1231 insertions(+), 556 deletions(-) create mode 100644 cinder/tests/unit/test_quota_utils.py create mode 100644 releasenotes/notes/split-out-nested-quota-driver-e9493f478d2b8be5.yaml diff --git a/cinder/api/contrib/quotas.py b/cinder/api/contrib/quotas.py index b5e0bb954..574983e86 100644 --- a/cinder/api/contrib/quotas.py +++ b/cinder/api/contrib/quotas.py @@ -15,11 +15,6 @@ import webob -from keystoneclient.auth.identity.generic import token -from keystoneclient import client -from keystoneclient import exceptions -from keystoneclient import session - from cinder.api import extensions from cinder.api.openstack import wsgi from cinder.api import xmlutil @@ -28,6 +23,7 @@ from cinder.db.sqlalchemy import api as sqlalchemy_api from cinder import exception from cinder.i18n import _ from cinder import quota +from cinder import quota_utils from cinder import utils from oslo_config import cfg @@ -57,17 +53,6 @@ class QuotaTemplate(xmlutil.TemplateBuilder): class QuotaSetsController(wsgi.Controller): - class GenericProjectInfo(object): - - """Abstraction layer for Keystone V2 and V3 project objects""" - - def __init__(self, project_id, project_keystone_api_version, - project_parent_id=None, project_subtree=None): - self.id = project_id - self.keystone_api_version = project_keystone_api_version - self.parent_id = project_parent_id - self.subtree = project_subtree - def _format_quota_set(self, project_id, quota_set): """Convert the quota object to a result dict.""" @@ -75,20 +60,6 @@ class QuotaSetsController(wsgi.Controller): return dict(quota_set=quota_set) - def _keystone_client(self, context): - """Creates and returns an instance of a generic keystone client. - - :param context: The request context - :return: keystoneclient.client.Client object - """ - auth_plugin = token.Token( - auth_url=CONF.keystone_authtoken.auth_uri, - token=context.auth_token, - project_id=context.project_id) - client_session = session.Session(auth=auth_plugin) - return client.Client(auth_url=CONF.keystone_authtoken.auth_uri, - session=client_session) - def _validate_existing_resource(self, key, value, quota_values): if key == 'per_volume_gigabytes': return @@ -103,7 +74,11 @@ class QuotaSetsController(wsgi.Controller): limit = self.validate_integer(quota[key], key, min_value=-1, max_value=db.MAX_INT) - if parent_project_quotas: + # If a parent quota is unlimited (-1) no validation needs to happen + # for the amount of existing free quota + # TODO(mc_nair): will need to recurse up for nested quotas once + # -1 child project values are enabled + if parent_project_quotas and parent_project_quotas[key]['limit'] != -1: free_quota = (parent_project_quotas[key]['limit'] - parent_project_quotas[key]['in_use'] - parent_project_quotas[key]['reserved'] - @@ -112,15 +87,22 @@ class QuotaSetsController(wsgi.Controller): current = 0 if project_quotas.get(key): current = project_quotas[key]['limit'] + # -1 limit doesn't change free quota available in parent + if current == -1: + current = 0 + + # Add back the existing quota limit (if any is set) from the + # current free quota since it will be getting reset and is part + # of the parent's allocated value + free_quota += current - if limit - current > free_quota: + if limit > free_quota: msg = _("Free quota available is %s.") % free_quota raise webob.exc.HTTPBadRequest(explanation=msg) return limit - def _get_quotas(self, context, id, usages=False, parent_project_id=None): - values = QUOTAS.get_project_quotas(context, id, usages=usages, - parent_project_id=parent_project_id) + def _get_quotas(self, context, id, usages=False): + values = QUOTAS.get_project_quotas(context, id, usages=usages) if usages: return values @@ -199,27 +181,6 @@ class QuotaSetsController(wsgi.Controller): return True return False - def _get_project(self, context, id, subtree_as_ids=False): - """A Helper method to get the project hierarchy. - - Along with Hierachical Multitenancy in keystone API v3, projects can be - hierarchically organized. Therefore, we need to know the project - hierarchy, if any, in order to do quota operations properly. - """ - try: - keystone = self._keystone_client(context) - generic_project = self.GenericProjectInfo(id, keystone.version) - if keystone.version == 'v3': - project = keystone.projects.get(id, - subtree_as_ids=subtree_as_ids) - generic_project.parent_id = project.parent_id - generic_project.subtree = ( - project.subtree if subtree_as_ids else None) - except exceptions.NotFound: - msg = (_("Tenant ID: %s does not exist.") % id) - raise webob.exc.HTTPNotFound(explanation=msg) - return generic_project - @wsgi.serializers(xml=QuotaTemplate) def show(self, req, id): """Show quota for a particular tenant @@ -242,21 +203,16 @@ class QuotaSetsController(wsgi.Controller): else: usage = False - try: + if QUOTAS.using_nested_quotas(): # With hierarchical projects, only the admin of the current project # or the root project has privilege to perform quota show # operations. - target_project = self._get_project(context, target_project_id) - context_project = self._get_project(context, context.project_id, - subtree_as_ids=True) + target_project = quota_utils.get_project_hierarchy( + context, target_project_id) + context_project = quota_utils.get_project_hierarchy( + context, context.project_id, subtree_as_ids=True) self._authorize_show(context_project, target_project) - parent_project_id = target_project.parent_id - except exceptions.Forbidden: - # NOTE(e0ne): Keystone API v2 requires admin permissions for - # project_get method. We ignore Forbidden exception for - # non-admin users. - parent_project_id = None try: sqlalchemy_api.authorize_project_context(context, @@ -264,8 +220,7 @@ class QuotaSetsController(wsgi.Controller): except exception.NotAuthorized: raise webob.exc.HTTPForbidden() - quotas = self._get_quotas(context, target_project_id, usage, - parent_project_id=parent_project_id) + quotas = self._get_quotas(context, target_project_id, usage) return self._format_quota_set(target_project_id, quotas) @wsgi.serializers(xml=QuotaTemplate) @@ -311,22 +266,25 @@ class QuotaSetsController(wsgi.Controller): msg = _("Bad key(s) in quota set: %s") % ",".join(bad_keys) raise webob.exc.HTTPBadRequest(explanation=msg) - # Get the parent_id of the target project to verify whether we are - # dealing with hierarchical namespace or non-hierarchical namespace. - target_project = self._get_project(context, target_project_id) - parent_id = target_project.parent_id + # Saving off this value since we need to use it multiple times + use_nested_quotas = QUOTAS.using_nested_quotas() + if use_nested_quotas: + # Get the parent_id of the target project to verify whether we are + # dealing with hierarchical namespace or non-hierarchical namespace + target_project = quota_utils.get_project_hierarchy( + context, target_project_id) + parent_id = target_project.parent_id - if parent_id: - # Get the children of the project which the token is scoped to - # in order to know if the target_project is in its hierarchy. - context_project = self._get_project(context, - context.project_id, - subtree_as_ids=True) - self._authorize_update_or_delete(context_project, - target_project.id, - parent_id) - parent_project_quotas = QUOTAS.get_project_quotas( - context, parent_id) + if parent_id: + # Get the children of the project which the token is scoped to + # in order to know if the target_project is in its hierarchy. + context_project = quota_utils.get_project_hierarchy( + context, context.project_id, subtree_as_ids=True) + self._authorize_update_or_delete(context_project, + target_project.id, + parent_id) + parent_project_quotas = QUOTAS.get_project_quotas( + context, parent_id) # NOTE(ankit): Pass #2 - In this loop for body['quota_set'].keys(), # we validate the quota limits to ensure that we can bail out if @@ -344,10 +302,17 @@ class QuotaSetsController(wsgi.Controller): if not skip_flag: self._validate_existing_resource(key, value, quota_values) - if parent_id: + if use_nested_quotas and parent_id: value = self._validate_quota_limit(body['quota_set'], key, quota_values, parent_project_quotas) + + if value < 0: + # TODO(mc_nair): extend to handle -1 limits and recurse up + # the hierarchy + msg = _("Quota can't be set to -1 for child projects.") + raise webob.exc.HTTPBadRequest(explanation=msg) + original_quota = 0 if quota_values.get(key): original_quota = quota_values[key]['limit'] @@ -373,7 +338,7 @@ class QuotaSetsController(wsgi.Controller): # If hierarchical projects, update child's quota first # and then parents quota. In future this needs to be an # atomic operation. - if parent_id: + if use_nested_quotas and parent_id: if key in allocated_quotas.keys(): try: db.quota_allocated_update(context, parent_id, key, @@ -383,24 +348,15 @@ class QuotaSetsController(wsgi.Controller): db.quota_create(context, parent_id, key, parent_limit, allocated=allocated_quotas[key]) - return {'quota_set': self._get_quotas(context, target_project_id, - parent_project_id=parent_id)} + return {'quota_set': self._get_quotas(context, target_project_id)} @wsgi.serializers(xml=QuotaTemplate) def defaults(self, req, id): context = req.environ['cinder.context'] authorize_show(context) - try: - project = self._get_project(context, context.project_id) - parent_id = project.parent_id - except exceptions.Forbidden: - # NOTE(e0ne): Keystone API v2 requires admin permissions for - # project_get method. We ignore Forbidden exception for - # non-admin users. - parent_id = context.project_id return self._format_quota_set(id, QUOTAS.get_defaults( - context, parent_project_id=parent_id)) + context, project_id=id)) @wsgi.serializers(xml=QuotaTemplate) def delete(self, req, id): @@ -416,20 +372,30 @@ class QuotaSetsController(wsgi.Controller): context = req.environ['cinder.context'] authorize_delete(context) - # Get the parent_id of the target project to verify whether we are - # dealing with hierarchical namespace or non-hierarchical namespace. - target_project = self._get_project(context, id) - parent_id = target_project.parent_id + if QUOTAS.using_nested_quotas(): + self._delete_nested_quota(context, id) + else: + try: + db.quota_destroy_by_project(context, id) + except exception.AdminRequired: + raise webob.exc.HTTPForbidden() + def _delete_nested_quota(self, ctxt, proj_id): + # Get the parent_id of the target project to verify whether we are + # dealing with hierarchical namespace or non-hierarchical + # namespace. try: project_quotas = QUOTAS.get_project_quotas( - context, target_project.id, usages=True, - parent_project_id=parent_id, defaults=False) + ctxt, proj_id, usages=True, defaults=False) except exception.NotAuthorized: raise webob.exc.HTTPForbidden() - # If the project which is being deleted has allocated part of its quota - # to its subprojects, then subprojects' quotas should be deleted first. + target_project = quota_utils.get_project_hierarchy( + ctxt, proj_id) + parent_id = target_project.parent_id + # If the project which is being deleted has allocated part of its + # quota to its subprojects, then subprojects' quotas should be + # deleted first. for key, value in project_quotas.items(): if 'allocated' in project_quotas[key].keys(): if project_quotas[key]['allocated'] != 0: @@ -438,35 +404,48 @@ class QuotaSetsController(wsgi.Controller): raise webob.exc.HTTPBadRequest(explanation=msg) if parent_id: - # Get the children of the project which the token is scoped to in - # order to know if the target_project is in its hierarchy. - context_project = self._get_project(context, - context.project_id, - subtree_as_ids=True) + # Get the children of the project which the token is scoped to + # in order to know if the target_project is in its hierarchy. + context_project = quota_utils.get_project_hierarchy( + ctxt, ctxt.project_id, subtree_as_ids=True) self._authorize_update_or_delete(context_project, target_project.id, parent_id) parent_project_quotas = QUOTAS.get_project_quotas( - context, parent_id, parent_project_id=parent_id) + ctxt, parent_id) # Delete child quota first and later update parent's quota. try: - db.quota_destroy_by_project(context, target_project.id) + db.quota_destroy_by_project(ctxt, target_project.id) except exception.AdminRequired: raise webob.exc.HTTPForbidden() - # Update the allocated of the parent + # The parent "gives" quota to its child using the "allocated" value + # and since the child project is getting deleted, we should restore + # the child projects quota to the parent quota, but lowering it's + # allocated value for key, value in project_quotas.items(): project_hard_limit = project_quotas[key]['limit'] parent_allocated = parent_project_quotas[key]['allocated'] parent_allocated -= project_hard_limit - db.quota_allocated_update(context, parent_id, key, + db.quota_allocated_update(ctxt, parent_id, key, parent_allocated) - else: - try: - db.quota_destroy_by_project(context, target_project.id) - except exception.AdminRequired: - raise webob.exc.HTTPForbidden() + + def validate_setup_for_nested_quota_use(self, req): + """Validates that the setup supports using nested quotas. + + Ensures that Keystone v3 or greater is being used, and that the + existing quotas make sense to nest in the current hierarchy (e.g. that + no child quota would be larger than it's parent). + """ + ctxt = req.environ['cinder.context'] + params = req.params + try: + quota_utils.validate_setup_for_nested_quota_use( + ctxt, QUOTAS.resources, quota.NestedDbQuotaDriver(), + fix_allocated_quotas=params.get('fix_allocated_quotas')) + except exception.InvalidNestedQuotaSetup as e: + raise webob.exc.HTTPBadRequest(explanation=e.msg) class Quotas(extensions.ExtensionDescriptor): @@ -480,9 +459,10 @@ class Quotas(extensions.ExtensionDescriptor): def get_resources(self): resources = [] - res = extensions.ResourceExtension('os-quota-sets', - QuotaSetsController(), - member_actions={'defaults': 'GET'}) + res = extensions.ResourceExtension( + 'os-quota-sets', QuotaSetsController(), + member_actions={'defaults': 'GET'}, + collection_actions={'validate_setup_for_nested_quota_use': 'GET'}) resources.append(res) return resources diff --git a/cinder/exception.py b/cinder/exception.py index 1911a57e8..ce5cfe57d 100644 --- a/cinder/exception.py +++ b/cinder/exception.py @@ -376,6 +376,11 @@ class InvalidQuotaValue(Invalid): "resources: %(unders)s") +class InvalidNestedQuotaSetup(CinderException): + message = _("Project quotas are not properly setup for nested quotas: " + "%(reason)s.") + + class QuotaNotFound(NotFound): message = _("Quota could not be found") diff --git a/cinder/quota.py b/cinder/quota.py index f32931db2..bf5974049 100644 --- a/cinder/quota.py +++ b/cinder/quota.py @@ -16,7 +16,7 @@ """Quotas for volumes.""" - +from collections import deque import datetime from oslo_config import cfg @@ -30,6 +30,7 @@ from cinder import context from cinder import db from cinder import exception from cinder.i18n import _, _LE +from cinder import quota_utils LOG = logging.getLogger(__name__) @@ -65,7 +66,7 @@ quota_opts = [ default=0, help='Number of seconds between subsequent usage refreshes'), cfg.StrOpt('quota_driver', - default='cinder.quota.DbQuotaDriver', + default="cinder.quota.DbQuotaDriver", help='Default driver to use for quota checks'), cfg.BoolOpt('use_default_quota_class', default=True, @@ -97,18 +98,12 @@ class DbQuotaDriver(object): return db.quota_class_get(context, quota_class, resource_name) - def get_default(self, context, resource, parent_project_id=None): - """Get a specific default quota for a resource. - - :param parent_project_id: The id of the current project's parent, - if any. - """ - + def get_default(self, context, resource, project_id): + """Get a specific default quota for a resource.""" default_quotas = db.quota_class_get_default(context) - default_quota_value = 0 if parent_project_id else resource.default - return default_quotas.get(resource.name, default_quota_value) + return default_quotas.get(resource.name, resource.default) - def get_defaults(self, context, resources, parent_project_id=None): + def get_defaults(self, context, resources, project_id=None): """Given a list of resources, retrieve the default quotas. Use the class quotas named `_DEFAULT_QUOTA_NAME` as default quotas, @@ -116,13 +111,12 @@ class DbQuotaDriver(object): :param context: The request context, for access checks. :param resources: A dictionary of the registered resources. - :param parent_project_id: The id of the current project's parent, - if any. + :param project_id: The id of the current project """ quotas = {} default_quotas = {} - if CONF.use_default_quota_class and not parent_project_id: + if CONF.use_default_quota_class: default_quotas = db.quota_class_get_default(context) for resource in resources.values(): @@ -135,8 +129,7 @@ class DbQuotaDriver(object): "default quota class for default " "quota.") % {'res': resource.name}) quotas[resource.name] = default_quotas.get(resource.name, - (0 if parent_project_id - else resource.default)) + resource.default) return quotas def get_class_quotas(self, context, resources, quota_class, @@ -170,7 +163,7 @@ class DbQuotaDriver(object): def get_project_quotas(self, context, resources, project_id, quota_class=None, defaults=True, - usages=True, parent_project_id=None): + usages=True): """Retrieve quotas for a project. Given a list of resources, retrieve the quotas for the given @@ -190,12 +183,11 @@ class DbQuotaDriver(object): specific value for the resource. :param usages: If True, the current in_use, reserved and allocated counts will also be returned. - :param parent_project_id: The id of the current project's parent, - if any. """ quotas = {} project_quotas = db.quota_get_all_by_project(context, project_id) + allocated_quotas = None if usages: project_usages = db.quota_usage_get_all_by_project(context, project_id) @@ -214,8 +206,8 @@ class DbQuotaDriver(object): else: class_quotas = {} - default_quotas = self.get_defaults(context, resources, - parent_project_id=parent_project_id) + # TODO(mc_nair): change this to be lazy loaded + default_quotas = self.get_defaults(context, resources, project_id) for resource in resources.values(): # Omit default/quota class values @@ -237,15 +229,12 @@ class DbQuotaDriver(object): quotas[resource.name].update( in_use=usage.get('in_use', 0), reserved=usage.get('reserved', 0), ) - - if parent_project_id or allocated_quotas: - quotas[resource.name].update( - allocated=allocated_quotas.get(resource.name, 0), ) - + if allocated_quotas: + quotas[resource.name].update( + allocated=allocated_quotas.get(resource.name, 0), ) return quotas - def _get_quotas(self, context, resources, keys, has_sync, project_id=None, - parent_project_id=None): + def _get_quotas(self, context, resources, keys, has_sync, project_id=None): """A helper method which retrieves the quotas for specific resources. This specific resource is identified by keys, and which apply to the @@ -261,8 +250,6 @@ class DbQuotaDriver(object): :param project_id: Specify the project_id if current context is admin and admin wants to impact on common user's tenant. - :param parent_project_id: The id of the current project's parent, - if any. """ # Filter resources @@ -282,8 +269,7 @@ class DbQuotaDriver(object): # Grab and return the quotas (without usages) quotas = self.get_project_quotas(context, sub_resources, project_id, - context.quota_class, usages=False, - parent_project_id=parent_project_id) + context.quota_class, usages=False) return {k: v['limit'] for k, v in quotas.items()} @@ -452,6 +438,134 @@ class DbQuotaDriver(object): db.reservation_expire(context) +class NestedDbQuotaDriver(DbQuotaDriver): + def validate_nested_setup(self, ctxt, resources, project_tree, + fix_allocated_quotas=False): + """Ensures project_tree has quotas that make sense as nested quotas. + + Validates the following: + * No child projects have a limit of -1 + * No parent project has child_projects who have more combined quota + than the parent's quota limit + * No child quota has a larger in-use value than it's current limit + (could happen before because child default values weren't enforced) + * All parent projects' "allocated" quotas match the sum of the limits + of its children projects + """ + project_queue = deque(project_tree.items()) + borked_allocated_quotas = {} + + while project_queue: + # Tuple of (current root node, subtree) + cur_project_id, project_subtree = project_queue.popleft() + + # If we're on a leaf node, no need to do validation on it, and in + # order to avoid complication trying to get its children, skip it. + if not project_subtree: + continue + + cur_project_quotas = self.get_project_quotas( + ctxt, resources, cur_project_id) + + child_project_ids = project_subtree.keys() + child_project_quotas = {child_id: self.get_project_quotas( + ctxt, resources, child_id) for child_id in child_project_ids} + + # Validate each resource when compared to it's child quotas + for resource in cur_project_quotas.keys(): + child_limit_sum = 0 + for child_id, child_quota in child_project_quotas.items(): + child_limit = child_quota[resource]['limit'] + # Don't want to continue validation if -1 limit for child + # TODO(mc_nair) - remove when allowing -1 for subprojects + if child_limit < 0: + msg = _("Quota limit is -1 for child project " + "'%(proj)s' for resource '%(res)s'") % { + 'proj': child_id, 'res': resource + } + raise exception.InvalidNestedQuotaSetup(reason=msg) + # Handle the case that child default quotas weren't being + # properly enforced before + elif child_quota[resource].get('in_use', 0) > child_limit: + msg = _("Quota limit invalid for project '%(proj)s' " + "for resource '%(res)s': limit of %(limit)d " + "is less than in-use value of %(used)d") % { + 'proj': child_id, 'res': resource, + 'limit': child_limit, + 'used': child_quota[resource]['in_use'] + } + raise exception.InvalidNestedQuotaSetup(reason=msg) + + child_limit_sum += child_quota[resource]['limit'] + + parent_quota = cur_project_quotas[resource] + parent_limit = parent_quota['limit'] + parent_usage = parent_quota['in_use'] + parent_allocated = parent_quota.get('allocated', 0) + + if parent_limit > 0: + parent_free_quota = parent_limit - parent_usage + if parent_free_quota < child_limit_sum: + msg = _("Sum of child limits '%(sum)s' is greater " + "than free quota of '%(free)s' for project " + "'%(proj)s' for resource '%(res)s'. Please " + "lower the limit for one or more of the " + "following projects: '%(child_ids)s'") % { + 'sum': child_limit_sum, 'free': parent_free_quota, + 'proj': cur_project_id, 'res': resource, + 'child_ids': ', '.join(child_project_ids) + } + raise exception.InvalidNestedQuotaSetup(reason=msg) + + # Deal with the fact that using -1 limits in the past may + # have messed some allocated values in DB + if parent_allocated != child_limit_sum: + # Decide whether to fix the allocated val or just + # keep track of what's messed up + if fix_allocated_quotas: + try: + db.quota_allocated_update(ctxt, cur_project_id, + resource, + child_limit_sum) + except exception.ProjectQuotaNotFound: + # Handles the case that the project is using + # default quota value so nothing present to update + db.quota_create( + ctxt, cur_project_id, resource, + parent_limit, allocated=child_limit_sum) + else: + if cur_project_id not in borked_allocated_quotas: + borked_allocated_quotas[cur_project_id] = {} + + borked_allocated_quotas[cur_project_id][resource] = { + 'db_allocated_quota': parent_allocated, + 'expected_allocated_quota': child_limit_sum} + + project_queue.extend(project_subtree.items()) + + if borked_allocated_quotas: + msg = _("Invalid allocated quotas defined for the following " + "project quotas: %s") % borked_allocated_quotas + raise exception.InvalidNestedQuotaSetup(message=msg) + + def get_default(self, context, resource, project_id): + """Get a specific default quota for a resource.""" + resource = super(NestedDbQuotaDriver, self).get_default( + context, resource, project_id) + + return 0 if quota_utils.get_parent_project_id( + context, project_id) else resource.default + + def get_defaults(self, context, resources, project_id=None): + defaults = super(NestedDbQuotaDriver, self).get_defaults( + context, resources, project_id) + # All defaults are 0 for child project + if quota_utils.get_parent_project_id(context, project_id): + for key in defaults.keys(): + defaults[key] = 0 + return defaults + + class BaseResource(object): """Describe a single resource for quota checking.""" @@ -626,14 +740,31 @@ class QuotaEngine(object): def __init__(self, quota_driver_class=None): """Initialize a Quota object.""" - if not quota_driver_class: - quota_driver_class = CONF.quota_driver + self._resources = {} + self._quota_driver_class = quota_driver_class + self._driver_class = None + + @property + def _driver(self): + # Lazy load the driver so we give a chance for the config file to + # be read before grabbing the config for which QuotaDriver to use + if self._driver_class: + return self._driver_class - if isinstance(quota_driver_class, six.string_types): - quota_driver_class = importutils.import_object(quota_driver_class) + if not self._quota_driver_class: + # Grab the current driver class from CONF + self._quota_driver_class = CONF.quota_driver - self._resources = {} - self._driver = quota_driver_class + if isinstance(self._quota_driver_class, six.string_types): + self._quota_driver_class = importutils.import_object( + self._quota_driver_class) + + self._driver_class = self._quota_driver_class + return self._driver_class + + def using_nested_quotas(self): + """Returns true if nested quotas are being used""" + return isinstance(self._driver, NestedDbQuotaDriver) def __contains__(self, resource): return resource in self.resources @@ -669,16 +800,15 @@ class QuotaEngine(object): return self._driver.get_default(context, resource, parent_project_id=parent_project_id) - def get_defaults(self, context, parent_project_id=None): + def get_defaults(self, context, project_id=None): """Retrieve the default quotas. :param context: The request context, for access checks. - :param parent_project_id: The id of the current project's parent, - if any. + :param project_id: The id of the current project """ return self._driver.get_defaults(context, self.resources, - parent_project_id) + project_id) def get_class_quotas(self, context, quota_class, defaults=True): """Retrieve the quotas for the given quota class. @@ -695,7 +825,7 @@ class QuotaEngine(object): quota_class, defaults=defaults) def get_project_quotas(self, context, project_id, quota_class=None, - defaults=True, usages=True, parent_project_id=None): + defaults=True, usages=True): """Retrieve the quotas for the given project. :param context: The request context, for access checks. @@ -709,17 +839,12 @@ class QuotaEngine(object): specific value for the resource. :param usages: If True, the current in_use, reserved and allocated counts will also be returned. - :param parent_project_id: The id of the current project's parent, - if any. """ - return self._driver.get_project_quotas(context, self.resources, project_id, quota_class=quota_class, defaults=defaults, - usages=usages, - parent_project_id= - parent_project_id) + usages=usages) def count(self, context, resource, *args, **kwargs): """Count a resource. diff --git a/cinder/quota_utils.py b/cinder/quota_utils.py index ca5c08175..89de09992 100644 --- a/cinder/quota_utils.py +++ b/cinder/quota_utils.py @@ -12,20 +12,42 @@ # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the # License for the specific language governing permissions and limitations # under the License. +import webob +from oslo_config import cfg from oslo_log import log as logging +from keystoneclient.auth.identity.generic import token +from keystoneclient import client +from keystoneclient import exceptions +from keystoneclient import session + from cinder import exception -from cinder.i18n import _LW -from cinder import quota +from cinder.i18n import _, _LW + +CONF = cfg.CONF LOG = logging.getLogger(__name__) -QUOTAS = quota.QUOTAS + + +class GenericProjectInfo(object): + """Abstraction layer for Keystone V2 and V3 project objects""" + def __init__(self, project_id, project_keystone_api_version, + project_parent_id=None, + project_subtree=None, + project_parent_tree=None): + self.id = project_id + self.keystone_api_version = project_keystone_api_version + self.parent_id = project_parent_id + self.subtree = project_subtree + self.parents = project_parent_tree def get_volume_type_reservation(ctxt, volume, type_id, reserve_vol_type_only=False): + from cinder import quota + QUOTAS = quota.QUOTAS # Reserve quotas for the given volume type try: reserve_opts = {'volumes': 1, 'gigabytes': volume['size']} @@ -78,3 +100,104 @@ def get_volume_type_reservation(ctxt, volume, type_id, raise exception.VolumeLimitExceeded( allowed=quotas[over]) return reservations + + +def get_project_hierarchy(context, project_id, subtree_as_ids=False): + """A Helper method to get the project hierarchy. + + Along with hierarchical multitenancy in keystone API v3, projects can be + hierarchically organized. Therefore, we need to know the project + hierarchy, if any, in order to do nested quota operations properly. + """ + try: + keystone = _keystone_client(context) + generic_project = GenericProjectInfo(project_id, keystone.version) + if keystone.version == 'v3': + project = keystone.projects.get(project_id, + subtree_as_ids=subtree_as_ids) + generic_project.parent_id = project.parent_id + generic_project.subtree = ( + project.subtree if subtree_as_ids else None) + except exceptions.NotFound: + msg = (_("Tenant ID: %s does not exist.") % project_id) + raise webob.exc.HTTPNotFound(explanation=msg) + + return generic_project + + +def get_parent_project_id(context, project_id): + return get_project_hierarchy(context, project_id).parent_id + + +def get_all_projects(context): + # Right now this would have to be done as cloud admin with Keystone v3 + return _keystone_client(context, (3, 0)).projects.list() + + +def get_all_root_project_ids(context): + project_list = get_all_projects(context) + + # Find every project which does not have a parent, meaning it is the + # root of the tree + project_roots = [project.id for project in project_list + if not project.parent_id] + + return project_roots + + +def validate_setup_for_nested_quota_use(ctxt, resources, + nested_quota_driver, + fix_allocated_quotas=False): + """Validates the setup supports using nested quotas. + + Ensures that Keystone v3 or greater is being used, that the current + user is of the cloud admin role, and that the existing quotas make sense to + nest in the current hierarchy (e.g. that no child quota would be larger + than it's parent). + + :param resources: the quota resources to validate + :param nested_quota_driver: nested quota driver used to validate each tree + :param fix_allocated_quotas: if True, parent projects "allocated" total + will be calculated based on the existing child limits and the DB will + be updated. If False, an exception is raised reporting any parent + allocated quotas are currently incorrect. + """ + try: + project_roots = get_all_root_project_ids(ctxt) + + # Now that we've got the roots of each tree, validate the trees + # to ensure that each is setup logically for nested quotas + for root in project_roots: + root_proj = get_project_hierarchy(ctxt, root, + subtree_as_ids=True) + nested_quota_driver.validate_nested_setup( + ctxt, + resources, + {root_proj.id: root_proj.subtree}, + fix_allocated_quotas=fix_allocated_quotas + ) + except exceptions.VersionNotAvailable: + msg = _("Keystone version 3 or greater must be used to get nested " + "quota support.") + raise exception.CinderException(message=msg) + except exceptions.Forbidden: + msg = _("Must run this command as cloud admin using " + "a Keystone policy.json which allows cloud " + "admin to list and get any project.") + raise exception.CinderException(message=msg) + + +def _keystone_client(context, version=(3, 0)): + """Creates and returns an instance of a generic keystone client. + + :param context: The request context + :param version: version of Keystone to request + :return: keystoneclient.client.Client object + """ + auth_plugin = token.Token( + auth_url=CONF.keystone_authtoken.auth_uri, + token=context.auth_token, + project_id=context.project_id) + client_session = session.Session(auth=auth_plugin) + return client.Client(auth_url=CONF.keystone_authtoken.auth_uri, + session=client_session, version=version) diff --git a/cinder/tests/unit/api/contrib/test_quotas.py b/cinder/tests/unit/api/contrib/test_quotas.py index 4bcc43fba..91e7fa18f 100644 --- a/cinder/tests/unit/api/contrib/test_quotas.py +++ b/cinder/tests/unit/api/contrib/test_quotas.py @@ -29,10 +29,10 @@ import webob.exc from cinder.api.contrib import quotas from cinder import context from cinder import db +from cinder import quota from cinder import test from cinder.tests.unit import test_db_api -from keystoneclient import exceptions from keystonemiddleware import auth_token from oslo_config import cfg from oslo_config import fixture as config_fixture @@ -43,7 +43,7 @@ CONF = cfg.CONF def make_body(root=True, gigabytes=1000, snapshots=10, volumes=10, backups=10, backup_gigabytes=1000, - tenant_id='foo', per_volume_gigabytes=-1): + tenant_id='foo', per_volume_gigabytes=-1, is_child=False): resources = {'gigabytes': gigabytes, 'snapshots': snapshots, 'volumes': volumes, @@ -52,10 +52,18 @@ def make_body(root=True, gigabytes=1000, snapshots=10, 'per_volume_gigabytes': per_volume_gigabytes, } # need to consider preexisting volume types as well volume_types = db.volume_type_get_all(context.get_admin_context()) - for volume_type in volume_types: - resources['gigabytes_' + volume_type] = -1 - resources['snapshots_' + volume_type] = -1 - resources['volumes_' + volume_type] = -1 + + if not is_child: + for volume_type in volume_types: + resources['gigabytes_' + volume_type] = -1 + resources['snapshots_' + volume_type] = -1 + resources['volumes_' + volume_type] = -1 + elif per_volume_gigabytes < 0: + # In the case that we're dealing with a child project, we aren't + # allowing -1 limits for the time being, so hack this to some large + # enough value for the tests that it's essentially unlimited + # TODO(mc_nair): remove when -1 limits for child projects are allowed + resources['per_volume_gigabytes'] = 10000 if tenant_id: resources['id'] = tenant_id @@ -75,7 +83,7 @@ def make_subproject_body(root=True, gigabytes=0, snapshots=0, per_volume_gigabytes=per_volume_gigabytes) -class QuotaSetsControllerTest(test.TestCase): +class QuotaSetsControllerTestBase(test.TestCase): class FakeProject(object): @@ -85,15 +93,30 @@ class QuotaSetsControllerTest(test.TestCase): self.subtree = None def setUp(self): - super(QuotaSetsControllerTest, self).setUp() + super(QuotaSetsControllerTestBase, self).setUp() + self.controller = quotas.QuotaSetsController() self.req = mock.Mock() self.req.environ = {'cinder.context': context.get_admin_context()} self.req.environ['cinder.context'].is_admin = True + self.req.params = {} self._create_project_hierarchy() + get_patcher = mock.patch('cinder.quota_utils.get_project_hierarchy', + self._get_project) + get_patcher.start() + self.addCleanup(get_patcher.stop) + + def _list_projects(context): + return self.project_by_id.values() + + list_patcher = mock.patch('cinder.quota_utils.get_all_projects', + _list_projects) + list_patcher.start() + self.addCleanup(list_patcher.stop) + self.auth_url = 'http://localhost:5000' self.fixture = self.useFixture(config_fixture.Config(auth_token.CONF)) self.fixture.config(auth_uri=self.auth_url, group='keystone_authtoken') @@ -127,111 +150,24 @@ class QuotaSetsControllerTest(test.TestCase): def _get_project(self, context, id, subtree_as_ids=False): return self.project_by_id.get(id, self.FakeProject()) - @mock.patch('keystoneclient.client.Client') - @mock.patch('keystoneclient.session.Session') - def test_keystone_client_instantiation(self, ksclient_session, - ksclient_class): - context = self.req.environ['cinder.context'] - self.controller._keystone_client(context) - ksclient_class.assert_called_once_with(auth_url=self.auth_url, - session=ksclient_session()) - @mock.patch('keystoneclient.client.Client') - def test_get_project_keystoneclient_v2(self, ksclient_class): - context = self.req.environ['cinder.context'] - keystoneclient = ksclient_class.return_value - keystoneclient.version = 'v2.0' - expected_project = self.controller.GenericProjectInfo( - context.project_id, 'v2.0') - project = self.controller._get_project(context, context.project_id) - self.assertEqual(expected_project.__dict__, project.__dict__) - - @mock.patch('keystoneclient.client.Client') - def test_get_project_keystoneclient_v3(self, ksclient_class): - context = self.req.environ['cinder.context'] - keystoneclient = ksclient_class.return_value - keystoneclient.version = 'v3' - returned_project = self.FakeProject(context.project_id, 'bar') - del returned_project.subtree - keystoneclient.projects.get.return_value = returned_project - expected_project = self.controller.GenericProjectInfo( - context.project_id, 'v3', 'bar') - project = self.controller._get_project(context, context.project_id) - self.assertEqual(expected_project.__dict__, project.__dict__) - - @mock.patch('keystoneclient.client.Client') - def test_get_project_keystoneclient_v3_with_subtree(self, ksclient_class): - context = self.req.environ['cinder.context'] - keystoneclient = ksclient_class.return_value - keystoneclient.version = 'v3' - returned_project = self.FakeProject(context.project_id, 'bar') - subtree_dict = {'baz': {'quux': None}} - returned_project.subtree = subtree_dict - keystoneclient.projects.get.return_value = returned_project - expected_project = self.controller.GenericProjectInfo( - context.project_id, 'v3', 'bar', subtree_dict) - project = self.controller._get_project(context, context.project_id, - subtree_as_ids=True) - keystoneclient.projects.get.assert_called_once_with( - context.project_id, subtree_as_ids=True) - self.assertEqual(expected_project.__dict__, project.__dict__) +class QuotaSetsControllerTest(QuotaSetsControllerTestBase): + def setUp(self): + super(QuotaSetsControllerTest, self).setUp() + fixture = self.useFixture(config_fixture.Config(quota.CONF)) + fixture.config(quota_driver="cinder.quota.DbQuotaDriver") + quotas.QUOTAS = quota.VolumeTypeQuotaEngine() + self.controller = quotas.QuotaSetsController() def test_defaults(self): - self.controller._get_project = mock.Mock() - self.controller._get_project.side_effect = self._get_project result = self.controller.defaults(self.req, 'foo') self.assertDictMatch(make_body(), result) - def test_subproject_defaults(self): - self.controller._get_project = mock.Mock() - self.controller._get_project.side_effect = self._get_project - context = self.req.environ['cinder.context'] - context.project_id = self.B.id - result = self.controller.defaults(self.req, self.B.id) - expected = make_subproject_body(tenant_id=self.B.id) - self.assertDictMatch(expected, result) - def test_show(self): - self.controller._get_project = mock.Mock() - self.controller._get_project.side_effect = self._get_project result = self.controller.show(self.req, 'foo') self.assertDictMatch(make_body(), result) - def test_subproject_show(self): - self.controller._get_project = mock.Mock() - self.controller._get_project.side_effect = self._get_project - self.req.environ['cinder.context'].project_id = self.A.id - result = self.controller.show(self.req, self.B.id) - expected = make_subproject_body(tenant_id=self.B.id) - self.assertDictMatch(expected, result) - - def test_subproject_show_in_hierarchy(self): - self.controller._get_project = mock.Mock() - self.controller._get_project.side_effect = self._get_project - # An user scoped to a root project in an hierarchy can see its children - # quotas. - self.req.environ['cinder.context'].project_id = self.A.id - result = self.controller.show(self.req, self.D.id) - expected = make_subproject_body(tenant_id=self.D.id) - self.assertDictMatch(expected, result) - # An user scoped to a parent project can see its immediate children - # quotas. - self.req.environ['cinder.context'].project_id = self.B.id - result = self.controller.show(self.req, self.D.id) - expected = make_subproject_body(tenant_id=self.D.id) - self.assertDictMatch(expected, result) - - def test_subproject_show_target_project_equals_to_context_project(self): - self.controller._get_project = mock.Mock() - self.controller._get_project.side_effect = self._get_project - self.req.environ['cinder.context'].project_id = self.B.id - result = self.controller.show(self.req, self.B.id) - expected = make_subproject_body(tenant_id=self.B.id) - self.assertDictMatch(expected, result) - def test_show_not_authorized(self): - self.controller._get_project = mock.Mock() - self.controller._get_project.side_effect = self._get_project self.req.environ['cinder.context'].is_admin = False self.req.environ['cinder.context'].user_id = 'bad_user' self.req.environ['cinder.context'].project_id = 'bad_project' @@ -239,29 +175,14 @@ class QuotaSetsControllerTest(test.TestCase): self.req, 'foo') def test_show_non_admin_user(self): - self.controller._get_project = mock.Mock() - self.controller._get_project.side_effect = exceptions.Forbidden self.controller._get_quotas = mock.Mock(side_effect= self.controller._get_quotas) result = self.controller.show(self.req, 'foo') self.assertDictMatch(make_body(), result) self.controller._get_quotas.assert_called_with( - self.req.environ['cinder.context'], 'foo', False, - parent_project_id=None) - - def test_subproject_show_not_authorized(self): - self.controller._get_project = mock.Mock() - self.controller._get_project.side_effect = self._get_project - self.req.environ['cinder.context'].project_id = self.B.id - self.assertRaises(webob.exc.HTTPForbidden, self.controller.show, - self.req, self.C.id) - self.req.environ['cinder.context'].project_id = self.B.id - self.assertRaises(webob.exc.HTTPForbidden, self.controller.show, - self.req, self.A.id) + self.req.environ['cinder.context'], 'foo', False) def test_update(self): - self.controller._get_project = mock.Mock() - self.controller._get_project.side_effect = self._get_project body = make_body(gigabytes=2000, snapshots=15, volumes=5, backups=5, tenant_id=None) result = self.controller.update(self.req, 'foo', body) @@ -271,65 +192,9 @@ class QuotaSetsControllerTest(test.TestCase): result = self.controller.update(self.req, 'foo', body) self.assertDictMatch(body, result) - def test_update_subproject(self): - self.controller._get_project = mock.Mock() - self.controller._get_project.side_effect = self._get_project - # Update the project A quota. - self.req.environ['cinder.context'].project_id = self.A.id - body = make_body(gigabytes=2000, snapshots=15, - volumes=5, backups=5, tenant_id=None) - result = self.controller.update(self.req, self.A.id, body) - self.assertDictMatch(body, result) - # Update the quota of B to be equal to its parent quota - self.req.environ['cinder.context'].project_id = self.A.id - body = make_body(gigabytes=2000, snapshots=15, - volumes=5, backups=5, tenant_id=None) - result = self.controller.update(self.req, self.B.id, body) - self.assertDictMatch(body, result) - # Try to update the quota of C, it will not be allowed, since the - # project A doesn't have free quota available. - self.req.environ['cinder.context'].project_id = self.A.id - body = make_body(gigabytes=2000, snapshots=15, - volumes=5, backups=5, tenant_id=None) - self.assertRaises(webob.exc.HTTPBadRequest, self.controller.update, - self.req, self.C.id, body) - # Successfully update the quota of D. - self.req.environ['cinder.context'].project_id = self.A.id - body = make_body(gigabytes=1000, snapshots=7, - volumes=3, backups=3, tenant_id=None) - result = self.controller.update(self.req, self.D.id, body) - self.assertDictMatch(body, result) - # An admin of B can also update the quota of D, since D is its an - # immediate child. - self.req.environ['cinder.context'].project_id = self.B.id - body = make_body(gigabytes=1500, snapshots=10, - volumes=4, backups=4, tenant_id=None) - result = self.controller.update(self.req, self.D.id, body) - - def test_update_subproject_repetitive(self): - self.controller._get_project = mock.Mock() - self.controller._get_project.side_effect = self._get_project - # Update the project A volumes quota. - self.req.environ['cinder.context'].project_id = self.A.id - body = make_body(gigabytes=2000, snapshots=15, - volumes=10, backups=5, tenant_id=None) - result = self.controller.update(self.req, self.A.id, body) - self.assertDictMatch(body, result) - # Update the quota of B to be equal to its parent quota - # three times should be successful, the quota will not be - # allocated to 'allocated' value of parent project - for i in range(0, 3): - self.req.environ['cinder.context'].project_id = self.A.id - body = make_body(gigabytes=2000, snapshots=15, - volumes=10, backups=5, tenant_id=None) - result = self.controller.update(self.req, self.B.id, body) - self.assertDictMatch(body, result) - - def test_update_subproject_not_in_hierarchy(self): - self.controller._get_project = mock.Mock() - self.controller._get_project.side_effect = self._get_project - - # Create another project hierarchy + def test_update_subproject_not_in_hierarchy_non_nested(self): + # When not using nested quotas, the hierarchy should not be considered + # for an update E = self.FakeProject(id=uuid.uuid4().hex, parent_id=None) F = self.FakeProject(id=uuid.uuid4().hex, parent_id=E.id) E.subtree = {F.id: F.subtree} @@ -342,50 +207,19 @@ class QuotaSetsControllerTest(test.TestCase): volumes=5, backups=5, tenant_id=None) result = self.controller.update(self.req, self.A.id, body) self.assertDictMatch(body, result) - # Try to update the quota of F, it will not be allowed, since the - # project E doesn't belongs to the project hierarchy of A. + # Try to update the quota of F, it will be allowed even though + # project E doesn't belong to the project hierarchy of A, because + # we are NOT using the nested quota driver self.req.environ['cinder.context'].project_id = self.A.id body = make_body(gigabytes=2000, snapshots=15, volumes=5, backups=5, tenant_id=None) - self.assertRaises(webob.exc.HTTPForbidden, self.controller.update, - self.req, F.id, body) - - def test_update_subproject_with_not_root_context_project(self): - self.controller._get_project = mock.Mock() - self.controller._get_project.side_effect = self._get_project - # Update the project A quota. - self.req.environ['cinder.context'].project_id = self.A.id - body = make_body(gigabytes=2000, snapshots=15, - volumes=5, backups=5, tenant_id=None) - result = self.controller.update(self.req, self.A.id, body) - self.assertDictMatch(body, result) - # Try to update the quota of B, it will not be allowed, since the - # project in the context (B) is not a root project. - self.req.environ['cinder.context'].project_id = self.B.id - body = make_body(gigabytes=2000, snapshots=15, - volumes=5, backups=5, tenant_id=None) - self.assertRaises(webob.exc.HTTPForbidden, self.controller.update, - self.req, self.B.id, body) - - def test_update_subproject_quota_when_parent_has_default_quotas(self): - self.controller._get_project = mock.Mock() - self.controller._get_project.side_effect = self._get_project - # Since the quotas of the project A were not updated, it will have - # default quotas. - self.req.environ['cinder.context'].project_id = self.A.id - # Update the project B quota. - expected = make_body(gigabytes=1000, snapshots=10, - volumes=5, backups=5, tenant_id=None) - result = self.controller.update(self.req, self.B.id, expected) - self.assertDictMatch(expected, result) + self.controller.update(self.req, F.id, body) @mock.patch( 'cinder.api.openstack.wsgi.Controller.validate_string_length') @mock.patch( 'cinder.api.openstack.wsgi.Controller.validate_integer') def test_update_limit(self, mock_validate_integer, mock_validate): - self.controller._get_project = mock.Mock() - self.controller._get_project.side_effect = self._get_project mock_validate_integer.return_value = 10 body = {'quota_set': {'volumes': 10}} @@ -401,22 +235,16 @@ class QuotaSetsControllerTest(test.TestCase): self.req, 'foo', body) def test_update_invalid_value_key_value(self): - self.controller._get_project = mock.Mock() - self.controller._get_project.side_effect = self._get_project body = {'quota_set': {'gigabytes': "should_be_int"}} self.assertRaises(webob.exc.HTTPBadRequest, self.controller.update, self.req, 'foo', body) def test_update_invalid_type_key_value(self): - self.controller._get_project = mock.Mock() - self.controller._get_project.side_effect = self._get_project body = {'quota_set': {'gigabytes': None}} self.assertRaises(webob.exc.HTTPBadRequest, self.controller.update, self.req, 'foo', body) def test_update_multi_value_with_bad_data(self): - self.controller._get_project = mock.Mock() - self.controller._get_project.side_effect = self._get_project orig_quota = self.controller.show(self.req, 'foo') body = make_body(gigabytes=2000, snapshots=15, volumes="should_be_int", backups=5, tenant_id=None) @@ -427,8 +255,6 @@ class QuotaSetsControllerTest(test.TestCase): self.assertDictMatch(orig_quota, new_quota) def test_update_bad_quota_limit(self): - self.controller._get_project = mock.Mock() - self.controller._get_project.side_effect = self._get_project body = {'quota_set': {'gigabytes': -1000}} self.assertRaises(webob.exc.HTTPBadRequest, self.controller.update, self.req, 'foo', body) @@ -437,8 +263,6 @@ class QuotaSetsControllerTest(test.TestCase): self.req, 'foo', body) def test_update_no_admin(self): - self.controller._get_project = mock.Mock() - self.controller._get_project.side_effect = self._get_project self.req.environ['cinder.context'].is_admin = False self.req.environ['cinder.context'].project_id = 'foo' self.req.environ['cinder.context'].user_id = 'foo_user' @@ -468,8 +292,6 @@ class QuotaSetsControllerTest(test.TestCase): db.quota_usage_get_all_by_project(ctxt, 'foo')) def test_update_lower_than_existing_resources_when_skip_false(self): - self.controller._get_project = mock.Mock() - self.controller._get_project.side_effect = self._get_project self._commit_quota_reservation() body = {'quota_set': {'volumes': 0}, 'skip_validation': 'false'} @@ -481,8 +303,6 @@ class QuotaSetsControllerTest(test.TestCase): self.req, 'foo', body) def test_update_lower_than_existing_resources_when_skip_true(self): - self.controller._get_project = mock.Mock() - self.controller._get_project.side_effect = self._get_project self._commit_quota_reservation() body = {'quota_set': {'volumes': 0}, 'skip_validation': 'true'} @@ -491,8 +311,6 @@ class QuotaSetsControllerTest(test.TestCase): result['quota_set']['volumes']) def test_update_lower_than_existing_resources_without_skip_argument(self): - self.controller._get_project = mock.Mock() - self.controller._get_project.side_effect = self._get_project self._commit_quota_reservation() body = {'quota_set': {'volumes': 0}} result = self.controller.update(self.req, 'foo', body) @@ -500,8 +318,6 @@ class QuotaSetsControllerTest(test.TestCase): result['quota_set']['volumes']) def test_delete(self): - self.controller._get_project = mock.Mock() - self.controller._get_project.side_effect = self._get_project result_show = self.controller.show(self.req, 'foo') self.assertDictMatch(make_body(), result_show) @@ -516,9 +332,7 @@ class QuotaSetsControllerTest(test.TestCase): result_show_after = self.controller.show(self.req, 'foo') self.assertDictMatch(result_show, result_show_after) - def test_subproject_delete(self): - self.controller._get_project = mock.Mock() - self.controller._get_project.side_effect = self._get_project + def test_delete_with_allocated_quota_different_from_zero(self): self.req.environ['cinder.context'].project_id = self.A.id body = make_body(gigabytes=2000, snapshots=15, @@ -539,42 +353,350 @@ class QuotaSetsControllerTest(test.TestCase): result_show_after = self.controller.show(self.req, self.A.id) self.assertDictMatch(result_show, result_show_after) - def test_subproject_delete_not_considering_default_quotas(self): - """Test delete subprojects' quotas won't consider default quotas. + def test_delete_no_admin(self): + self.req.environ['cinder.context'].is_admin = False + self.assertRaises(webob.exc.HTTPForbidden, self.controller.delete, + self.req, 'foo') - Test plan: - - Update the volume quotas of project A - - Update the volume quotas of project B - - Delete the quotas of project B + def test_subproject_show_not_using_nested_quotas(self): + # Current roles say for non-nested quotas, an admin should be able to + # see anyones quota + self.req.environ['cinder.context'].project_id = self.B.id + self.controller.show(self.req, self.C.id) + self.controller.show(self.req, self.A.id) - Resources with default quotas aren't expected to be considered when - updating the allocated values of the parent project. Thus, the delete - operation should succeed. + +class QuotaSetControllerValidateNestedQuotaSetup(QuotaSetsControllerTestBase): + """Validates the setup before using NestedQuota driver. + + Test case validates flipping on NestedQuota driver after using the + non-nested quota driver for some time. + """ + + def _create_project_hierarchy(self): + """Sets an environment used for nested quotas tests. + + Create a project hierarchy such as follows: + +-----------------+ + | | + | A G E | + | / \ \ | + | B C F | + | / | + | D | + +-----------------+ """ - self.controller._get_project = mock.Mock() - self.controller._get_project.side_effect = self._get_project + super(QuotaSetControllerValidateNestedQuotaSetup, + self)._create_project_hierarchy() + # Project A, B, C, D are already defined by parent test class + self.E = self.FakeProject(id=uuid.uuid4().hex, parent_id=None) + self.F = self.FakeProject(id=uuid.uuid4().hex, parent_id=self.E.id) + self.G = self.FakeProject(id=uuid.uuid4().hex, parent_id=None) + + self.E.subtree = {self.F.id: self.F.subtree} + + self.project_by_id.update({self.E.id: self.E, self.F.id: self.F, + self.G.id: self.G}) + + def test_validate_nested_quotas_no_in_use_vols(self): + # Update the project A quota. self.req.environ['cinder.context'].project_id = self.A.id + quota = {'volumes': 5} + body = {'quota_set': quota} + self.controller.update(self.req, self.A.id, body) + + quota['volumes'] = 3 + self.controller.update(self.req, self.B.id, body) + # Allocated value for quota A is borked, because update was done + # without nested quota driver + self.assertRaises(webob.exc.HTTPBadRequest, + self.controller.validate_setup_for_nested_quota_use, + self.req) + + # Fix the allocated values in DB + self.req.params['fix_allocated_quotas'] = True + self.controller.validate_setup_for_nested_quota_use( + self.req) + + self.req.params['fix_allocated_quotas'] = False + # Ensure that we've properly fixed the allocated quotas + self.controller.validate_setup_for_nested_quota_use(self.req) + + # Over-allocate the quotas between children + self.controller.update(self.req, self.C.id, body) + + # This is we should fail because the child limits are too big + self.assertRaises(webob.exc.HTTPBadRequest, + self.controller.validate_setup_for_nested_quota_use, + self.req) + + quota['volumes'] = 1 + self.controller.update(self.req, self.C.id, body) + + # Make sure we're validating all hierarchy trees + self.req.environ['cinder.context'].project_id = self.E.id + quota['volumes'] = 1 + self.controller.update(self.req, self.E.id, body) + quota['volumes'] = 3 + self.controller.update(self.req, self.F.id, body) + + self.assertRaises( + webob.exc.HTTPBadRequest, + self.controller.validate_setup_for_nested_quota_use, + self.req) + + # Put quotas in a good state + quota['volumes'] = 1 + self.controller.update(self.req, self.F.id, body) + self.req.params['fix_allocated_quotas'] = True + self.controller.validate_setup_for_nested_quota_use(self.req) + + def _fake_quota_usage_get_all_by_project(self, context, project_id): + proj_vals = { + self.A.id: {'in_use': 1}, + self.B.id: {'in_use': 1}, + self.D.id: {'in_use': 0}, + self.C.id: {'in_use': 3}, + self.E.id: {'in_use': 0}, + self.F.id: {'in_use': 0}, + self.G.id: {'in_use': 0}, + } + return {'volumes': proj_vals[project_id]} + + @mock.patch('cinder.db.quota_usage_get_all_by_project') + def test_validate_nested_quotas_in_use_vols(self, mock_usage): + mock_usage.side_effect = self._fake_quota_usage_get_all_by_project - body = {'quota_set': {'volumes': 5}} + # Update the project A quota. + self.req.environ['cinder.context'].project_id = self.A.id + quota_limit = {'volumes': 7} + body = {'quota_set': quota_limit} + self.controller.update(self.req, self.A.id, body) + + quota_limit['volumes'] = 3 + self.controller.update(self.req, self.B.id, body) + + quota_limit['volumes'] = 3 + self.controller.update(self.req, self.C.id, body) + + self.req.params['fix_allocated_quotas'] = True + self.controller.validate_setup_for_nested_quota_use(self.req) + + quota_limit['volumes'] = 6 + self.controller.update(self.req, self.A.id, body) + + # Should fail because the one in_use volume of 'A' + self.assertRaises( + webob.exc.HTTPBadRequest, + self.controller.validate_setup_for_nested_quota_use, + self.req) + + @mock.patch('cinder.db.quota_usage_get_all_by_project') + def test_validate_nested_quotas_quota_borked(self, mock_usage): + mock_usage.side_effect = self._fake_quota_usage_get_all_by_project + + # Update the project A quota. + self.req.environ['cinder.context'].project_id = self.A.id + quota_limit = {'volumes': 7} + body = {'quota_set': quota_limit} + self.controller.update(self.req, self.A.id, body) + + # Other quotas would default to 0 but already have some limit being + # used + self.assertRaises( + webob.exc.HTTPBadRequest, + self.controller.validate_setup_for_nested_quota_use, + self.req) + + def test_validate_nested_quota_negative_limits(self): + # When we're validating, update the allocated values since we've + # been updating child limits + self.req.params['fix_allocated_quotas'] = True + self.controller.validate_setup_for_nested_quota_use(self.req) + # Update the project A quota. + self.req.environ['cinder.context'].project_id = self.A.id + quota_limit = {'volumes': -1} + body = {'quota_set': quota_limit} + self.controller.update(self.req, self.A.id, body) + + quota_limit['volumes'] = 4 + self.controller.update(self.req, self.B.id, body) + + self.controller.validate_setup_for_nested_quota_use(self.req) + + quota_limit['volumes'] = -1 + self.controller.update(self.req, self.F.id, body) + # Should not work because can't have a child with negative limits + self.assertRaises( + webob.exc.HTTPBadRequest, + self.controller.validate_setup_for_nested_quota_use, + self.req) + + +class QuotaSetsControllerNestedQuotasTest(QuotaSetsControllerTestBase): + def setUp(self): + super(QuotaSetsControllerNestedQuotasTest, self).setUp() + fixture = self.useFixture(config_fixture.Config(quota.CONF)) + fixture.config(quota_driver="cinder.quota.NestedDbQuotaDriver") + quotas.QUOTAS = quota.VolumeTypeQuotaEngine() + self.controller = quotas.QuotaSetsController() + + def test_subproject_defaults(self): + context = self.req.environ['cinder.context'] + context.project_id = self.B.id + result = self.controller.defaults(self.req, self.B.id) + expected = make_subproject_body(tenant_id=self.B.id) + self.assertDictMatch(expected, result) + + def test_subproject_show(self): + self.req.environ['cinder.context'].project_id = self.A.id + result = self.controller.show(self.req, self.B.id) + expected = make_subproject_body(tenant_id=self.B.id) + self.assertDictMatch(expected, result) + + def test_subproject_show_in_hierarchy(self): + # A user scoped to a root project in a hierarchy can see its children + # quotas. + self.req.environ['cinder.context'].project_id = self.A.id + result = self.controller.show(self.req, self.D.id) + expected = make_subproject_body(tenant_id=self.D.id) + self.assertDictMatch(expected, result) + # A user scoped to a parent project can see its immediate children + # quotas. + self.req.environ['cinder.context'].project_id = self.B.id + result = self.controller.show(self.req, self.D.id) + expected = make_subproject_body(tenant_id=self.D.id) + self.assertDictMatch(expected, result) + + def test_subproject_show_target_project_equals_to_context_project( + self): + self.req.environ['cinder.context'].project_id = self.B.id + result = self.controller.show(self.req, self.B.id) + expected = make_subproject_body(tenant_id=self.B.id) + self.assertDictMatch(expected, result) + + def test_subproject_show_not_authorized(self): + self.req.environ['cinder.context'].project_id = self.B.id + self.assertRaises(webob.exc.HTTPForbidden, self.controller.show, + self.req, self.C.id) + self.req.environ['cinder.context'].project_id = self.B.id + self.assertRaises(webob.exc.HTTPForbidden, self.controller.show, + self.req, self.A.id) + + def test_update_subproject_not_in_hierarchy(self): + + # Create another project hierarchy + E = self.FakeProject(id=uuid.uuid4().hex, parent_id=None) + F = self.FakeProject(id=uuid.uuid4().hex, parent_id=E.id) + E.subtree = {F.id: F.subtree} + self.project_by_id[E.id] = E + self.project_by_id[F.id] = F + + # Update the project A quota. + self.req.environ['cinder.context'].project_id = self.A.id + body = make_body(gigabytes=2000, snapshots=15, + volumes=5, backups=5, tenant_id=None) result = self.controller.update(self.req, self.A.id, body) - self.assertEqual(body['quota_set']['volumes'], - result['quota_set']['volumes']) + self.assertDictMatch(body, result) + # Try to update the quota of F, it will not be allowed, since the + # project E doesn't belongs to the project hierarchy of A. + self.req.environ['cinder.context'].project_id = self.A.id + body = make_body(gigabytes=2000, snapshots=15, + volumes=5, backups=5, tenant_id=None) + self.assertRaises(webob.exc.HTTPForbidden, + self.controller.update, self.req, F.id, body) - body = {'quota_set': {'volumes': 2}} + def test_update_subproject(self): + # Update the project A quota. + self.req.environ['cinder.context'].project_id = self.A.id + body = make_body(gigabytes=2000, snapshots=15, + volumes=5, backups=5, tenant_id=None) + result = self.controller.update(self.req, self.A.id, body) + self.assertDictMatch(body, result) + # Update the quota of B to be equal to its parent quota + self.req.environ['cinder.context'].project_id = self.A.id + body = make_body(gigabytes=2000, snapshots=15, + volumes=5, backups=5, tenant_id=None, is_child=True) result = self.controller.update(self.req, self.B.id, body) - self.assertEqual(body['quota_set']['volumes'], - result['quota_set']['volumes']) + self.assertDictMatch(body, result) + # Try to update the quota of C, it will not be allowed, since the + # project A doesn't have free quota available. + self.req.environ['cinder.context'].project_id = self.A.id + body = make_body(gigabytes=2000, snapshots=15, + volumes=5, backups=5, tenant_id=None, is_child=True) + self.assertRaises(webob.exc.HTTPBadRequest, self.controller.update, + self.req, self.C.id, body) + # Successfully update the quota of D. + self.req.environ['cinder.context'].project_id = self.A.id + body = make_body(gigabytes=1000, snapshots=7, + volumes=3, backups=3, tenant_id=None, is_child=True) + result = self.controller.update(self.req, self.D.id, body) + self.assertDictMatch(body, result) + # An admin of B can also update the quota of D, since D is its + # immediate child. + self.req.environ['cinder.context'].project_id = self.B.id + body = make_body(gigabytes=1500, snapshots=10, + volumes=4, backups=4, tenant_id=None, is_child=True) + self.controller.update(self.req, self.D.id, body) - self.controller.delete(self.req, self.B.id) + def test_update_subproject_negative_limit(self): + # Should not be able to set a negative limit for a child project (will + # require further fixes to allow for this) + self.req.environ['cinder.context'].project_id = self.A.id + body = make_body(volumes=-1, is_child=True) + self.assertRaises(webob.exc.HTTPBadRequest, + self.controller.update, self.req, self.B.id, body) - def test_delete_with_allocated_quota_different_from_zero(self): - self.controller._get_project = mock.Mock() - self.controller._get_project.side_effect = self._get_project + def test_update_subproject_repetitive(self): + # Update the project A volumes quota. + self.req.environ['cinder.context'].project_id = self.A.id + body = make_body(gigabytes=2000, snapshots=15, + volumes=10, backups=5, tenant_id=None) + result = self.controller.update(self.req, self.A.id, body) + self.assertDictMatch(body, result) + # Update the quota of B to be equal to its parent quota + # three times should be successful, the quota will not be + # allocated to 'allocated' value of parent project + for i in range(0, 3): + self.req.environ['cinder.context'].project_id = self.A.id + body = make_body(gigabytes=2000, snapshots=15, + volumes=10, backups=5, tenant_id=None, + is_child=True) + result = self.controller.update(self.req, self.B.id, body) + self.assertDictMatch(body, result) + + def test_update_subproject_with_not_root_context_project(self): + # Update the project A quota. + self.req.environ['cinder.context'].project_id = self.A.id + body = make_body(gigabytes=2000, snapshots=15, + volumes=5, backups=5, tenant_id=None) + result = self.controller.update(self.req, self.A.id, body) + self.assertDictMatch(body, result) + # Try to update the quota of B, it will not be allowed, since the + # project in the context (B) is not a root project. + self.req.environ['cinder.context'].project_id = self.B.id + body = make_body(gigabytes=2000, snapshots=15, + volumes=5, backups=5, tenant_id=None) + self.assertRaises(webob.exc.HTTPForbidden, self.controller.update, + self.req, self.B.id, body) + + def test_update_subproject_quota_when_parent_has_default_quotas(self): + # Since the quotas of the project A were not updated, it will have + # default quotas. + self.req.environ['cinder.context'].project_id = self.A.id + # Update the project B quota. + expected = make_body(gigabytes=1000, snapshots=10, + volumes=5, backups=5, tenant_id=None, + is_child=True) + result = self.controller.update(self.req, self.B.id, expected) + self.assertDictMatch(expected, result) + + def test_subproject_delete(self): self.req.environ['cinder.context'].project_id = self.A.id body = make_body(gigabytes=2000, snapshots=15, volumes=5, backups=5, - backup_gigabytes=1000, tenant_id=None) + backup_gigabytes=1000, tenant_id=None, is_child=True) result_update = self.controller.update(self.req, self.A.id, body) self.assertDictMatch(body, result_update) @@ -590,12 +712,46 @@ class QuotaSetsControllerTest(test.TestCase): result_show_after = self.controller.show(self.req, self.A.id) self.assertDictMatch(result_show, result_show_after) - def test_delete_no_admin(self): - self.controller._get_project = mock.Mock() - self.controller._get_project.side_effect = self._get_project - self.req.environ['cinder.context'].is_admin = False - self.assertRaises(webob.exc.HTTPForbidden, self.controller.delete, - self.req, 'foo') + def test_subproject_delete_not_considering_default_quotas(self): + """Test delete subprojects' quotas won't consider default quotas. + + Test plan: + - Update the volume quotas of project A + - Update the volume quotas of project B + - Delete the quotas of project B + + Resources with default quotas aren't expected to be considered when + updating the allocated values of the parent project. Thus, the delete + operation should succeed. + """ + self.req.environ['cinder.context'].project_id = self.A.id + + body = {'quota_set': {'volumes': 5}} + result = self.controller.update(self.req, self.A.id, body) + self.assertEqual(body['quota_set']['volumes'], + result['quota_set']['volumes']) + + body = {'quota_set': {'volumes': 2}} + result = self.controller.update(self.req, self.B.id, body) + self.assertEqual(body['quota_set']['volumes'], + result['quota_set']['volumes']) + + self.controller.delete(self.req, self.B.id) + + def test_subproject_delete_with_child_present(self): + # Update the project A quota. + self.req.environ['cinder.context'].project_id = self.A.id + body = make_body(volumes=5) + self.controller.update(self.req, self.A.id, body) + + # Allocate some of that quota to a child project + body = make_body(volumes=3, is_child=True) + self.controller.update(self.req, self.B.id, body) + + # Deleting 'A' should be disallowed since 'B' is using some of that + # quota + self.assertRaises(webob.exc.HTTPBadRequest, self.controller.delete, + self.req, self.A.id) class QuotaSerializerTest(test.TestCase): diff --git a/cinder/tests/unit/test_quota.py b/cinder/tests/unit/test_quota.py index 11f792d44..d90f10235 100644 --- a/cinder/tests/unit/test_quota.py +++ b/cinder/tests/unit/test_quota.py @@ -20,6 +20,7 @@ import datetime import mock from oslo_config import cfg +from oslo_config import fixture as config_fixture from oslo_utils import timeutils import six @@ -37,6 +38,8 @@ from cinder import test import cinder.tests.unit.image.fake from cinder import volume +from keystonemiddleware import auth_token + CONF = cfg.CONF @@ -358,11 +361,9 @@ class FakeDriver(object): return resources def get_project_quotas(self, context, resources, project_id, - quota_class=None, defaults=True, usages=True, - parent_project_id=None): + quota_class=None, defaults=True, usages=True): self.called.append(('get_project_quotas', context, resources, - project_id, quota_class, defaults, usages, - parent_project_id)) + project_id, quota_class, defaults, usages)) return resources def limit_check(self, context, resources, values, project_id=None): @@ -613,7 +614,6 @@ class QuotaEngineTestCase(test.TestCase): def test_get_project_quotas(self): context = FakeContext(None, None) driver = FakeDriver() - parent_project_id = None quota_obj = self._make_quota_obj(driver) result1 = quota_obj.get_project_quotas(context, 'test_project') result2 = quota_obj.get_project_quotas(context, 'test_project', @@ -628,33 +628,26 @@ class QuotaEngineTestCase(test.TestCase): 'test_project', None, True, - True, - parent_project_id), + True), ('get_project_quotas', context, quota_obj.resources, 'test_project', 'test_class', False, - False, - parent_project_id), ], driver.called) + False), ], driver.called) self.assertEqual(quota_obj.resources, result1) self.assertEqual(quota_obj.resources, result2) def test_get_subproject_quotas(self): context = FakeContext(None, None) driver = FakeDriver() - parent_project_id = 'test_parent_project_id' quota_obj = self._make_quota_obj(driver) - result1 = quota_obj.get_project_quotas(context, 'test_project', - parent_project_id= - parent_project_id) + result1 = quota_obj.get_project_quotas(context, 'test_project') result2 = quota_obj.get_project_quotas(context, 'test_project', quota_class='test_class', defaults=False, - usages=False, - parent_project_id= - parent_project_id) + usages=False) self.assertEqual([ ('get_project_quotas', @@ -663,16 +656,14 @@ class QuotaEngineTestCase(test.TestCase): 'test_project', None, True, - True, - parent_project_id), + True), ('get_project_quotas', context, quota_obj.resources, 'test_project', 'test_class', False, - False, - parent_project_id), ], driver.called) + False), ], driver.called) self.assertEqual(quota_obj.resources, result1) self.assertEqual(quota_obj.resources, result2) @@ -886,9 +877,9 @@ class VolumeTypeQuotaEngineTestCase(test.TestCase): db.volume_type_destroy(ctx, vtype2['id']) -class DbQuotaDriverTestCase(test.TestCase): +class DbQuotaDriverBaseTestCase(test.TestCase): def setUp(self): - super(DbQuotaDriverTestCase, self).setUp() + super(DbQuotaDriverBaseTestCase, self).setUp() self.flags(quota_volumes=10, quota_snapshots=10, @@ -900,7 +891,21 @@ class DbQuotaDriverTestCase(test.TestCase): max_age=0, ) - self.driver = quota.DbQuotaDriver() + # These can be used for expected defaults for child/non-child + self._default_quotas_non_child = dict( + volumes=10, + snapshots=10, + gigabytes=1000, + backups=10, + backup_gigabytes=1000, + per_volume_gigabytes=-1) + self._default_quotas_child = dict( + volumes=0, + snapshots=0, + gigabytes=0, + backups=0, + backup_gigabytes=0, + per_volume_gigabytes=0) self.calls = [] @@ -909,38 +914,6 @@ class DbQuotaDriverTestCase(test.TestCase): self.mock_utcnow = patcher.start() self.mock_utcnow.return_value = datetime.datetime.utcnow() - def test_get_defaults(self): - # Use our pre-defined resources - self._stub_quota_class_get_default() - self._stub_volume_type_get_all() - result = self.driver.get_defaults(None, quota.QUOTAS.resources) - - self.assertEqual( - dict( - volumes=10, - snapshots=10, - gigabytes=1000, - backups=10, - backup_gigabytes=1000, - per_volume_gigabytes=-1), result) - - def test_subproject_get_defaults(self): - # Test subproject default values. - self._stub_volume_type_get_all() - parent_project_id = 'test_parent_project_id' - result = self.driver.get_defaults(None, - quota.QUOTAS.resources, - parent_project_id) - - self.assertEqual( - dict( - volumes=0, - snapshots=0, - gigabytes=0, - backups=0, - backup_gigabytes=0, - per_volume_gigabytes=0), result) - def _stub_quota_class_get_default(self): # Stub out quota_class_get_default def fake_qcgd(context): @@ -967,6 +940,37 @@ class DbQuotaDriverTestCase(test.TestCase): backup_gigabytes=500) self.stubs.Set(db, 'quota_class_get_all_by_name', fake_qcgabn) + def _stub_allocated_get_all_by_project(self, allocated_quota=False): + def fake_qagabp(context, project_id): + self.calls.append('quota_allocated_get_all_by_project') + if allocated_quota: + return dict(project_id=project_id, volumes=3) + return dict(project_id=project_id) + + self.stubs.Set(db, 'quota_allocated_get_all_by_project', fake_qagabp) + + +class DbQuotaDriverTestCase(DbQuotaDriverBaseTestCase): + def setUp(self): + super(DbQuotaDriverTestCase, self).setUp() + + self.driver = quota.DbQuotaDriver() + + def test_get_defaults(self): + # Use our pre-defined resources + self._stub_quota_class_get_default() + self._stub_volume_type_get_all() + result = self.driver.get_defaults(None, quota.QUOTAS.resources) + + self.assertEqual( + dict( + volumes=10, + snapshots=10, + gigabytes=1000, + backups=10, + backup_gigabytes=1000, + per_volume_gigabytes=-1), result) + def test_get_class_quotas(self): self._stub_quota_class_get_all_by_name() self._stub_volume_type_get_all() @@ -1017,33 +1021,6 @@ class DbQuotaDriverTestCase(test.TestCase): self._stub_quota_class_get_all_by_name() self._stub_quota_class_get_default() - def _stub_get_by_subproject(self): - def fake_qgabp(context, project_id): - self.calls.append('quota_get_all_by_project') - self.assertEqual('test_project', project_id) - return dict(volumes=10, gigabytes=50, reserved=0) - - def fake_qugabp(context, project_id): - self.calls.append('quota_usage_get_all_by_project') - self.assertEqual('test_project', project_id) - return dict(volumes=dict(in_use=2, reserved=0), - gigabytes=dict(in_use=10, reserved=0)) - - self.stubs.Set(db, 'quota_get_all_by_project', fake_qgabp) - self.stubs.Set(db, 'quota_usage_get_all_by_project', fake_qugabp) - - self._stub_quota_class_get_all_by_name() - - def _stub_allocated_get_all_by_project(self, allocated_quota=False): - def fake_qagabp(context, project_id): - self.calls.append('quota_allocated_get_all_by_project') - self.assertEqual('test_project', project_id) - if allocated_quota: - return dict(project_id=project_id, volumes=3) - return dict(project_id=project_id) - - self.stubs.Set(db, 'quota_allocated_get_all_by_project', fake_qagabp) - def test_get_project_quotas(self): self._stub_get_by_project() self._stub_volume_type_get_all() @@ -1115,45 +1092,6 @@ class DbQuotaDriverTestCase(test.TestCase): allocated=0) ), result) - def test_get_subproject_quotas(self): - self._stub_get_by_subproject() - self._stub_volume_type_get_all() - self._stub_allocated_get_all_by_project(allocated_quota=True) - parent_project_id = 'test_parent_project_id' - result = self.driver.get_project_quotas( - FakeContext('test_project', None), - quota.QUOTAS.resources, 'test_project', - parent_project_id=parent_project_id) - - self.assertEqual(['quota_get_all_by_project', - 'quota_usage_get_all_by_project', - 'quota_allocated_get_all_by_project', ], self.calls) - self.assertEqual(dict(volumes=dict(limit=10, - in_use=2, - reserved=0, - allocated=3, ), - snapshots=dict(limit=0, - in_use=0, - reserved=0, - allocated=0, ), - gigabytes=dict(limit=50, - in_use=10, - reserved=0, - allocated=0, ), - backups=dict(limit=0, - in_use=0, - reserved=0, - allocated=0, ), - backup_gigabytes=dict(limit=0, - in_use=0, - reserved=0, - allocated=0, ), - per_volume_gigabytes=dict(in_use=0, - limit=0, - reserved=0, - allocated=0) - ), result) - def test_get_project_quotas_alt_context_no_class(self): self._stub_get_by_project() self._stub_volume_type_get_all() @@ -1423,6 +1361,231 @@ class DbQuotaDriverTestCase(test.TestCase): self.calls) +class NestedDbQuotaDriverBaseTestCase(DbQuotaDriverBaseTestCase): + def setUp(self): + super(NestedDbQuotaDriverBaseTestCase, self).setUp() + self.context = context.RequestContext('user_id', + 'project_id', + is_admin=True, + auth_token="fake_token") + self.auth_url = 'http://localhost:5000' + self._child_proj_id = 'child_id' + self._non_child_proj_id = 'non_child_id' + + keystone_mock = mock.Mock() + keystone_mock.version = 'v3' + + class FakeProject(object): + def __init__(self, parent_id): + self.parent_id = parent_id + + def fake_get_project(project_id, subtree_as_ids=False): + # Enable imitation of projects with and without parents + if project_id == self._child_proj_id: + return FakeProject('parent_id') + else: + return FakeProject(None) + + keystone_mock.projects.get.side_effect = fake_get_project + + def _keystone_mock(self): + return keystone_mock + + keystone_patcher = mock.patch('cinder.quota_utils._keystone_client', + _keystone_mock) + keystone_patcher.start() + self.addCleanup(keystone_patcher.stop) + + self.fixture = self.useFixture(config_fixture.Config(auth_token.CONF)) + self.fixture.config(auth_uri=self.auth_url, group='keystone_authtoken') + self.driver = quota.NestedDbQuotaDriver() + + def _stub_get_by_subproject(self): + def fake_qgabp(context, project_id): + self.calls.append('quota_get_all_by_project') + return dict(volumes=10, gigabytes=50, reserved=0) + + def fake_qugabp(context, project_id): + self.calls.append('quota_usage_get_all_by_project') + return dict(volumes=dict(in_use=2, reserved=0), + gigabytes=dict(in_use=10, reserved=0)) + + self.stubs.Set(db, 'quota_get_all_by_project', fake_qgabp) + self.stubs.Set(db, 'quota_usage_get_all_by_project', fake_qugabp) + + self._stub_quota_class_get_all_by_name() + + +class NestedDbQuotaDriverTestCase(NestedDbQuotaDriverBaseTestCase): + def test_get_defaults(self): + self._stub_volume_type_get_all() + + # Test for child project defaults + result = self.driver.get_defaults(self.context, + quota.QUOTAS.resources, + self._child_proj_id) + self.assertEqual(self._default_quotas_child, result) + + # Test for non-child project defaults + result = self.driver.get_defaults(self.context, + quota.QUOTAS.resources, + self._non_child_proj_id) + self.assertEqual(self._default_quotas_non_child, result) + + def test_subproject_enforce_defaults(self): + # Non-child defaults should allow volume to get created + self.driver.reserve(self.context, + quota.QUOTAS.resources, + {'volumes': 1, 'gigabytes': 1}, + project_id=self._non_child_proj_id) + + # Child defaults should not allow volume to be created + self.assertRaises(exception.OverQuota, + self.driver.reserve, self.context, + quota.QUOTAS.resources, + {'volumes': 1, 'gigabytes': 1}, + project_id=self._child_proj_id) + + def test_get_subproject_quotas(self): + self._stub_get_by_subproject() + self._stub_volume_type_get_all() + self._stub_allocated_get_all_by_project(allocated_quota=True) + result = self.driver.get_project_quotas( + self.context, + quota.QUOTAS.resources, self._child_proj_id) + + self.assertEqual(['quota_get_all_by_project', + 'quota_usage_get_all_by_project', + 'quota_allocated_get_all_by_project', ], self.calls) + self.assertEqual(dict(volumes=dict(limit=10, + in_use=2, + reserved=0, + allocated=3, ), + snapshots=dict(limit=0, + in_use=0, + reserved=0, + allocated=0, ), + gigabytes=dict(limit=50, + in_use=10, + reserved=0, + allocated=0, ), + backups=dict(limit=0, + in_use=0, + reserved=0, + allocated=0, ), + backup_gigabytes=dict(limit=0, + in_use=0, + reserved=0, + allocated=0, ), + per_volume_gigabytes=dict(in_use=0, + limit=0, + reserved=0, + allocated=0) + ), result) + + +class NestedQuotaValidation(NestedDbQuotaDriverBaseTestCase): + def setUp(self): + super(NestedQuotaValidation, self).setUp() + """ + Quota hierarchy setup like so + +-----------+ + | | + | A | + | / \ | + | B C | + | / | + | D | + +-----------+ + """ + self.project_tree = {'A': {'B': {'D': None}, 'C': None}} + self.proj_vals = { + 'A': {'limit': 7, 'in_use': 1, 'alloc': 6}, + 'B': {'limit': 3, 'in_use': 1, 'alloc': 2}, + 'D': {'limit': 2, 'in_use': 0}, + 'C': {'limit': 3, 'in_use': 3}, + } + + # Just using one resource currently for simplicity of test + self.resources = {'volumes': quota.ReservableResource( + 'volumes', '_sync_volumes', 'quota_volumes')} + + to_patch = [('cinder.db.quota_allocated_get_all_by_project', + self._fake_quota_allocated_get_all_by_project), + ('cinder.db.quota_get_all_by_project', + self._fake_quota_get_all_by_project), + ('cinder.db.quota_usage_get_all_by_project', + self._fake_quota_usage_get_all_by_project)] + + for patch_path, patch_obj in to_patch: + patcher = mock.patch(patch_path, patch_obj) + patcher.start() + self.addCleanup(patcher.stop) + + def _fake_quota_get_all_by_project(self, context, project_id): + return {'volumes': self.proj_vals[project_id]['limit']} + + def _fake_quota_usage_get_all_by_project(self, context, project_id): + return {'volumes': self.proj_vals[project_id]} + + def _fake_quota_allocated_get_all_by_project(self, context, project_id): + ret = {'project_id': project_id} + proj_val = self.proj_vals[project_id] + if 'alloc' in proj_val: + ret['volumes'] = proj_val['alloc'] + return ret + + def test_validate_nested_quotas(self): + self.driver.validate_nested_setup(self.context, + self.resources, self.project_tree) + + # Fail because 7 - 2 < 3 + 3 + self.proj_vals['A']['in_use'] = 2 + self.assertRaises(exception.InvalidNestedQuotaSetup, + self.driver.validate_nested_setup, + self.context, + self.resources, self.project_tree) + self.proj_vals['A']['in_use'] = 1 + + # Fail because 7 - 1 < 3 + 7 + self.proj_vals['C']['limit'] = 7 + self.assertRaises(exception.InvalidNestedQuotaSetup, + self.driver.validate_nested_setup, + self.context, + self.resources, self.project_tree) + self.proj_vals['C']['limit'] = 3 + + # Fail because 3 < 4 + self.proj_vals['D']['limit'] = 4 + self.assertRaises(exception.InvalidNestedQuotaSetup, + self.driver.validate_nested_setup, + self.context, + self.resources, self.project_tree) + self.proj_vals['D']['limit'] = 2 + + def test_validate_nested_quotas_negative_child_limit(self): + self.proj_vals['B']['limit'] = -1 + self.assertRaises( + exception.InvalidNestedQuotaSetup, + self.driver.validate_nested_setup, + self.context, self.resources, self.project_tree) + + def test_validate_nested_quotas_usage_over_limit(self): + + self.proj_vals['D']['in_use'] = 5 + self.assertRaises(exception.InvalidNestedQuotaSetup, + self.driver.validate_nested_setup, + self.context, self.resources, self.project_tree) + + def test_validate_nested_quota_bad_allocated_quotas(self): + + self.proj_vals['A']['alloc'] = 5 + self.proj_vals['B']['alloc'] = 8 + self.assertRaises(exception.InvalidNestedQuotaSetup, + self.driver.validate_nested_setup, + self.context, self.resources, self.project_tree) + + class FakeSession(object): def begin(self): return self diff --git a/cinder/tests/unit/test_quota_utils.py b/cinder/tests/unit/test_quota_utils.py new file mode 100644 index 000000000..c220651e3 --- /dev/null +++ b/cinder/tests/unit/test_quota_utils.py @@ -0,0 +1,110 @@ +# Copyright 2016 OpenStack Foundation +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import mock + +from cinder import context +from cinder import exception +from cinder import quota_utils +from cinder import test + +from keystoneclient import exceptions +from keystonemiddleware import auth_token + +from oslo_config import cfg +from oslo_config import fixture as config_fixture + +CONF = cfg.CONF + + +class QuotaUtilsTest(test.TestCase): + class FakeProject(object): + def __init__(self, id='foo', parent_id=None): + self.id = id + self.parent_id = parent_id + self.subtree = None + + def setUp(self): + super(QuotaUtilsTest, self).setUp() + + self.auth_url = 'http://localhost:5000' + self.context = context.RequestContext('fake_user', 'fake_proj_id') + self.fixture = self.useFixture(config_fixture.Config(auth_token.CONF)) + self.fixture.config(auth_uri=self.auth_url, group='keystone_authtoken') + + @mock.patch('keystoneclient.client.Client') + @mock.patch('keystoneclient.session.Session') + def test_keystone_client_instantiation(self, ksclient_session, + ksclient_class): + quota_utils._keystone_client(self.context) + ksclient_class.assert_called_once_with(auth_url=self.auth_url, + session=ksclient_session(), + version=(3, 0)) + + @mock.patch('keystoneclient.client.Client') + def test_get_project_keystoneclient_v2(self, ksclient_class): + keystoneclient = ksclient_class.return_value + keystoneclient.version = 'v2.0' + expected_project = quota_utils.GenericProjectInfo( + self.context.project_id, 'v2.0') + project = quota_utils.get_project_hierarchy( + self.context, self.context.project_id) + self.assertEqual(expected_project.__dict__, project.__dict__) + + @mock.patch('keystoneclient.client.Client') + def test_get_project_keystoneclient_v3(self, ksclient_class): + keystoneclient = ksclient_class.return_value + keystoneclient.version = 'v3' + returned_project = self.FakeProject(self.context.project_id, 'bar') + del returned_project.subtree + keystoneclient.projects.get.return_value = returned_project + expected_project = quota_utils.GenericProjectInfo( + self.context.project_id, 'v3', 'bar') + project = quota_utils.get_project_hierarchy( + self.context, self.context.project_id) + self.assertEqual(expected_project.__dict__, project.__dict__) + + @mock.patch('keystoneclient.client.Client') + def test_get_project_keystoneclient_v3_with_subtree(self, ksclient_class): + keystoneclient = ksclient_class.return_value + keystoneclient.version = 'v3' + returned_project = self.FakeProject(self.context.project_id, 'bar') + subtree_dict = {'baz': {'quux': None}} + returned_project.subtree = subtree_dict + keystoneclient.projects.get.return_value = returned_project + expected_project = quota_utils.GenericProjectInfo( + self.context.project_id, 'v3', 'bar', subtree_dict) + project = quota_utils.get_project_hierarchy( + self.context, self.context.project_id, subtree_as_ids=True) + keystoneclient.projects.get.assert_called_once_with( + self.context.project_id, subtree_as_ids=True) + self.assertEqual(expected_project.__dict__, project.__dict__) + + @mock.patch('cinder.quota_utils._keystone_client') + def test_validate_nested_projects_with_keystone_v2(self, _keystone_client): + _keystone_client.side_effect = exceptions.VersionNotAvailable + + self.assertRaises(exception.CinderException, + quota_utils.validate_setup_for_nested_quota_use, + self.context, [], None) + + @mock.patch('cinder.quota_utils._keystone_client') + def test_validate_nested_projects_non_cloud_admin(self, _keystone_client): + # Covers not cloud admin or using old policy.json + _keystone_client.side_effect = exceptions.Forbidden + + self.assertRaises(exception.CinderException, + quota_utils.validate_setup_for_nested_quota_use, + self.context, [], None) diff --git a/etc/cinder/policy.json b/etc/cinder/policy.json index 47d1cadbd..9e280cc64 100644 --- a/etc/cinder/policy.json +++ b/etc/cinder/policy.json @@ -39,6 +39,7 @@ "volume_extension:quotas:update": "rule:admin_api", "volume_extension:quotas:delete": "rule:admin_api", "volume_extension:quota_classes": "rule:admin_api", + "volume_extension:quota_classes:validate_setup_for_nested_quota_use": "rule: admin_api", "volume_extension:volume_admin_actions:reset_status": "rule:admin_api", "volume_extension:snapshot_admin_actions:reset_status": "rule:admin_api", diff --git a/releasenotes/notes/split-out-nested-quota-driver-e9493f478d2b8be5.yaml b/releasenotes/notes/split-out-nested-quota-driver-e9493f478d2b8be5.yaml new file mode 100644 index 000000000..43a33a3df --- /dev/null +++ b/releasenotes/notes/split-out-nested-quota-driver-e9493f478d2b8be5.yaml @@ -0,0 +1,12 @@ +--- +features: + - Split nested quota support into a separate driver. In + order to use nested quotas, change the following config + ``quota_driver = cinder.quota.NestedDbQuotaDriver`` after + running the following admin API + "os-quota-sets/validate_setup_for_nested_quota_use" command + to ensure the existing quota values make sense to nest. +upgrade: + - Nested quotas will no longer be used by default, but can be + configured by setting + ``quota_driver = cinder.quota.NestedDbQuotaDriver`` -- 2.45.2