]> review.fuel-infra Code Review - openstack-build/neutron-build.git/commitdiff
Introduce usage data tracking for Neutron
authorSalvatore Orlando <salv.orlando@gmail.com>
Thu, 16 Apr 2015 18:56:27 +0000 (11:56 -0700)
committerSalvatore Orlando <salv.orlando@gmail.com>
Tue, 28 Jul 2015 18:55:03 +0000 (11:55 -0700)
This patch introduces application logic support for tracking
Neutron resource usage data, thus introducing a different
way of enforcing quota limits, which now relies on records
tracking resource usage for each tenant.

When these records go "out-of-sync" with actual resource usage,
the quota usage table entry is marked "dirty".
And it will be resynchronized the next time a resource count is
requested. Changes in resource utilization are detected using
SQLAlchemy events.

This patch however does not automatically enable resource usage
tracking for any plugin. Plugins must explicitly declare for which
resources they wish to enable tracking. To this aim, this patch
provides the @tracked_resources decorator.

As some operators might wish to not use resource usage tracking, this
patch adds the track_quota_usage configuration option. If set to
False, DB-level usage tracking will not be performed, and
resources will be counted as before, ie: invoking a plugin method.

Usage tracking is performed using a new resource type,
TrackedResource, which can work side by side with CountableResource,
the one Neutron used so far. To this aim a ResourceRegistry class
has been introduced for registering and managing resources. Most
of the resource management code previously embedded in the
QuotaEngine class is being moved to ResourceRegistry as a part of
this patch.

Partially implements blueprint better-quota

Change-Id: If461399900973a38d7b82e0b3ac54fc052f09529

18 files changed:
neutron/api/rpc/handlers/dhcp_rpc.py
neutron/api/v2/base.py
neutron/api/v2/resource_helper.py
neutron/api/v2/router.py
neutron/db/ipam_backend_mixin.py
neutron/db/models_v2.py
neutron/extensions/quotasv2.py
neutron/extensions/securitygroup.py
neutron/plugins/nec/extensions/packetfilter.py
neutron/quota/__init__.py
neutron/quota/resource.py [new file with mode: 0644]
neutron/quota/resource_registry.py [new file with mode: 0644]
neutron/tests/unit/api/v2/test_base.py
neutron/tests/unit/extensions/extensionattribute.py
neutron/tests/unit/extensions/test_quotasv2.py
neutron/tests/unit/quota/__init__.py [new file with mode: 0644]
neutron/tests/unit/quota/test_resource.py [new file with mode: 0644]
neutron/tests/unit/quota/test_resource_registry.py [new file with mode: 0644]

index 7d97b7c52269671bc4200b51594ab209eb95e51a..07438334a3881a37e0126402347af3d8d0766f50 100644 (file)
@@ -29,7 +29,7 @@ from neutron.common import utils
 from neutron.extensions import portbindings
 from neutron.i18n import _LW
 from neutron import manager
-
+from neutron.quota import resource_registry
 
 LOG = logging.getLogger(__name__)
 
@@ -203,6 +203,7 @@ class DhcpRpcCallback(object):
         LOG.warning(_LW('Updating lease expiration is now deprecated. Issued  '
                         'from host %s.'), host)
 
+    @resource_registry.mark_resources_dirty
     def create_dhcp_port(self, context, **kwargs):
         """Create and return dhcp port information.
 
index 3f72e08f7b0d0f2b94b790248195d3e0559b7e8b..a1841b8cb8df0c86c0c365864bc432d05730dae3 100644 (file)
@@ -34,6 +34,7 @@ from neutron.db import api as db_api
 from neutron.i18n import _LE, _LI
 from neutron import policy
 from neutron import quota
+from neutron.quota import resource_registry
 
 
 LOG = logging.getLogger(__name__)
@@ -207,7 +208,15 @@ class Controller(object):
                                name,
                                resource,
                                pluralized=self._collection)
-                return getattr(self._plugin, name)(*arg_list, **kwargs)
+                ret_value = getattr(self._plugin, name)(*arg_list, **kwargs)
+                # It is simply impossible to predict whether one of this
+                # actions alters resource usage. For instance a tenant port
+                # is created when a router interface is added. Therefore it is
+                # important to mark as dirty resources whose counters have
+                # been altered by this operation
+                resource_registry.set_resources_dirty(request.context)
+                return ret_value
+
             return _handle_action
         else:
             raise AttributeError()
@@ -280,6 +289,9 @@ class Controller(object):
         pagination_links = pagination_helper.get_links(obj_list)
         if pagination_links:
             collection[self._collection + "_links"] = pagination_links
+        # Synchronize usage trackers, if needed
+        resource_registry.resync_resource(
+            request.context, self._resource, request.context.tenant_id)
         return collection
 
     def _item(self, request, id, do_authz=False, field_list=None,
@@ -436,6 +448,12 @@ class Controller(object):
                                          **kwargs)
 
         def notify(create_result):
+            # Ensure usage trackers for all resources affected by this API
+            # operation are marked as dirty
+            # TODO(salv-orlando): This operation will happen in a single
+            # transaction with reservation commit once that is implemented
+            resource_registry.set_resources_dirty(request.context)
+
             notifier_method = self._resource + '.create.end'
             self._notifier.info(request.context,
                                 notifier_method,
@@ -497,6 +515,9 @@ class Controller(object):
 
         obj_deleter = getattr(self._plugin, action)
         obj_deleter(request.context, id, **kwargs)
+        # A delete operation usually alters resource usage, so mark affected
+        # usage trackers as dirty
+        resource_registry.set_resources_dirty(request.context)
         notifier_method = self._resource + '.delete.end'
         self._notifier.info(request.context,
                             notifier_method,
@@ -561,6 +582,12 @@ class Controller(object):
         if parent_id:
             kwargs[self._parent_id_name] = parent_id
         obj = obj_updater(request.context, id, **kwargs)
+        # Usually an update operation does not alter resource usage, but as
+        # there might be side effects it might be worth checking for changes
+        # in resource usage here as well (e.g: a tenant port is created when a
+        # router interface is added)
+        resource_registry.set_resources_dirty(request.context)
+
         result = {self._resource: self._view(request.context, obj)}
         notifier_method = self._resource + '.update.end'
         self._notifier.info(request.context, notifier_method, result)
index 05e403d030da1cf1de5c408987b8b1693205773b..c506320c91dedf90d76b129705f6e525c6a8c187 100644 (file)
@@ -20,7 +20,7 @@ from neutron.api import extensions
 from neutron.api.v2 import base
 from neutron import manager
 from neutron.plugins.common import constants
-from neutron import quota
+from neutron.quota import resource_registry
 
 LOG = logging.getLogger(__name__)
 
@@ -80,7 +80,7 @@ def build_resource_info(plural_mappings, resource_map, which_service,
         if translate_name:
             collection_name = collection_name.replace('_', '-')
         if register_quota:
-            quota.QUOTAS.register_resource_by_name(resource_name)
+            resource_registry.register_resource_by_name(resource_name)
         member_actions = action_map.get(resource_name, {})
         controller = base.create_resource(
             collection_name, resource_name, plugin, params,
index c76f2d02ac55ba75e0bc3dabd20d5266fead6a25..bd59d854b0e91d0ea33f7e807bbc5f2255ce9231 100644 (file)
@@ -27,7 +27,7 @@ from neutron.api.v2 import attributes
 from neutron.api.v2 import base
 from neutron import manager
 from neutron import policy
-from neutron import quota
+from neutron.quota import resource_registry
 from neutron import wsgi
 
 
@@ -106,7 +106,7 @@ class APIRouter(wsgi.Router):
             _map_resource(RESOURCES[resource], resource,
                           attributes.RESOURCE_ATTRIBUTE_MAP.get(
                               RESOURCES[resource], dict()))
-            quota.QUOTAS.register_resource_by_name(resource)
+            resource_registry.register_resource_by_name(resource)
 
         for resource in SUB_RESOURCES:
             _map_resource(SUB_RESOURCES[resource]['collection_name'], resource,
index ca69f460b787a678e0f8abc48181a240088ec5a8..4d732ec1becf7a2b35dfa1798195ef7cb33c21e9 100644 (file)
@@ -409,7 +409,7 @@ class IpamBackendMixin(db_base_plugin_common.DbBasePluginCommon):
                  enable_eagerloads(False).filter_by(id=port_id))
         if not context.is_admin:
             query = query.filter_by(tenant_id=context.tenant_id)
-        query.delete()
+        context.session.delete(query.first())
 
     def _save_subnet(self, context,
                      network,
index 8ba70db7790974425565f7fe8f6038118018c115..23bcde49117a2038605469fd9bca77495e06898c 100644 (file)
@@ -133,7 +133,8 @@ class Port(model_base.BASEV2, HasId, HasTenant):
     name = sa.Column(sa.String(attr.NAME_MAX_LEN))
     network_id = sa.Column(sa.String(36), sa.ForeignKey("networks.id"),
                            nullable=False)
-    fixed_ips = orm.relationship(IPAllocation, backref='port', lazy='joined')
+    fixed_ips = orm.relationship(IPAllocation, backref='port', lazy='joined',
+                                 passive_deletes='all')
     mac_address = sa.Column(sa.String(32), nullable=False)
     admin_state_up = sa.Column(sa.Boolean(), nullable=False)
     status = sa.Column(sa.String(16), nullable=False)
index a47a1adb98cb14edd583d5e38f28875b45742cf9..f9a3ae9915f28667c566062c5032eed31fde1ae7 100644 (file)
@@ -25,6 +25,7 @@ from neutron.common import constants as const
 from neutron.common import exceptions as n_exc
 from neutron import manager
 from neutron import quota
+from neutron.quota import resource_registry
 from neutron import wsgi
 
 
@@ -48,7 +49,7 @@ class QuotaSetsController(wsgi.Controller):
         self._update_extended_attributes = True
 
     def _update_attributes(self):
-        for quota_resource in QUOTAS.resources.keys():
+        for quota_resource in resource_registry.get_all_resources().keys():
             attr_dict = EXTENDED_ATTRIBUTES_2_0[RESOURCE_COLLECTION]
             attr_dict[quota_resource] = {
                 'allow_post': False,
@@ -60,7 +61,9 @@ class QuotaSetsController(wsgi.Controller):
 
     def _get_quotas(self, request, tenant_id):
         return self._driver.get_tenant_quotas(
-            request.context, QUOTAS.resources, tenant_id)
+            request.context,
+            resource_registry.get_all_resources(),
+            tenant_id)
 
     def create(self, request, body=None):
         msg = _('POST requests are not supported on this resource.')
@@ -70,7 +73,8 @@ class QuotaSetsController(wsgi.Controller):
         context = request.context
         self._check_admin(context)
         return {self._resource_name + "s":
-                self._driver.get_all_quotas(context, QUOTAS.resources)}
+                self._driver.get_all_quotas(
+                    context, resource_registry.get_all_resources())}
 
     def tenant(self, request):
         """Retrieve the tenant info in context."""
index 8e863e831d6dc1071ff8f3727ac1b675541f36ac..f199f12025ab4e768a65f1b03715605485f3c15b 100644 (file)
@@ -26,7 +26,7 @@ from neutron.api.v2 import base
 from neutron.common import constants as const
 from neutron.common import exceptions as nexception
 from neutron import manager
-from neutron import quota
+from neutron.quota import resource_registry
 
 
 # Security group Exceptions
@@ -305,7 +305,7 @@ class Securitygroup(extensions.ExtensionDescriptor):
         for resource_name in ['security_group', 'security_group_rule']:
             collection_name = resource_name.replace('_', '-') + "s"
             params = RESOURCE_ATTRIBUTE_MAP.get(resource_name + "s", dict())
-            quota.QUOTAS.register_resource_by_name(resource_name)
+            resource_registry.register_resource_by_name(resource_name)
             controller = base.create_resource(collection_name,
                                               resource_name,
                                               plugin, params, allow_bulk=True,
index 7c9971f8a967e33d5548c958dcd6aa9db66a2248..3d89cf4e25a81e525a41c690eed0295337ec0627 100644 (file)
@@ -21,7 +21,8 @@ from neutron.api.v2 import base
 from neutron.common import constants
 from neutron.common import exceptions
 from neutron import manager
-from neutron import quota
+from neutron.quota import resource as quota_resource
+from neutron.quota import resource_registry
 
 
 quota_packet_filter_opts = [
@@ -180,10 +181,10 @@ class Packetfilter(extensions.ExtensionDescriptor):
 
     @classmethod
     def get_resources(cls):
-        qresource = quota.CountableResource(RESOURCE,
-                                            quota._count_resource,
-                                            'quota_%s' % RESOURCE)
-        quota.QUOTAS.register_resource(qresource)
+        qresource = quota_resource.CountableResource(
+            RESOURCE, quota_resource._count_resource, 'quota_%s' % RESOURCE)
+
+        resource_registry.register_resource(qresource)
 
         resource = base.create_resource(COLLECTION, RESOURCE,
                                         manager.NeutronManager.get_plugin(),
index f71b14aabe1d71a74ba067bc912bc51b78063e52..97b466e872a79473fa7474f4cef21bc8a8fdd065 100644 (file)
@@ -1,4 +1,4 @@
-#    Copyright 2011 OpenStack Foundation
+# Copyright (c) 2015 OpenStack Foundation.  All rights reserved.
 #
 #    Licensed under the Apache License, Version 2.0 (the "License"); you may
 #    not use this file except in compliance with the License. You may obtain
@@ -25,6 +25,7 @@ import webob
 
 from neutron.common import exceptions
 from neutron.i18n import _LI, _LW
+from neutron.quota import resource_registry
 
 
 LOG = logging.getLogger(__name__)
@@ -33,6 +34,7 @@ QUOTA_DB_DRIVER = '%s.DbQuotaDriver' % QUOTA_DB_MODULE
 QUOTA_CONF_DRIVER = 'neutron.quota.ConfDriver'
 default_quota_items = ['network', 'subnet', 'port']
 
+
 quota_opts = [
     cfg.ListOpt('quota_items',
                 default=default_quota_items,
@@ -59,6 +61,11 @@ quota_opts = [
     cfg.StrOpt('quota_driver',
                default=QUOTA_DB_DRIVER,
                help=_('Default driver to use for quota checks')),
+    cfg.BoolOpt('track_quota_usage',
+                default=True,
+                help=_('Keep in track in the database of current resource'
+                       'quota usage. Plugins which do not leverage the '
+                       'neutron database should set this flag to False')),
 ]
 # Register the configuration options
 cfg.CONF.register_opts(quota_opts, 'QUOTAS')
@@ -146,67 +153,19 @@ class ConfDriver(object):
         raise webob.exc.HTTPForbidden(msg)
 
 
-class BaseResource(object):
-    """Describe a single resource for quota checking."""
-
-    def __init__(self, name, flag):
-        """Initializes a resource.
-
-        :param name: The name of the resource, i.e., "instances".
-        :param flag: The name of the flag or configuration option
-        """
-
-        self.name = name
-        self.flag = flag
-
-    @property
-    def default(self):
-        """Return the default value of the quota."""
-        # Any negative value will be interpreted as an infinite quota,
-        # and stored as -1 for compatibility with current behaviour
-        value = getattr(cfg.CONF.QUOTAS,
-                        self.flag,
-                        cfg.CONF.QUOTAS.default_quota)
-        return max(value, -1)
-
-
-class CountableResource(BaseResource):
-    """Describe a resource where the counts are determined by a function."""
-
-    def __init__(self, name, count, flag=None):
-        """Initializes a CountableResource.
-
-        Countable resources are those resources which directly
-        correspond to objects in the database, i.e., netowk, subnet,
-        etc.,.  A CountableResource must be constructed with a counting
-        function, which will be called to determine the current counts
-        of the resource.
-
-        The counting function will be passed the context, along with
-        the extra positional and keyword arguments that are passed to
-        Quota.count().  It should return an integer specifying the
-        count.
-
-        :param name: The name of the resource, i.e., "instances".
-        :param count: A callable which returns the count of the
-                      resource.  The arguments passed are as described
-                      above.
-        :param flag: The name of the flag or configuration option
-                     which specifies the default value of the quota
-                     for this resource.
-        """
-
-        super(CountableResource, self).__init__(name, flag=flag)
-        self.count = count
-
-
 class QuotaEngine(object):
     """Represent the set of recognized quotas."""
 
+    _instance = None
+
+    @classmethod
+    def get_instance(cls):
+        if not cls._instance:
+            cls._instance = cls()
+        return cls._instance
+
     def __init__(self, quota_driver_class=None):
         """Initialize a Quota object."""
-
-        self._resources = {}
         self._driver = None
         self._driver_class = quota_driver_class
 
@@ -232,29 +191,7 @@ class QuotaEngine(object):
             LOG.info(_LI('Loaded quota_driver: %s.'), _driver_class)
         return self._driver
 
-    def __contains__(self, resource):
-        return resource in self._resources
-
-    def register_resource(self, resource):
-        """Register a resource."""
-        if resource.name in self._resources:
-            LOG.warn(_LW('%s is already registered.'), resource.name)
-            return
-        self._resources[resource.name] = resource
-
-    def register_resource_by_name(self, resourcename):
-        """Register a resource by name."""
-        resource = CountableResource(resourcename, _count_resource,
-                                     'quota_' + resourcename)
-        self.register_resource(resource)
-
-    def register_resources(self, resources):
-        """Register a list of resources."""
-
-        for resource in resources:
-            self.register_resource(resource)
-
-    def count(self, context, resource, *args, **kwargs):
+    def count(self, context, resource_name, *args, **kwargs):
         """Count a resource.
 
         For countable resources, invokes the count() function and
@@ -263,13 +200,13 @@ class QuotaEngine(object):
         the resource.
 
         :param context: The request context, for access checks.
-        :param resource: The name of the resource, as a string.
+        :param resource_name: The name of the resource, as a string.
         """
 
         # Get the resource
-        res = self._resources.get(resource)
+        res = resource_registry.get_resource(resource_name)
         if not res or not hasattr(res, 'count'):
-            raise exceptions.QuotaResourceUnknown(unknown=[resource])
+            raise exceptions.QuotaResourceUnknown(unknown=[resource_name])
 
         return res.count(context, *args, **kwargs)
 
@@ -297,7 +234,8 @@ class QuotaEngine(object):
         """
         # Verify that resources are managed by the quota engine
         requested_resources = set(values.keys())
-        managed_resources = set([res for res in self._resources.keys()
+        managed_resources = set([res for res in
+                                 resource_registry.get_all_resources()
                                  if res in requested_resources])
 
         # Make sure we accounted for all of them...
@@ -306,31 +244,11 @@ class QuotaEngine(object):
             raise exceptions.QuotaResourceUnknown(
                 unknown=sorted(unknown_resources))
 
-        return self.get_driver().limit_check(context, tenant_id,
-                                             self._resources, values)
-
-    @property
-    def resources(self):
-        return self._resources
-
-
-QUOTAS = QuotaEngine()
-
+        return self.get_driver().limit_check(
+            context, tenant_id, resource_registry.get_all_resources(), values)
 
-def _count_resource(context, plugin, resources, tenant_id):
-    count_getter_name = "get_%s_count" % resources
 
-    # Some plugins support a count method for particular resources,
-    # using a DB's optimized counting features. We try to use that one
-    # if present. Otherwise just use regular getter to retrieve all objects
-    # and count in python, allowing older plugins to still be supported
-    try:
-        obj_count_getter = getattr(plugin, count_getter_name)
-        return obj_count_getter(context, filters={'tenant_id': [tenant_id]})
-    except (NotImplementedError, AttributeError):
-        obj_getter = getattr(plugin, "get_%s" % resources)
-        obj_list = obj_getter(context, filters={'tenant_id': [tenant_id]})
-        return len(obj_list) if obj_list else 0
+QUOTAS = QuotaEngine.get_instance()
 
 
 def register_resources_from_config():
@@ -342,12 +260,9 @@ def register_resources_from_config():
                  "quota_items option is deprecated as of Liberty."
                  "Resource REST controllers should take care of registering "
                  "resources with the quota engine."))
-    resources = []
     for resource_item in (set(cfg.CONF.QUOTAS.quota_items) -
                           set(default_quota_items)):
-        resources.append(CountableResource(resource_item, _count_resource,
-                                           'quota_' + resource_item))
-    QUOTAS.register_resources(resources)
+        resource_registry.register_resource_by_name(resource_item)
 
 
 register_resources_from_config()
diff --git a/neutron/quota/resource.py b/neutron/quota/resource.py
new file mode 100644 (file)
index 0000000..25bcba7
--- /dev/null
@@ -0,0 +1,241 @@
+# Copyright (c) 2015 OpenStack Foundation.  All rights reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from oslo_concurrency import lockutils
+from oslo_config import cfg
+from oslo_db import api as oslo_db_api
+from oslo_db import exception as oslo_db_exception
+from oslo_log import log
+from sqlalchemy import event
+
+from neutron.db import api as db_api
+from neutron.db.quota import api as quota_api
+from neutron.i18n import _LE
+
+LOG = log.getLogger(__name__)
+
+
+def _count_resource(context, plugin, resources, tenant_id):
+    count_getter_name = "get_%s_count" % resources
+
+    # Some plugins support a count method for particular resources,
+    # using a DB's optimized counting features. We try to use that one
+    # if present. Otherwise just use regular getter to retrieve all objects
+    # and count in python, allowing older plugins to still be supported
+    try:
+        obj_count_getter = getattr(plugin, count_getter_name)
+        meh = obj_count_getter(context, filters={'tenant_id': [tenant_id]})
+        return meh
+    except (NotImplementedError, AttributeError):
+        obj_getter = getattr(plugin, "get_%s" % resources)
+        obj_list = obj_getter(context, filters={'tenant_id': [tenant_id]})
+        return len(obj_list) if obj_list else 0
+
+
+class BaseResource(object):
+    """Describe a single resource for quota checking."""
+
+    def __init__(self, name, flag):
+        """Initializes a resource.
+
+        :param name: The name of the resource, i.e., "instances".
+        :param flag: The name of the flag or configuration option
+        """
+
+        self.name = name
+        self.flag = flag
+
+    @property
+    def default(self):
+        """Return the default value of the quota."""
+        # Any negative value will be interpreted as an infinite quota,
+        # and stored as -1 for compatibility with current behaviour
+        value = getattr(cfg.CONF.QUOTAS,
+                        self.flag,
+                        cfg.CONF.QUOTAS.default_quota)
+        return max(value, -1)
+
+    @property
+    def dirty(self):
+        """Return the current state of the Resource instance.
+
+        :returns: True if the resource count is out of sync with actual date,
+                  False if it is in sync, and None if the resource instance
+                  does not track usage.
+        """
+
+
+class CountableResource(BaseResource):
+    """Describe a resource where the counts are determined by a function."""
+
+    def __init__(self, name, count, flag=None):
+        """Initializes a CountableResource.
+
+        Countable resources are those resources which directly
+        correspond to objects in the database, i.e., netowk, subnet,
+        etc.,.  A CountableResource must be constructed with a counting
+        function, which will be called to determine the current counts
+        of the resource.
+
+        The counting function will be passed the context, along with
+        the extra positional and keyword arguments that are passed to
+        Quota.count().  It should return an integer specifying the
+        count.
+
+        :param name: The name of the resource, i.e., "instances".
+        :param count: A callable which returns the count of the
+                      resource.  The arguments passed are as described
+                      above.
+        :param flag: The name of the flag or configuration option
+                     which specifies the default value of the quota
+                     for this resource.
+        """
+
+        super(CountableResource, self).__init__(name, flag=flag)
+        self.count = count
+
+
+class TrackedResource(BaseResource):
+    """Resource which keeps track of its usage data."""
+
+    def __init__(self, name, model_class, flag):
+        """Initializes an instance for a given resource.
+
+        TrackedResource are directly mapped to data model classes.
+        Resource usage is tracked in the database, and the model class to
+        which this resource refers is monitored to ensure always "fresh"
+        usage data are employed when performing quota checks.
+
+        This class operates under the assumption that the model class
+        describing the resource has a tenant identifier attribute.
+
+        :param name: The name of the resource, i.e., "networks".
+        :param model_class: The sqlalchemy model class of the resource for
+                            which this instance is being created
+        :param flag: The name of the flag or configuration option
+                     which specifies the default value of the quota
+                     for this resource.
+        """
+        super(TrackedResource, self).__init__(name, flag)
+        # Register events for addition/removal of records in the model class
+        # As tenant_id is immutable for all Neutron objects there is no need
+        # to register a listener for update events
+        self._model_class = model_class
+        self._dirty_tenants = set()
+        self._out_of_sync_tenants = set()
+
+    @property
+    def dirty(self):
+        return self._dirty_tenants
+
+    @lockutils.synchronized('dirty_tenants')
+    def mark_dirty(self, context, nested=False):
+        if not self._dirty_tenants:
+            return
+        with context.session.begin(nested=nested, subtransactions=True):
+            for tenant_id in self._dirty_tenants:
+                quota_api.set_quota_usage_dirty(context, self.name, tenant_id)
+                LOG.debug(("Persisted dirty status for tenant:%(tenant_id)s "
+                           "on resource:%(resource)s"),
+                          {'tenant_id': tenant_id, 'resource': self.name})
+        self._out_of_sync_tenants |= self._dirty_tenants
+        self._dirty_tenants.clear()
+
+    @lockutils.synchronized('dirty_tenants')
+    def _db_event_handler(self, mapper, _conn, target):
+        tenant_id = target.get('tenant_id')
+        if not tenant_id:
+            # NOTE: This is an unexpected error condition. Log anomaly but do
+            # not raise as this might have unexpected effects on other
+            # operations
+            LOG.error(_LE("Model class %s does not have tenant_id attribute"),
+                      target)
+            return
+        self._dirty_tenants.add(tenant_id)
+
+    # Retry the operation if a duplicate entry exception is raised. This
+    # can happen is two or more workers are trying to create a resource of a
+    # give kind for the same tenant concurrently. Retrying the operation will
+    # ensure that an UPDATE statement is emitted rather than an INSERT one
+    @oslo_db_api.wrap_db_retry(
+        max_retries=db_api.MAX_RETRIES,
+        exception_checker=lambda exc:
+        isinstance(exc, oslo_db_exception.DBDuplicateEntry))
+    def _set_quota_usage(self, context, tenant_id, in_use):
+        return quota_api.set_quota_usage(context, self.name,
+                                         tenant_id, in_use=in_use)
+
+    def _resync(self, context, tenant_id, in_use):
+        # Update quota usage
+        usage_info = self._set_quota_usage(
+            context, tenant_id, in_use=in_use)
+        self._dirty_tenants.discard(tenant_id)
+        self._out_of_sync_tenants.discard(tenant_id)
+        LOG.debug(("Unset dirty status for tenant:%(tenant_id)s on "
+                   "resource:%(resource)s"),
+                  {'tenant_id': tenant_id, 'resource': self.name})
+        return usage_info
+
+    def resync(self, context, tenant_id):
+        if tenant_id not in self._out_of_sync_tenants:
+            return
+        LOG.debug(("Synchronizing usage tracker for tenant:%(tenant_id)s on "
+                   "resource:%(resource)s"),
+                  {'tenant_id': tenant_id, 'resource': self.name})
+        in_use = context.session.query(self._model_class).filter_by(
+            tenant_id=tenant_id).count()
+        # Update quota usage
+        return self._resync(context, tenant_id, in_use)
+
+    def count(self, context, _plugin, _resources, tenant_id,
+              resync_usage=False):
+        """Return the current usage count for the resource."""
+        # Load current usage data
+        usage_info = quota_api.get_quota_usage_by_resource_and_tenant(
+            context, self.name, tenant_id)
+        # If dirty or missing, calculate actual resource usage querying
+        # the database and set/create usage info data
+        # NOTE: this routine "trusts" usage counters at service startup. This
+        # assumption is generally valid, but if the database is tampered with,
+        # or if data migrations do not take care of usage counters, the
+        # assumption will not hold anymore
+        if (tenant_id in self._dirty_tenants or not usage_info
+            or usage_info.dirty):
+            LOG.debug(("Usage tracker for resource:%(resource)s and tenant:"
+                       "%(tenant_id)s is out of sync, need to count used "
+                       "quota"), {'resource': self.name,
+                                  'tenant_id': tenant_id})
+            in_use = context.session.query(self._model_class).filter_by(
+                tenant_id=tenant_id).count()
+            # Update quota usage, if requested (by default do not do that, as
+            # typically one counts before adding a record, and that would mark
+            # the usage counter as dirty again)
+            if resync_usage or not usage_info:
+                usage_info = self._resync(context, tenant_id, in_use)
+            else:
+                usage_info = quota_api.QuotaUsageInfo(usage_info.resource,
+                                                      usage_info.tenant_id,
+                                                      in_use,
+                                                      usage_info.reserved,
+                                                      usage_info.dirty)
+
+        return usage_info.total
+
+    def register_events(self):
+        event.listen(self._model_class, 'after_insert', self._db_event_handler)
+        event.listen(self._model_class, 'after_delete', self._db_event_handler)
+
+    def unregister_events(self):
+        event.remove(self._model_class, 'after_insert', self._db_event_handler)
+        event.remove(self._model_class, 'after_delete', self._db_event_handler)
diff --git a/neutron/quota/resource_registry.py b/neutron/quota/resource_registry.py
new file mode 100644 (file)
index 0000000..a0dfeeb
--- /dev/null
@@ -0,0 +1,245 @@
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from oslo_config import cfg
+from oslo_log import log
+import six
+
+from neutron.i18n import _LI, _LW
+from neutron.quota import resource
+
+LOG = log.getLogger(__name__)
+
+
+# Wrappers for easing access to the ResourceRegistry singleton
+
+
+def register_resource(resource):
+    ResourceRegistry.get_instance().register_resource(resource)
+
+
+def register_resource_by_name(resource_name):
+    ResourceRegistry.get_instance().register_resource_by_name(resource_name)
+
+
+def get_all_resources():
+    return ResourceRegistry.get_instance().resources
+
+
+def get_resource(resource_name):
+    return ResourceRegistry.get_instance().get_resource(resource_name)
+
+
+def is_tracked(resource_name):
+    return ResourceRegistry.get_instance().is_tracked(resource_name)
+
+
+# auxiliary functions and decorators
+
+
+def set_resources_dirty(context):
+    """Sets the dirty bit for resources with usage changes.
+
+    This routine scans all registered resources, and, for those whose
+    dirty status is True, sets the dirty bit to True in the database
+    for the appropriate tenants.
+
+    Please note that this routine begins a nested transaction, and it
+    is not recommended that this transaction begins within another
+    transaction. For this reason the function will raise a SqlAlchemy
+    exception if such an attempt is made.
+
+    :param context: a Neutron request context with a DB session
+    """
+    if not cfg.CONF.QUOTAS.track_quota_usage:
+        return
+
+    for res in get_all_resources().values():
+        with context.session.begin():
+            if is_tracked(res.name) and res.dirty:
+                res.mark_dirty(context, nested=True)
+
+
+def resync_resource(context, resource_name, tenant_id):
+    if not cfg.CONF.QUOTAS.track_quota_usage:
+        return
+
+    if is_tracked(resource_name):
+        res = get_resource(resource_name)
+        # If the resource is tracked count supports the resync_usage parameter
+        res.resync(context, tenant_id)
+
+
+def mark_resources_dirty(f):
+    """Decorator for functions which alter resource usage.
+
+    This decorator ensures set_resource_dirty is invoked after completion
+    of the decorated function.
+    """
+
+    @six.wraps(f)
+    def wrapper(_self, context, *args, **kwargs):
+        ret_val = f(_self, context, *args, **kwargs)
+        set_resources_dirty(context)
+        return ret_val
+
+    return wrapper
+
+
+class tracked_resources(object):
+    """Decorator for specifying resources for which usage should be tracked.
+
+    A plugin class can use this decorator to specify for which resources
+    usage info should be tracked into an appropriate table rather than being
+    explicitly counted.
+    """
+
+    def __init__(self, override=False, **kwargs):
+        self._tracked_resources = kwargs
+        self._override = override
+
+    def __call__(self, f):
+
+        @six.wraps(f)
+        def wrapper(*args, **kwargs):
+            registry = ResourceRegistry.get_instance()
+            for resource_name in self._tracked_resources:
+                registry.set_tracked_resource(
+                    resource_name,
+                    self._tracked_resources[resource_name],
+                    self._override)
+            return f(*args, **kwargs)
+
+        return wrapper
+
+
+class ResourceRegistry(object):
+    """Registry for resource subject to quota limits.
+
+    This class keeps track of Neutron resources for which quota limits are
+    enforced, regardless of whether their usage is being tracked or counted.
+
+    For tracked-usage resources, that is to say those resources for which
+    there are usage counters which are kept in sync with the actual number
+    of rows in the database, this class allows the plugin to register their
+    names either explicitly or through the @tracked_resources decorator,
+    which should preferrably be applied to the __init__ method of the class.
+    """
+
+    _instance = None
+
+    @classmethod
+    def get_instance(cls):
+        if cls._instance is None:
+            cls._instance = cls()
+        return cls._instance
+
+    def __init__(self):
+        self._resources = {}
+        # Map usage tracked resources to the correspondent db model class
+        self._tracked_resource_mappings = {}
+
+    def __contains__(self, resource):
+        return resource in self._resources
+
+    def _create_resource_instance(self, resource_name):
+        """Factory function for quota Resource.
+
+        This routine returns a resource instance of the appropriate type
+        according to system configuration.
+
+        If QUOTAS.track_quota_usage is True, and there is a model mapping for
+        the current resource, this function will return an instance of
+        AccountedResource; otherwise an instance of CountableResource.
+        """
+
+        if (not cfg.CONF.QUOTAS.track_quota_usage or
+            resource_name not in self._tracked_resource_mappings):
+            LOG.info(_LI("Creating instance of CountableResource for "
+                         "resource:%s"), resource_name)
+            return resource.CountableResource(
+                resource_name, resource._count_resource,
+                'quota_%s' % resource_name)
+        else:
+            LOG.info(_LI("Creating instance of TrackedResource for "
+                         "resource:%s"), resource_name)
+            return resource.TrackedResource(
+                resource_name,
+                self._tracked_resource_mappings[resource_name],
+                'quota_%s' % resource_name)
+
+    def set_tracked_resource(self, resource_name, model_class, override=False):
+        # Do not do anything if tracking is disabled by config
+        if not cfg.CONF.QUOTAS.track_quota_usage:
+            return
+
+        current_model_class = self._tracked_resource_mappings.setdefault(
+            resource_name, model_class)
+
+        # Check whether setdefault also set the entry in the dict
+        if current_model_class != model_class:
+            LOG.debug("A model class is already defined for %(resource)s: "
+                      "%(current_model_class)s. Override:%(override)s",
+                      {'resource': resource_name,
+                       'current_model_class': current_model_class,
+                       'override': override})
+            if override:
+                self._tracked_resource_mappings[resource_name] = model_class
+        LOG.debug("Tracking information for resource: %s configured",
+                  resource_name)
+
+    def is_tracked(self, resource_name):
+        """Find out if a resource if tracked or not.
+
+        :param resource_name: name of the resource.
+        :returns True if resource_name is registered and tracked, otherwise
+                 False. Please note that here when False it returned it
+                 simply means that resource_name is not a TrackedResource
+                 instance, it does not necessarily mean that the resource
+                 is not registered.
+        """
+        return resource_name in self._tracked_resource_mappings
+
+    def register_resource(self, resource):
+        if resource.name in self._resources:
+            LOG.warn(_LW('%s is already registered'), resource.name)
+        if resource.name in self._tracked_resource_mappings:
+            resource.register_events()
+        self._resources[resource.name] = resource
+
+    def register_resources(self, resources):
+        for res in resources:
+            self.register_resource(res)
+
+    def register_resource_by_name(self, resource_name):
+        """Register a resource by name."""
+        resource = self._create_resource_instance(resource_name)
+        self.register_resource(resource)
+
+    def unregister_resources(self):
+        """Unregister all resources."""
+        for (res_name, res) in self._resources.items():
+            if res_name in self._tracked_resource_mappings:
+                res.unregister_events()
+        self._resources.clear()
+        self._tracked_resource_mappings.clear()
+
+    def get_resource(self, resource_name):
+        """Return a resource given its name.
+
+        :returns: The resource instance or None if the resource is not found
+        """
+        return self._resources.get(resource_name)
+
+    @property
+    def resources(self):
+        return self._resources
index eae4dd21b53f06730f72401f525cf5ead91c0e64..0ee9c2ec3132c7e8384b34d993fe0d1b0589ccf1 100644 (file)
@@ -37,6 +37,7 @@ from neutron import context
 from neutron import manager
 from neutron import policy
 from neutron import quota
+from neutron.quota import resource_registry
 from neutron.tests import base
 from neutron.tests import fake_notifier
 from neutron.tests import tools
@@ -1289,6 +1290,12 @@ class NotificationTest(APIv2TestBase):
 
 
 class DHCPNotificationTest(APIv2TestBase):
+
+    def setUp(self):
+        # This test does not have database support so tracking cannot be used
+        cfg.CONF.set_override('track_quota_usage', False, group='QUOTAS')
+        super(DHCPNotificationTest, self).setUp()
+
     def _test_dhcp_notifier(self, opname, resource, initial_input=None):
         instance = self.plugin.return_value
         instance.get_networks.return_value = initial_input
@@ -1340,6 +1347,23 @@ class DHCPNotificationTest(APIv2TestBase):
 
 
 class QuotaTest(APIv2TestBase):
+
+    def setUp(self):
+        # This test does not have database support so tracking cannot be used
+        cfg.CONF.set_override('track_quota_usage', False, group='QUOTAS')
+        super(QuotaTest, self).setUp()
+        # Use mock to let the API use a different QuotaEngine instance for
+        # unit test in this class. This will ensure resource are registered
+        # again and instanciated with neutron.quota.resource.CountableResource
+        replacement_registry = resource_registry.ResourceRegistry()
+        registry_patcher = mock.patch('neutron.quota.resource_registry.'
+                                      'ResourceRegistry.get_instance')
+        mock_registry = registry_patcher.start().return_value
+        mock_registry.get_resource = replacement_registry.get_resource
+        mock_registry.resources = replacement_registry.resources
+        # Register a resource
+        replacement_registry.register_resource_by_name('network')
+
     def test_create_network_quota(self):
         cfg.CONF.set_override('quota_network', 1, group='QUOTAS')
         initial_input = {'network': {'name': 'net1', 'tenant_id': _uuid()}}
@@ -1384,9 +1408,10 @@ class QuotaTest(APIv2TestBase):
 
 class ExtensionTestCase(base.BaseTestCase):
     def setUp(self):
+        # This test does not have database support so tracking cannot be used
+        cfg.CONF.set_override('track_quota_usage', False, group='QUOTAS')
         super(ExtensionTestCase, self).setUp()
         plugin = 'neutron.neutron_plugin_base_v2.NeutronPluginBaseV2'
-
         # Ensure existing ExtensionManager is not used
         extensions.PluginAwareExtensionManager._instance = None
 
index f289c8b0625d1d338a7813d1e3b8cc7f920093eb..dcf2c8c2385e129fc2687f6fc9fe2eb933c4a4b0 100644 (file)
@@ -18,7 +18,7 @@ import abc
 from neutron.api import extensions
 from neutron.api.v2 import base
 from neutron import manager
-from neutron import quota
+from neutron.quota import resource_registry
 
 
 # Attribute Map
@@ -69,7 +69,7 @@ class Extensionattribute(extensions.ExtensionDescriptor):
         collection_name = resource_name + "s"
         params = RESOURCE_ATTRIBUTE_MAP.get(collection_name, dict())
 
-        quota.QUOTAS.register_resource_by_name(resource_name)
+        resource_registry.register_resource_by_name(resource_name)
 
         controller = base.create_resource(collection_name,
                                           resource_name,
index bf1ac304cae991bd3f3cb7bcf8aa93883ef31aec..e0780e1ee7874e02e30601cba0476c02fba06b71 100644 (file)
@@ -29,6 +29,7 @@ from neutron.common import exceptions
 from neutron import context
 from neutron.db.quota import driver
 from neutron import quota
+from neutron.quota import resource_registry
 from neutron.tests import base
 from neutron.tests import tools
 from neutron.tests.unit.api.v2 import test_base
@@ -64,7 +65,7 @@ class QuotaExtensionTestCase(testlib_api.WebTestCase):
         self.plugin.return_value.supported_extension_aliases = ['quotas']
         # QUOTAS will register the items in conf when starting
         # extra1 here is added later, so have to do it manually
-        quota.QUOTAS.register_resource_by_name('extra1')
+        resource_registry.register_resource_by_name('extra1')
         ext_mgr = extensions.PluginAwareExtensionManager.get_instance()
         app = config.load_paste_app('extensions_test_app')
         ext_middleware = extensions.ExtensionMiddleware(app, ext_mgr=ext_mgr)
diff --git a/neutron/tests/unit/quota/__init__.py b/neutron/tests/unit/quota/__init__.py
new file mode 100644 (file)
index 0000000..3fd4469
--- /dev/null
@@ -0,0 +1,28 @@
+# Copyright (c) 2015 OpenStack Foundation.  All rights reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import sqlalchemy as sa
+
+from neutron.db import model_base
+from neutron.db import models_v2
+
+# Model classes for test resources
+
+
+class MehModel(model_base.BASEV2, models_v2.HasTenant):
+    meh = sa.Column(sa.String(8), primary_key=True)
+
+
+class OtherMehModel(model_base.BASEV2, models_v2.HasTenant):
+    othermeh = sa.Column(sa.String(8), primary_key=True)
diff --git a/neutron/tests/unit/quota/test_resource.py b/neutron/tests/unit/quota/test_resource.py
new file mode 100644 (file)
index 0000000..d1e890c
--- /dev/null
@@ -0,0 +1,232 @@
+# Copyright (c) 2015 OpenStack Foundation.  All rights reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import random
+
+import mock
+from oslo_config import cfg
+
+from neutron import context
+from neutron.db import api as db_api
+from neutron.db.quota import api as quota_api
+from neutron.quota import resource
+from neutron.tests import base
+from neutron.tests.unit import quota as test_quota
+from neutron.tests.unit import testlib_api
+
+
+meh_quota_flag = 'quota_meh'
+meh_quota_opts = [cfg.IntOpt(meh_quota_flag, default=99)]
+random.seed()
+
+
+class TestTrackedResource(testlib_api.SqlTestCaseLight):
+
+    def _add_data(self, tenant_id=None):
+        session = db_api.get_session()
+        with session.begin():
+            tenant_id = tenant_id or self.tenant_id
+            session.add(test_quota.MehModel(
+                meh='meh_%d' % random.randint(0, 10000),
+                tenant_id=tenant_id))
+            session.add(test_quota.MehModel(
+                meh='meh_%d' % random.randint(0, 10000),
+                tenant_id=tenant_id))
+
+    def _delete_data(self):
+        session = db_api.get_session()
+        with session.begin():
+            query = session.query(test_quota.MehModel).filter_by(
+                tenant_id=self.tenant_id)
+            for item in query:
+                session.delete(item)
+
+    def _update_data(self):
+        session = db_api.get_session()
+        with session.begin():
+            query = session.query(test_quota.MehModel).filter_by(
+                tenant_id=self.tenant_id)
+            for item in query:
+                item['meh'] = 'meh-%s' % item['meh']
+                session.add(item)
+
+    def setUp(self):
+        base.BaseTestCase.config_parse()
+        cfg.CONF.register_opts(meh_quota_opts, 'QUOTAS')
+        self.addCleanup(cfg.CONF.reset)
+        self.resource = 'meh'
+        self.other_resource = 'othermeh'
+        self.tenant_id = 'meh'
+        self.context = context.Context(
+            user_id='', tenant_id=self.tenant_id, is_admin=False)
+        super(TestTrackedResource, self).setUp()
+
+    def _register_events(self, res):
+        res.register_events()
+        self.addCleanup(res.unregister_events)
+
+    def _create_resource(self):
+        res = resource.TrackedResource(
+            self.resource, test_quota.MehModel, meh_quota_flag)
+        self._register_events(res)
+        return res
+
+    def _create_other_resource(self):
+        res = resource.TrackedResource(
+            self.other_resource, test_quota.OtherMehModel, meh_quota_flag)
+        self._register_events(res)
+        return res
+
+    def test_count_first_call_with_dirty_false(self):
+        quota_api.set_quota_usage(
+            self.context, self.resource, self.tenant_id, in_use=1)
+        res = self._create_resource()
+        self._add_data()
+        # explicitly set dirty flag to False
+        quota_api.set_all_quota_usage_dirty(
+            self.context, self.resource, dirty=False)
+        # Expect correct count to be returned anyway since the first call to
+        # count() always resyncs with the db
+        self.assertEqual(2, res.count(self.context,
+                                      None, None,
+                                      self.tenant_id))
+
+    def _test_count(self):
+        res = self._create_resource()
+        quota_api.set_quota_usage(
+            self.context, res.name, self.tenant_id, in_use=0)
+        self._add_data()
+        return res
+
+    def test_count_with_dirty_false(self):
+        res = self._test_count()
+        res.count(self.context, None, None, self.tenant_id)
+        # At this stage count has been invoked, and the dirty flag should be
+        # false. Another invocation of count should not query the model class
+        set_quota = 'neutron.db.quota.api.set_quota_usage'
+        with mock.patch(set_quota) as mock_set_quota:
+            self.assertEqual(0, mock_set_quota.call_count)
+            self.assertEqual(2, res.count(self.context,
+                                          None, None,
+                                          self.tenant_id))
+
+    def test_count_with_dirty_true_resync(self):
+        res = self._test_count()
+        # Expect correct count to be returned, which also implies
+        # set_quota_usage has been invoked with the correct parameters
+        self.assertEqual(2, res.count(self.context,
+                                      None, None,
+                                      self.tenant_id,
+                                      resync_usage=True))
+
+    def test_count_with_dirty_true_resync_calls_set_quota_usage(self):
+        res = self._test_count()
+        set_quota_usage = 'neutron.db.quota.api.set_quota_usage'
+        with mock.patch(set_quota_usage) as mock_set_quota_usage:
+            quota_api.set_quota_usage_dirty(self.context,
+                                            self.resource,
+                                            self.tenant_id)
+            res.count(self.context, None, None, self.tenant_id,
+                      resync_usage=True)
+            mock_set_quota_usage.assert_called_once_with(
+                self.context, self.resource, self.tenant_id, in_use=2)
+
+    def test_count_with_dirty_true_no_usage_info(self):
+        res = self._create_resource()
+        self._add_data()
+        # Invoke count without having usage info in DB - Expect correct
+        # count to be returned
+        self.assertEqual(2, res.count(self.context,
+                                      None, None,
+                                      self.tenant_id))
+
+    def test_count_with_dirty_true_no_usage_info_calls_set_quota_usage(self):
+        res = self._create_resource()
+        self._add_data()
+        set_quota_usage = 'neutron.db.quota.api.set_quota_usage'
+        with mock.patch(set_quota_usage) as mock_set_quota_usage:
+            quota_api.set_quota_usage_dirty(self.context,
+                                            self.resource,
+                                            self.tenant_id)
+            res.count(self.context, None, None, self.tenant_id,
+                      resync_usage=True)
+            mock_set_quota_usage.assert_called_once_with(
+                self.context, self.resource, self.tenant_id, in_use=2)
+
+    def test_add_delete_data_triggers_event(self):
+        res = self._create_resource()
+        other_res = self._create_other_resource()
+        # Validate dirty tenants since mock does not work well with sqlalchemy
+        # event handlers.
+        self._add_data()
+        self._add_data('someone_else')
+        self.assertEqual(2, len(res._dirty_tenants))
+        # Also, the dirty flag should not be set for other resources
+        self.assertEqual(0, len(other_res._dirty_tenants))
+        self.assertIn(self.tenant_id, res._dirty_tenants)
+        self.assertIn('someone_else', res._dirty_tenants)
+
+    def test_delete_data_triggers_event(self):
+        res = self._create_resource()
+        self._add_data()
+        self._add_data('someone_else')
+        # Artificially clear _dirty_tenants
+        res._dirty_tenants.clear()
+        self._delete_data()
+        # We did not delete "someone_else", so expect only a single dirty
+        # tenant
+        self.assertEqual(1, len(res._dirty_tenants))
+        self.assertIn(self.tenant_id, res._dirty_tenants)
+
+    def test_update_does_not_trigger_event(self):
+        res = self._create_resource()
+        self._add_data()
+        self._add_data('someone_else')
+        # Artificially clear _dirty_tenants
+        res._dirty_tenants.clear()
+        self._update_data()
+        self.assertEqual(0, len(res._dirty_tenants))
+
+    def test_mark_dirty(self):
+        res = self._create_resource()
+        self._add_data()
+        self._add_data('someone_else')
+        set_quota_usage = 'neutron.db.quota.api.set_quota_usage_dirty'
+        with mock.patch(set_quota_usage) as mock_set_quota_usage:
+            res.mark_dirty(self.context)
+            self.assertEqual(2, mock_set_quota_usage.call_count)
+            mock_set_quota_usage.assert_any_call(
+                self.context, self.resource, self.tenant_id)
+            mock_set_quota_usage.assert_any_call(
+                self.context, self.resource, 'someone_else')
+
+    def test_mark_dirty_no_dirty_tenant(self):
+        res = self._create_resource()
+        set_quota_usage = 'neutron.db.quota.api.set_quota_usage_dirty'
+        with mock.patch(set_quota_usage) as mock_set_quota_usage:
+            res.mark_dirty(self.context)
+            self.assertFalse(mock_set_quota_usage.call_count)
+
+    def test_resync(self):
+        res = self._create_resource()
+        self._add_data()
+        res.mark_dirty(self.context)
+        # self.tenant_id now is out of sync
+        set_quota_usage = 'neutron.db.quota.api.set_quota_usage'
+        with mock.patch(set_quota_usage) as mock_set_quota_usage:
+            res.resync(self.context, self.tenant_id)
+            # and now it should be in sync
+            self.assertNotIn(self.tenant_id, res._out_of_sync_tenants)
+            mock_set_quota_usage.assert_called_once_with(
+                self.context, self.resource, self.tenant_id, in_use=2)
diff --git a/neutron/tests/unit/quota/test_resource_registry.py b/neutron/tests/unit/quota/test_resource_registry.py
new file mode 100644 (file)
index 0000000..6d1d272
--- /dev/null
@@ -0,0 +1,159 @@
+# Copyright (c) 2015 OpenStack Foundation.  All rights reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+import mock
+
+from oslo_config import cfg
+
+from neutron import context
+from neutron.quota import resource
+from neutron.quota import resource_registry
+from neutron.tests import base
+from neutron.tests.unit import quota as test_quota
+
+
+class TestResourceRegistry(base.DietTestCase):
+
+    def setUp(self):
+        super(TestResourceRegistry, self).setUp()
+        self.registry = resource_registry.ResourceRegistry.get_instance()
+        # clean up the registry at every test
+        self.registry.unregister_resources()
+
+    def test_set_tracked_resource_new_resource(self):
+        self.registry.set_tracked_resource('meh', test_quota.MehModel)
+        self.assertEqual(test_quota.MehModel,
+                         self.registry._tracked_resource_mappings['meh'])
+
+    def test_set_tracked_resource_existing_with_override(self):
+        self.test_set_tracked_resource_new_resource()
+        self.registry.set_tracked_resource('meh', test_quota.OtherMehModel,
+                                           override=True)
+        # Overidde is set to True, the model class should change
+        self.assertEqual(test_quota.OtherMehModel,
+                         self.registry._tracked_resource_mappings['meh'])
+
+    def test_set_tracked_resource_existing_no_override(self):
+        self.test_set_tracked_resource_new_resource()
+        self.registry.set_tracked_resource('meh', test_quota.OtherMehModel)
+        # Overidde is set to false, the model class should not change
+        self.assertEqual(test_quota.MehModel,
+                         self.registry._tracked_resource_mappings['meh'])
+
+    def _test_register_resource_by_name(self, resource_name, expected_type):
+        self.assertNotIn(resource_name, self.registry._resources)
+        self.registry.register_resource_by_name(resource_name)
+        self.assertIn(resource_name, self.registry._resources)
+        self.assertIsInstance(self.registry.get_resource(resource_name),
+                              expected_type)
+
+    def test_register_resource_by_name_tracked(self):
+        self.test_set_tracked_resource_new_resource()
+        self._test_register_resource_by_name('meh', resource.TrackedResource)
+
+    def test_register_resource_by_name_not_tracked(self):
+        self._test_register_resource_by_name('meh', resource.CountableResource)
+
+    def test_register_resource_by_name_with_tracking_disabled_by_config(self):
+        cfg.CONF.set_override('track_quota_usage', False,
+                              group='QUOTAS')
+        # DietTestCase does not automatically cleans configuration overrides
+        self.addCleanup(cfg.CONF.reset)
+        self.registry.set_tracked_resource('meh', test_quota.MehModel)
+        self.assertNotIn(
+            'meh', self.registry._tracked_resource_mappings)
+        self._test_register_resource_by_name('meh', resource.CountableResource)
+
+
+class TestAuxiliaryFunctions(base.DietTestCase):
+
+    def setUp(self):
+        super(TestAuxiliaryFunctions, self).setUp()
+        self.registry = resource_registry.ResourceRegistry.get_instance()
+        # clean up the registry at every test
+        self.registry.unregister_resources()
+
+    def test_resync_tracking_disabled(self):
+        cfg.CONF.set_override('track_quota_usage', False,
+                              group='QUOTAS')
+        # DietTestCase does not automatically cleans configuration overrides
+        self.addCleanup(cfg.CONF.reset)
+        with mock.patch('neutron.quota.resource.'
+                        'TrackedResource.resync') as mock_resync:
+            self.registry.set_tracked_resource('meh', test_quota.MehModel)
+            self.registry.register_resource_by_name('meh')
+            resource_registry.resync_resource(mock.ANY, 'meh', 'tenant_id')
+            self.assertEqual(0, mock_resync.call_count)
+
+    def test_resync_tracked_resource(self):
+        with mock.patch('neutron.quota.resource.'
+                        'TrackedResource.resync') as mock_resync:
+            self.registry.set_tracked_resource('meh', test_quota.MehModel)
+            self.registry.register_resource_by_name('meh')
+            resource_registry.resync_resource(mock.ANY, 'meh', 'tenant_id')
+            mock_resync.assert_called_once_with(mock.ANY, 'tenant_id')
+
+    def test_resync_non_tracked_resource(self):
+        with mock.patch('neutron.quota.resource.'
+                        'TrackedResource.resync') as mock_resync:
+            self.registry.register_resource_by_name('meh')
+            resource_registry.resync_resource(mock.ANY, 'meh', 'tenant_id')
+            self.assertEqual(0, mock_resync.call_count)
+
+    def test_set_resources_dirty_invoked_with_tracking_disabled(self):
+        cfg.CONF.set_override('track_quota_usage', False,
+                              group='QUOTAS')
+        # DietTestCase does not automatically cleans configuration overrides
+        self.addCleanup(cfg.CONF.reset)
+        with mock.patch('neutron.quota.resource.'
+                        'TrackedResource.mark_dirty') as mock_mark_dirty:
+            self.registry.set_tracked_resource('meh', test_quota.MehModel)
+            self.registry.register_resource_by_name('meh')
+            resource_registry.set_resources_dirty(mock.ANY)
+            self.assertEqual(0, mock_mark_dirty.call_count)
+
+    def test_set_resources_dirty_no_dirty_resource(self):
+        ctx = context.Context('user_id', 'tenant_id',
+                              is_admin=False, is_advsvc=False)
+        with mock.patch('neutron.quota.resource.'
+                        'TrackedResource.mark_dirty') as mock_mark_dirty:
+            self.registry.set_tracked_resource('meh', test_quota.MehModel)
+            self.registry.register_resource_by_name('meh')
+            res = self.registry.get_resource('meh')
+            # This ensures dirty is false
+            res._dirty_tenants.clear()
+            resource_registry.set_resources_dirty(ctx)
+            self.assertEqual(0, mock_mark_dirty.call_count)
+
+    def test_set_resources_dirty_no_tracked_resource(self):
+        ctx = context.Context('user_id', 'tenant_id',
+                              is_admin=False, is_advsvc=False)
+        with mock.patch('neutron.quota.resource.'
+                        'TrackedResource.mark_dirty') as mock_mark_dirty:
+            self.registry.register_resource_by_name('meh')
+            resource_registry.set_resources_dirty(ctx)
+            self.assertEqual(0, mock_mark_dirty.call_count)
+
+    def test_set_resources_dirty(self):
+        ctx = context.Context('user_id', 'tenant_id',
+                              is_admin=False, is_advsvc=False)
+        with mock.patch('neutron.quota.resource.'
+                        'TrackedResource.mark_dirty') as mock_mark_dirty:
+            self.registry.set_tracked_resource('meh', test_quota.MehModel)
+            self.registry.register_resource_by_name('meh')
+            res = self.registry.get_resource('meh')
+            # This ensures dirty is true
+            res._dirty_tenants.add('tenant_id')
+            resource_registry.set_resources_dirty(ctx)
+            mock_mark_dirty.assert_called_once_with(ctx, nested=True)