This patch deals with the lock wait timeout and the deadlock errors
observed under high concurrency (api_workers >= 4) with the pymysql
driver. It includes the following changes:
- Stop setting dirty status for resource usage when creating
reservation, as usage of reserved resources is not tracked anymore;
- Add a variable, increasing delay when retrying make_reservation
upon a DBDeadlock error in order to reduce the chances of further
collisions;
- Enable transaction retry upon DBDeadlock errors for set_quota_usage;
- Do not resync quota usage while making reservation. This puts a lot
of stress on the database and is also wasteful since resource usage
is very likely to change again once the transaction is committed;
- Use autonested_transaction to simplify logic around when the
nested flag should be used.
Change-Id: I7a335f9ebea3c0d6fee6e6b757554e045a66075c
Closes-Bug: #
1486134
Related-Blueprint: better-quotas
from sqlalchemy.orm import exc as orm_exc
from sqlalchemy import sql
+from neutron.db import api as db_api
from neutron.db import common_db_mixin as common_db_api
from neutron.db.quota import models as quota_models
:param delta: Specifies whether in_use is an absolute number
or a delta (default to False)
"""
- query = common_db_api.model_query(context, quota_models.QuotaUsage)
- query = query.filter_by(resource=resource).filter_by(tenant_id=tenant_id)
- usage_data = query.first()
- with context.session.begin(subtransactions=True):
+ with db_api.autonested_transaction(context.session):
+ query = common_db_api.model_query(context, quota_models.QuotaUsage)
+ query = query.filter_by(resource=resource).filter_by(
+ tenant_id=tenant_id)
+ usage_data = query.first()
if not usage_data:
# Must create entry
usage_data = quota_models.QuotaUsage(
quota_models.ResourceDelta(resource=resource,
amount=delta,
reservation=resv))
- # quota_usage for all resources involved in this reservation must
- # be marked as dirty
- set_resources_quota_usage_dirty(
- context, deltas.keys(), tenant_id)
return ReservationInfo(resv['id'],
resv['tenant_id'],
resv['expiration'],
quota_models.ResourceDelta.resource,
quota_models.Reservation.expiration)
return dict((resource, total_reserved)
- for (resource, exp, total_reserved) in resv_query)
+ for (resource, exp, total_reserved) in resv_query)
def remove_expired_reservations(context, tenant_id=None):
context, tenant_id=tenant_id)
@oslo_db_api.wrap_db_retry(max_retries=db_api.MAX_RETRIES,
+ retry_interval=0.1,
+ inc_retry_interval=True,
retry_on_request=True,
retry_on_deadlock=True)
def make_reservation(self, context, tenant_id, resources, deltas, plugin):
# locks should be ok to use when support for sending "hotspot" writes
# to a single node will be avaialable.
requested_resources = deltas.keys()
- with context.session.begin():
+ with db_api.autonested_transaction(context.session):
# Gather current usage information
# TODO(salv-orlando): calling count() for every resource triggers
# multiple queries on quota usage. This should be improved, however
# instances
current_usages = dict(
(resource, resources[resource].count(
- context, plugin, tenant_id)) for
+ context, plugin, tenant_id, resync_usage=False)) for
resource in requested_resources)
# get_tenant_quotes needs in inout a dictionary mapping resource
# name to BaseResosurce instances so that the default quota can be
name, flag=flag, plural_name=plural_name)
self._count_func = count
- def count(self, context, plugin, tenant_id):
+ def count(self, context, plugin, tenant_id, **kwargs):
return self._count_func(context, plugin, self.plural_name, tenant_id)
def dirty(self):
return self._dirty_tenants
- def mark_dirty(self, context, nested=False):
+ def mark_dirty(self, context):
if not self._dirty_tenants:
return
- with context.session.begin(nested=nested, subtransactions=True):
+ with db_api.autonested_transaction(context.session):
# It is not necessary to protect this operation with a lock.
# Indeed when this method is called the request has been processed
# and therefore all resources created or deleted.
# ensure that an UPDATE statement is emitted rather than an INSERT one
@oslo_db_api.wrap_db_retry(
max_retries=db_api.MAX_RETRIES,
+ retry_on_deadlock=True,
exception_checker=lambda exc:
isinstance(exc, oslo_db_exception.DBDuplicateEntry))
def _set_quota_usage(self, context, tenant_id, in_use):
# Update quota usage
return self._resync(context, tenant_id, in_use)
- def count(self, context, _plugin, tenant_id, resync_usage=False):
+ def count(self, context, _plugin, tenant_id, resync_usage=True):
"""Return the current usage count for the resource.
This method will fetch aggregate information for resource usage
# Update quota usage, if requested (by default do not do that, as
# typically one counts before adding a record, and that would mark
# the usage counter as dirty again)
- if resync_usage or not usage_info:
+ if resync_usage:
usage_info = self._resync(context, tenant_id, in_use)
else:
- # NOTE(salv-orlando): Passing 0 for reserved amount as
- # reservations are currently not supported
- usage_info = quota_api.QuotaUsageInfo(usage_info.resource,
- usage_info.tenant_id,
- in_use,
- usage_info.dirty)
+ resource = usage_info.resource if usage_info else self.name
+ tenant_id = usage_info.tenant_id if usage_info else tenant_id
+ dirty = usage_info.dirty if usage_info else True
+ usage_info = quota_api.QuotaUsageInfo(
+ resource, tenant_id, in_use, dirty)
LOG.debug(("Quota usage for %(resource)s was recalculated. "
"Used quota:%(used)d."),
for res in get_all_resources().values():
with context.session.begin(subtransactions=True):
if is_tracked(res.name) and res.dirty:
- res.mark_dirty(context, nested=True)
+ res.mark_dirty(context)
def resync_resource(context, resource_name, tenant_id):
self.assertIsNone(quota_api.get_reservations_for_resources(
self.context, self.tenant_id, []))
- def _test_remove_reservation(self, set_dirty):
- resources = {'goals': 2, 'assists': 1}
- resv = self._create_reservation(resources)
- self.assertEqual(1, quota_api.remove_reservation(
- self.context, resv.reservation_id, set_dirty=set_dirty))
-
- def test_remove_reservation(self):
- self._test_remove_reservation(False)
-
- def test_remove_reservation_and_set_dirty(self):
- routine = 'neutron.db.quota.api.set_resources_quota_usage_dirty'
- with mock.patch(routine) as mock_routine:
- self._test_remove_reservation(False)
- mock_routine.assert_called_once_with(
- self.context, mock.ANY, self.tenant_id)
-
def test_remove_expired_reservations(self):
with mock.patch('neutron.db.quota.api.utcnow') as mock_utcnow:
mock_utcnow.return_value = datetime.datetime(
# This ensures dirty is true
res._dirty_tenants.add('tenant_id')
resource_registry.set_resources_dirty(ctx)
- mock_mark_dirty.assert_called_once_with(ctx, nested=True)
+ mock_mark_dirty.assert_called_once_with(ctx)
Code Review / openstack-build / neutron-build.git / commitdiff