From 85ddbde058d8bda0b938eb7a45ef73519a831b3b Mon Sep 17 00:00:00 2001 From: armando-migliaccio Date: Fri, 14 Feb 2014 10:59:17 -0800 Subject: [PATCH] Fix request timeout errors during calls to NSX controller MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit Sometimes two correlated exception traces are observed in the server log for the Neutron Server backed by NSX: RequestTimeout (The nsx request has timed out) and OperationalError (Lock wait timeout exceeded). This is generally described by Guru Salvatore Orlando as the, and I quote, the "infamous eventlet-mysql deadlock". This patch tries to address the issue by adding a cooperative yield in the nsx client code (it’s a good idea to call sleep(0) occasionally in any case) and also by avoiding the unnecessary spawning of another Greenthread within a call that is already executed in Greenthred itself. Closes-bug: #1267101 Related-bug: #1279497 Change-Id: I8e298468fb730f11a66fbd4211121ee7d3e2a548 --- neutron/plugins/nicira/api_client/request.py | 4 +++- neutron/plugins/nicira/api_client/request_eventlet.py | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/neutron/plugins/nicira/api_client/request.py b/neutron/plugins/nicira/api_client/request.py index 4cc00f868..a450e325b 100644 --- a/neutron/plugins/nicira/api_client/request.py +++ b/neutron/plugins/nicira/api_client/request.py @@ -21,6 +21,7 @@ from abc import ABCMeta from abc import abstractmethod import copy +import eventlet import httplib import logging import time @@ -186,7 +187,8 @@ class NvpApiRequest(object): LOG.info(_("[%(rid)d] Redirecting request to: %(conn)s"), {'rid': self._rid(), 'conn': self._request_str(conn, url)}) - + # yield here, just in case we are not out of the loop yet + eventlet.greenthread.sleep(0) # If we receive any of these responses, then # our server did not process our request and may be in an # errored state. Raise an exception, which will cause the diff --git a/neutron/plugins/nicira/api_client/request_eventlet.py b/neutron/plugins/nicira/api_client/request_eventlet.py index 0902feb68..2097ea1c5 100644 --- a/neutron/plugins/nicira/api_client/request_eventlet.py +++ b/neutron/plugins/nicira/api_client/request_eventlet.py @@ -139,9 +139,10 @@ class NvpApiRequestEventlet(request.NvpApiRequest): attempt = 0 response = None while response is None and attempt <= self._retries: + eventlet.greenthread.sleep(0) attempt += 1 - req = self.spawn(self._issue_request).wait() + req = self._issue_request() # automatically raises any exceptions returned. if isinstance(req, httplib.HTTPResponse): if attempt <= self._retries and not self._abort: -- 2.45.2