From: Subramanian Neelakantan Date: Thu, 6 Feb 2014 06:07:27 +0000 (+0530) Subject: vmware: intermittent failure in test_vmware_vmdk X-Git-Url: https://review.fuel-infra.org/gitweb?a=commitdiff_plain;h=50a126735f1f695722fce505aa3b497fd8509c12;p=openstack-build%2Fcinder-build.git vmware: intermittent failure in test_vmware_vmdk There was a race condition between the two Event() objects being created within loopingcall and Retry in api module. Did away with the Event object in Retry and used loopingcall the right way. Change-Id: I2534bad9294747fcd618ba3634abb01ceef92213 --- diff --git a/cinder/tests/test_vmware_vmdk.py b/cinder/tests/test_vmware_vmdk.py index 3642454f4..6ff8f286b 100644 --- a/cinder/tests/test_vmware_vmdk.py +++ b/cinder/tests/test_vmware_vmdk.py @@ -162,6 +162,7 @@ class VMwareEsxVmdkDriverTestCase(test.TestCase): def __init__(self): self.counter1 = 0 + self.counter2 = 0 @api.Retry(max_retry_count=2, inc_sleep_time=0.001, exceptions=(Exception)) @@ -169,9 +170,16 @@ class VMwareEsxVmdkDriverTestCase(test.TestCase): self.counter1 += 1 raise exception.CinderException('Fail') + @api.Retry(max_retry_count=2) + def success(self): + self.counter2 += 1 + return self.counter2 + test_obj = TestClass() self.assertRaises(exception.CinderException, test_obj.fail) self.assertEqual(test_obj.counter1, 3) + ret = test_obj.success() + self.assertEqual(1, ret) def test_create_session(self): """Test create_session.""" diff --git a/cinder/volume/drivers/vmware/api.py b/cinder/volume/drivers/vmware/api.py index 7b300193a..e7ca916b0 100644 --- a/cinder/volume/drivers/vmware/api.py +++ b/cinder/volume/drivers/vmware/api.py @@ -20,8 +20,6 @@ Session and API call management for VMware ESX/VC server. Provides abstraction over cinder.volume.drivers.vmware.vim.Vim SOAP calls. """ -from eventlet import event - from cinder.openstack.common import log as logging from cinder.openstack.common import loopingcall from cinder.volume.drivers.vmware import error_util @@ -67,32 +65,29 @@ class Retry(object): def __call__(self, f): - def _func(done, *args, **kwargs): + def _func(*args, **kwargs): try: result = f(*args, **kwargs) - done.send(result) except self._exceptions as excep: LOG.exception(_("Failure while invoking function: " "%(func)s. Error: %(excep)s.") % {'func': f.__name__, 'excep': excep}) if (self._max_retry_count != -1 and self._retry_count >= self._max_retry_count): - done.send_exception(excep) + raise excep else: self._retry_count += 1 self._sleep_time += self._inc_sleep_time return self._sleep_time except Exception as excep: - done.send_exception(excep) - return 0 + raise excep + # got result. Stop the loop. + raise loopingcall.LoopingCallDone(result) def func(*args, **kwargs): - done = event.Event() - loop = loopingcall.DynamicLoopingCall(_func, done, *args, **kwargs) - loop.start(periodic_interval_max=self._max_sleep_time) - result = done.wait() - loop.stop() - return result + loop = loopingcall.DynamicLoopingCall(_func, *args, **kwargs) + timer = loop.start(periodic_interval_max=self._max_sleep_time) + return timer.wait() return func @@ -231,15 +226,10 @@ class VMwareAPISession(object): :param task: Managed object reference of the task :return: Task info upon successful completion of the task """ - done = event.Event() - loop = loopingcall.FixedIntervalLoopingCall(self._poll_task, - task, done) - loop.start(self._task_poll_interval) - task_info = done.wait() - loop.stop() - return task_info - - def _poll_task(self, task, done): + loop = loopingcall.FixedIntervalLoopingCall(self._poll_task, task) + return loop.start(self._task_poll_interval).wait() + + def _poll_task(self, task): """Poll the given task. If the task completes successfully then returns task info. @@ -260,36 +250,29 @@ class VMwareAPISession(object): return elif task_info.state == 'success': LOG.debug(_("Task %s status: success.") % task) - done.send(task_info) else: error_msg = str(task_info.error.localizedMessage) LOG.exception(_("Task: %(task)s failed with error: %(err)s.") % {'task': task, 'err': error_msg}) - done.send_exception(error_util.VimFaultException([], - error_msg)) + raise error_util.VimFaultException([], error_msg) except Exception as excep: LOG.exception(_("Task: %(task)s failed with error: %(err)s.") % {'task': task, 'err': excep}) - done.send_exception(excep) + raise excep + # got the result. So stop the loop. + raise loopingcall.LoopingCallDone(task_info) def wait_for_lease_ready(self, lease): - done = event.Event() - loop = loopingcall.FixedIntervalLoopingCall(self._poll_lease, - lease, - done) - loop.start(self._task_poll_interval) - done.wait() - loop.stop() - - def _poll_lease(self, lease, done): + loop = loopingcall.FixedIntervalLoopingCall(self._poll_lease, lease) + return loop.start(self._task_poll_interval).wait() + + def _poll_lease(self, lease): try: state = self.invoke_api(vim_util, 'get_object_property', self.vim, lease, 'state') if state == 'ready': # done LOG.debug(_("Lease is ready.")) - done.send() - return elif state == 'initializing': LOG.debug(_("Lease initializing...")) return @@ -298,11 +281,13 @@ class VMwareAPISession(object): self.vim, lease, 'error') LOG.exception(error_msg) excep = error_util.VimFaultException([], error_msg) - done.send_exception(excep) + raise excep else: # unknown state - complain error_msg = _("Error: unknown lease state %s.") % state raise error_util.VimFaultException([], error_msg) except Exception as excep: LOG.exception(excep) - done.send_exception(excep) + raise excep + # stop the loop since state is ready + raise loopingcall.LoopingCallDone()