From 3e4caa552969dd298306b124bd2dc02e7a62c835 Mon Sep 17 00:00:00 2001 From: "Jay S. Bryant" Date: Mon, 22 Jun 2015 11:43:37 -0500 Subject: [PATCH] Add exception catch in report_state for DBError We discovered while testing Cinder in an HA environment that transient DB errors can be encountered that are not currently covered by the exception catch in service.py report_state(). The uncaught exceptions were causing the thread for report_state to prematurely exit and causing services to no longer update the DB when, in fact, they were still usable. This change adds an exception catch for DBError so that the thread can continue to function. Change-Id: I95f79b8d6c8f5d7b3e44665306b500b8d2ce3c7c Closes-bug: 1466991 --- cinder/service.py | 7 +++++++ cinder/tests/unit/test_service.py | 23 +++++++++++++++++++++++ 2 files changed, 30 insertions(+) diff --git a/cinder/service.py b/cinder/service.py index 2ad5878ad..434d2ecce 100644 --- a/cinder/service.py +++ b/cinder/service.py @@ -325,6 +325,13 @@ class Service(service.Service): self.model_disconnected = True LOG.exception(_LE('model server went away')) + # NOTE(jsbryant) Other DB errors can happen in HA configurations. + # such errors shouldn't kill this thread, so we handle them here. + except db_exc.DBError: + if not getattr(self, 'model_disconnected', False): + self.model_disconnected = True + LOG.exception(_LE('DBError encountered: ')) + class WSGIService(object): """Provides ability to launch API from a 'paste' configuration.""" diff --git a/cinder/tests/unit/test_service.py b/cinder/tests/unit/test_service.py index 8e101f0fc..5c95ed059 100644 --- a/cinder/tests/unit/test_service.py +++ b/cinder/tests/unit/test_service.py @@ -150,6 +150,29 @@ class ServiceTestCase(test.TestCase): self.assertTrue(serv.model_disconnected) self.assertFalse(mock_db.service_update.called) + def test_report_state_disconnected_DBError(self): + service_ref = {'host': self.host, + 'binary': self.binary, + 'topic': self.topic, + 'report_count': 0, + 'availability_zone': 'nova', + 'id': 1} + with mock.patch.object(service, 'db') as mock_db: + mock_db.service_get_by_args.side_effect = exception.NotFound() + mock_db.service_create.return_value = service_ref + mock_db.service_get.side_effect = db_exc.DBError() + + serv = service.Service( + self.host, + self.binary, + self.topic, + 'cinder.tests.unit.test_service.FakeManager' + ) + serv.start() + serv.report_state() + self.assertTrue(serv.model_disconnected) + self.assertFalse(mock_db.service_update.called) + def test_report_state_newly_connected(self): service_ref = {'host': self.host, 'binary': self.binary, -- 2.45.2