Add exception catch in report_state for DBError

We discovered while testing Cinder in an HA
environment that transient DB errors can be
encountered that are not currently covered by
the exception catch in service.py report_state().
The uncaught exceptions were causing the thread
for report_state to prematurely exit and causing
services to no longer update the DB when, in fact,
they were still usable.

This change adds an exception catch for DBError
so that the thread can continue to function.

Change-Id: I95f79b8d6c8f5d7b3e44665306b500b8d2ce3c7c
Closes-bug: 1466991

diff --git a/cinder/service.py b/cinder/service.py
index 2ad5878ad30590c45a20708b34f4e3e5a1e88af4..434d2eccedf2a8a519815d67b9cf0ec3096977bb 100644 (file)
--- a/cinder/service.py
+++ b/cinder/service.py
@@ -325,6 +325,13 @@ class Service(service.Service):
                 self.model_disconnected = True
                 LOG.exception(_LE('model server went away'))
 
+        # NOTE(jsbryant) Other DB errors can happen in HA configurations.
+        # such errors shouldn't kill this thread, so we handle them here.
+        except db_exc.DBError:
+            if not getattr(self, 'model_disconnected', False):
+                self.model_disconnected = True
+                LOG.exception(_LE('DBError encountered: '))
+
 
 class WSGIService(object):
     """Provides ability to launch API from a 'paste' configuration."""

diff --git a/cinder/tests/unit/test_service.py b/cinder/tests/unit/test_service.py
index 8e101f0fc0440783bd856b07cbebc3f5f388bfe1..5c95ed0593fa06d7af6309e8376cacd9ea582585 100644 (file)
--- a/cinder/tests/unit/test_service.py
+++ b/cinder/tests/unit/test_service.py
@@ -150,6 +150,29 @@ class ServiceTestCase(test.TestCase):
             self.assertTrue(serv.model_disconnected)
             self.assertFalse(mock_db.service_update.called)
 
+    def test_report_state_disconnected_DBError(self):
+        service_ref = {'host': self.host,
+                       'binary': self.binary,
+                       'topic': self.topic,
+                       'report_count': 0,
+                       'availability_zone': 'nova',
+                       'id': 1}
+        with mock.patch.object(service, 'db') as mock_db:
+            mock_db.service_get_by_args.side_effect = exception.NotFound()
+            mock_db.service_create.return_value = service_ref
+            mock_db.service_get.side_effect = db_exc.DBError()
+
+            serv = service.Service(
+                self.host,
+                self.binary,
+                self.topic,
+                'cinder.tests.unit.test_service.FakeManager'
+            )
+            serv.start()
+            serv.report_state()
+            self.assertTrue(serv.model_disconnected)
+            self.assertFalse(mock_db.service_update.called)
+
     def test_report_state_newly_connected(self):
         service_ref = {'host': self.host,
                        'binary': self.binary,