From: Jay S. Bryant <jsbryant@us.ibm.com>
Date: Mon, 22 Jun 2015 16:43:37 +0000 (-0500)
Subject: Add exception catch in report_state for DBError
X-Git-Url: https://review.fuel-infra.org/gitweb?a=commitdiff_plain;h=3e4caa552969dd298306b124bd2dc02e7a62c835;p=openstack-build%2Fcinder-build.git

Add exception catch in report_state for DBError

We discovered while testing Cinder in an HA
environment that transient DB errors can be
encountered that are not currently covered by
the exception catch in service.py report_state().
The uncaught exceptions were causing the thread
for report_state to prematurely exit and causing
services to no longer update the DB when, in fact,
they were still usable.

This change adds an exception catch for DBError
so that the thread can continue to function.

Change-Id: I95f79b8d6c8f5d7b3e44665306b500b8d2ce3c7c
Closes-bug: 1466991
---

diff --git a/cinder/service.py b/cinder/service.py
index 2ad5878ad..434d2ecce 100644
--- a/cinder/service.py
+++ b/cinder/service.py
@@ -325,6 +325,13 @@ class Service(service.Service):
                 self.model_disconnected = True
                 LOG.exception(_LE('model server went away'))
 
+        # NOTE(jsbryant) Other DB errors can happen in HA configurations.
+        # such errors shouldn't kill this thread, so we handle them here.
+        except db_exc.DBError:
+            if not getattr(self, 'model_disconnected', False):
+                self.model_disconnected = True
+                LOG.exception(_LE('DBError encountered: '))
+
 
 class WSGIService(object):
     """Provides ability to launch API from a 'paste' configuration."""
diff --git a/cinder/tests/unit/test_service.py b/cinder/tests/unit/test_service.py
index 8e101f0fc..5c95ed059 100644
--- a/cinder/tests/unit/test_service.py
+++ b/cinder/tests/unit/test_service.py
@@ -150,6 +150,29 @@ class ServiceTestCase(test.TestCase):
             self.assertTrue(serv.model_disconnected)
             self.assertFalse(mock_db.service_update.called)
 
+    def test_report_state_disconnected_DBError(self):
+        service_ref = {'host': self.host,
+                       'binary': self.binary,
+                       'topic': self.topic,
+                       'report_count': 0,
+                       'availability_zone': 'nova',
+                       'id': 1}
+        with mock.patch.object(service, 'db') as mock_db:
+            mock_db.service_get_by_args.side_effect = exception.NotFound()
+            mock_db.service_create.return_value = service_ref
+            mock_db.service_get.side_effect = db_exc.DBError()
+
+            serv = service.Service(
+                self.host,
+                self.binary,
+                self.topic,
+                'cinder.tests.unit.test_service.FakeManager'
+            )
+            serv.start()
+            serv.report_state()
+            self.assertTrue(serv.model_disconnected)
+            self.assertFalse(mock_db.service_update.called)
+
     def test_report_state_newly_connected(self):
         service_ref = {'host': self.host,
                        'binary': self.binary,