From 9cf6e694a37149f2e1e7b70c51c82bab4f135057 Mon Sep 17 00:00:00 2001 From: Michal Dulko Date: Thu, 12 Mar 2015 16:07:03 +0100 Subject: [PATCH] Add is_ready method to scheduler driver This commit adds is_ready method to scheduler driver to indicate if driver is ready to accept requests. Scheduler is considered ready when host_manager received capabilities for all active volume services. Partial-Bug: 1409012 Change-Id: I4a90e68a3836c44038ee8c937ae1ecf40e5c1f32 --- cinder/scheduler/driver.py | 9 +++ cinder/scheduler/host_manager.py | 16 +++++ cinder/tests/scheduler/test_host_manager.py | 76 +++++++++++++++++---- cinder/tests/scheduler/test_scheduler.py | 7 ++ 4 files changed, 94 insertions(+), 14 deletions(-) diff --git a/cinder/scheduler/driver.py b/cinder/scheduler/driver.py index c27fef152..f0db67ca8 100644 --- a/cinder/scheduler/driver.py +++ b/cinder/scheduler/driver.py @@ -69,6 +69,15 @@ class Scheduler(object): CONF.scheduler_host_manager) self.volume_rpcapi = volume_rpcapi.VolumeAPI() + def is_ready(self): + """Returns True if Scheduler is ready to accept requests. + + This is to handle scheduler service startup when it has no volume hosts + stats and will fail all the requests. + """ + + return self.host_manager.has_all_capabilities() + def update_service_capabilities(self, service_name, host, capabilities): """Process a capability update from a service node.""" self.host_manager.update_service_capabilities(service_name, diff --git a/cinder/scheduler/host_manager.py b/cinder/scheduler/host_manager.py index 1b8d1d1d3..3665e0177 100644 --- a/cinder/scheduler/host_manager.py +++ b/cinder/scheduler/host_manager.py @@ -23,6 +23,7 @@ from oslo_config import cfg from oslo_log import log as logging from oslo_utils import timeutils +from cinder import context as cinder_context from cinder import db from cinder import exception from cinder.i18n import _LI, _LW @@ -373,6 +374,9 @@ class HostManager(object): # Do nothing when some other scheduler is configured pass + self._no_capabilities_hosts = set() # Hosts having no capabilities + self._update_host_state_map(cinder_context.get_admin_context()) + def _choose_host_filters(self, filter_cls_names): """Return a list of available filter names. @@ -462,6 +466,11 @@ class HostManager(object): {'service_name': service_name, 'host': host, 'cap': capabilities}) + self._no_capabilities_hosts.discard(host) + + def has_all_capabilities(self): + return len(self._no_capabilities_hosts) == 0 + def _update_host_state_map(self, context): # Get resource usage across the available volume nodes: @@ -470,12 +479,17 @@ class HostManager(object): topic, disabled=False) active_hosts = set() + no_capabilities_hosts = set() for service in volume_services: host = service['host'] if not utils.service_is_up(service): LOG.warn(_LW("volume service is down. (host: %s)") % host) continue capabilities = self.service_states.get(host, None) + if capabilities is None: + no_capabilities_hosts.add(host) + continue + host_state = self.host_state_map.get(host) if not host_state: host_state = self.host_state_cls(host, @@ -489,6 +503,8 @@ class HostManager(object): dict(service.iteritems())) active_hosts.add(host) + self._no_capabilities_hosts = no_capabilities_hosts + # remove non-active hosts from host_state_map nonactive_hosts = set(self.host_state_map.keys()) - active_hosts for host in nonactive_hosts: diff --git a/cinder/tests/scheduler/test_host_manager.py b/cinder/tests/scheduler/test_host_manager.py index 38f58c133..f6a8c7a0e 100644 --- a/cinder/tests/scheduler/test_host_manager.py +++ b/cinder/tests/scheduler/test_host_manager.py @@ -116,6 +116,39 @@ class HostManagerTestCase(test.TestCase): 'host3': host3_volume_capabs} self.assertDictMatch(service_states, expected) + @mock.patch('cinder.utils.service_is_up') + @mock.patch('cinder.db.service_get_all_by_topic') + def test_has_all_capabilities(self, _mock_service_get_all_by_topic, + _mock_service_is_up): + _mock_service_is_up.return_value = True + services = [ + dict(id=1, host='host1', topic='volume', disabled=False, + availability_zone='zone1', updated_at=timeutils.utcnow()), + dict(id=2, host='host2', topic='volume', disabled=False, + availability_zone='zone1', updated_at=timeutils.utcnow()), + dict(id=3, host='host3', topic='volume', disabled=False, + availability_zone='zone1', updated_at=timeutils.utcnow()), + ] + _mock_service_get_all_by_topic.return_value = services + # Create host_manager again to let db.service_get_all_by_topic mock run + self.host_manager = host_manager.HostManager() + self.assertFalse(self.host_manager.has_all_capabilities()) + + host1_volume_capabs = dict(free_capacity_gb=4321, timestamp=1) + host2_volume_capabs = dict(free_capacity_gb=5432, timestamp=1) + host3_volume_capabs = dict(free_capacity_gb=6543, timestamp=1) + + service_name = 'volume' + self.host_manager.update_service_capabilities(service_name, 'host1', + host1_volume_capabs) + self.assertFalse(self.host_manager.has_all_capabilities()) + self.host_manager.update_service_capabilities(service_name, 'host2', + host2_volume_capabs) + self.assertFalse(self.host_manager.has_all_capabilities()) + self.host_manager.update_service_capabilities(service_name, 'host3', + host3_volume_capabs) + self.assertTrue(self.host_manager.has_all_capabilities()) + @mock.patch('cinder.db.service_get_all_by_topic') @mock.patch('cinder.utils.service_is_up') @mock.patch('oslo_utils.timeutils.utcnow') @@ -185,7 +218,24 @@ class HostManagerTestCase(test.TestCase): availability_zone='zone3', updated_at=timeutils.utcnow()), ] - # First test: service_is_up is always True, host5 is disabled + service_states = { + 'host1': dict(volume_backend_name='AAA', + total_capacity_gb=512, free_capacity_gb=200, + timestamp=None, reserved_percentage=0, + provisioned_capacity_gb=312), + 'host2': dict(volume_backend_name='BBB', + total_capacity_gb=256, free_capacity_gb=100, + timestamp=None, reserved_percentage=0, + provisioned_capacity_gb=156), + 'host3': dict(volume_backend_name='CCC', + total_capacity_gb=10000, free_capacity_gb=700, + timestamp=None, reserved_percentage=0, + provisioned_capacity_gb=9300), + } + + # First test: service_is_up is always True, host5 is disabled, + # host4 has no capabilities + self.host_manager.service_states = service_states _mock_service_get_all_by_topic.return_value = services _mock_service_is_up.return_value = True _mock_warning = mock.Mock() @@ -203,19 +253,19 @@ class HostManagerTestCase(test.TestCase): # Get host_state_map and make sure we have the first 4 hosts host_state_map = self.host_manager.host_state_map - self.assertEqual(len(host_state_map), 4) - for i in xrange(4): + self.assertEqual(len(host_state_map), 3) + for i in xrange(3): volume_node = services[i] host = volume_node['host'] self.assertEqual(host_state_map[host].service, volume_node) - # Second test: Now service_is_up returns False for host4 + # Second test: Now service_is_up returns False for host3 _mock_service_is_up.reset_mock() - _mock_service_is_up.side_effect = [True, True, True, False] + _mock_service_is_up.side_effect = [True, True, False, True] _mock_service_get_all_by_topic.reset_mock() _mock_warning.reset_mock() - # Get all states, make sure host 4 is reported as down + # Get all states, make sure host 3 is reported as down self.host_manager.get_all_host_states(context) _mock_service_get_all_by_topic.assert_called_with(context, topic, @@ -224,16 +274,14 @@ class HostManagerTestCase(test.TestCase): for service in services: expected.append(mock.call(service)) self.assertEqual(expected, _mock_service_is_up.call_args_list) - expected = [] - for num in ['4']: - expected.append(mock.call("volume service is down. " - "(host: host" + num + ")")) - self.assertEqual(expected, _mock_warning.call_args_list) + _mock_warning.assert_called_once_with("volume service is down. " + "(host: host3)") - # Get host_state_map and make sure we have the first 4 hosts + # Get host_state_map and make sure we have the first 2 hosts (host3 is + # down, host4 is missing capabilities) host_state_map = self.host_manager.host_state_map - self.assertEqual(len(host_state_map), 3) - for i in xrange(3): + self.assertEqual(len(host_state_map), 2) + for i in xrange(2): volume_node = services[i] host = volume_node['host'] self.assertEqual(host_state_map[host].service, diff --git a/cinder/tests/scheduler/test_scheduler.py b/cinder/tests/scheduler/test_scheduler.py index 6fa261b5d..784c1fc2b 100644 --- a/cinder/tests/scheduler/test_scheduler.py +++ b/cinder/tests/scheduler/test_scheduler.py @@ -255,6 +255,13 @@ class SchedulerTestCase(test.TestCase): _mock_update_cap.assert_called_once_with(service_name, host, capabilities) + @mock.patch('cinder.scheduler.host_manager.HostManager.' + 'has_all_capabilities', return_value=False) + def test_is_ready(self, _mock_has_caps): + ready = self.driver.is_ready() + _mock_has_caps.assert_called_once_with() + self.assertFalse(ready) + class SchedulerDriverBaseTestCase(SchedulerTestCase): """Test cases for base scheduler driver class methods -- 2.45.2