From 0e2783360ce730beed3423bee31ad9726a51c8e1 Mon Sep 17 00:00:00 2001 From: Mitsuhiro Tanino Date: Mon, 13 Jul 2015 11:17:00 -0400 Subject: [PATCH] Adds framework for get_capabilities() feature This patch adds a base framework of capabilities reporting feature. The get_capabilities RPC API returns dictionary which is consisted of two parts. First part includes static backend capabilities which are obtained by get_volume_stats(). Second part is properties which includes parameters correspond to extra specs. This properties part is consisted of cinder standard capabilities and vendor unique properties. These properties are created via these two methods. * _init_standard_capabilities() * _init_vendor_properties() Since _init_standard_capabilities() only exposes cinder standard capabilities into the properties dictionary, each backend driver needs to expose their own properties by overriding _init_vendor_properties(). ex. capabilities { 'host_name': 'block1', 'volume_backend_name': 'lvm', 'pool_name': 'pool', 'driver_version': '2.0.0', 'storage_protocol': 'iSCSI', 'properties:' { 'thin_provisioning': { 'title': 'Thin Provisioning', 'description': 'Sets thin provisioning.', 'type': 'boolean'}, 'compression': { 'title': 'Compression', 'description': 'Enables compression.', 'type': 'boolean'}, 'qos': { 'title': 'QoS', 'description': 'Enables QoS.', 'type': 'boolean'}, 'replication': { 'title': 'Replication', 'description': 'Enables replication.', 'type': 'boolean'}, 'vendor:compression_type': { 'title': 'Compression type', 'description': 'Specifies compression type.', 'type': 'string', 'enum': ["lossy", "lossless", "special"]}, 'vendor:minIOPS': { 'title': 'Minimum IOPS QoS', 'description': 'Sets minimum IOPS if QoS is enabled.', 'type': 'integer', 'minimum': 10, 'default': 100}, } } DocImpact Implements: blueprint get-volume-type-extra-specs Change-Id: I7a019f0296511bfda5b373e508071853d85e2376 --- cinder/tests/unit/test_volume.py | 157 +++++++++++++++++++++ cinder/tests/unit/test_volume_rpcapi.py | 7 + cinder/volume/driver.py | 175 +++++++++++++++++++++++- cinder/volume/manager.py | 13 +- cinder/volume/rpcapi.py | 8 +- 5 files changed, 357 insertions(+), 3 deletions(-) diff --git a/cinder/tests/unit/test_volume.py b/cinder/tests/unit/test_volume.py index de71c9a96..d65687787 100644 --- a/cinder/tests/unit/test_volume.py +++ b/cinder/tests/unit/test_volume.py @@ -5311,6 +5311,163 @@ class VolumeTestCase(BaseVolumeTestCase): volume_type=fake_type, consistencygroup=cg) + @mock.patch.object(fake_driver.FakeISCSIDriver, 'get_volume_stats') + @mock.patch.object(driver.BaseVD, '_init_vendor_properties') + def test_get_capabilities(self, mock_init_vendor, mock_get_volume_stats): + stats = { + 'volume_backend_name': 'lvm', + 'vendor_name': 'Open Source', + 'storage_protocol': 'iSCSI', + 'vendor_prefix': 'abcd' + } + expected = stats.copy() + expected['properties'] = { + 'compression': { + 'title': 'Compression', + 'description': 'Enables compression.', + 'type': 'boolean'}, + 'qos': { + 'title': 'QoS', + 'description': 'Enables QoS.', + 'type': 'boolean'}, + 'replication': { + 'title': 'Replication', + 'description': 'Enables replication.', + 'type': 'boolean'}, + 'thin_provisioning': { + 'title': 'Thin Provisioning', + 'description': 'Sets thin provisioning.', + 'type': 'boolean'}, + } + + # Test to get updated capabilities + discover = True + mock_get_volume_stats.return_value = stats + mock_init_vendor.return_value = ({}, None) + capabilities = self.volume.get_capabilities(self.context, + discover) + self.assertEqual(expected, capabilities) + mock_get_volume_stats.assert_called_once_with(True) + + # Test to get existing original capabilities + mock_get_volume_stats.reset_mock() + discover = False + capabilities = self.volume.get_capabilities(self.context, + discover) + self.assertEqual(expected, capabilities) + self.assertFalse(mock_get_volume_stats.called) + + # Normal test case to get vendor unique capabilities + def init_vendor_properties(self): + properties = {} + self._set_property( + properties, + "abcd:minIOPS", + "Minimum IOPS QoS", + "Sets minimum IOPS if QoS is enabled.", + "integer", + minimum=10, + default=100) + return properties, 'abcd' + + expected['properties'].update( + {'abcd:minIOPS': { + 'title': 'Minimum IOPS QoS', + 'description': 'Sets minimum IOPS if QoS is enabled.', + 'type': 'integer', + 'minimum': 10, + 'default': 100}}) + + mock_get_volume_stats.reset_mock() + mock_init_vendor.reset_mock() + discover = True + mock_init_vendor.return_value = ( + init_vendor_properties(self.volume.driver)) + capabilities = self.volume.get_capabilities(self.context, + discover) + self.assertEqual(expected, capabilities) + self.assertTrue(mock_get_volume_stats.called) + + @mock.patch.object(fake_driver.FakeISCSIDriver, 'get_volume_stats') + @mock.patch.object(driver.BaseVD, '_init_vendor_properties') + @mock.patch.object(driver.BaseVD, '_init_standard_capabilities') + def test_get_capabilities_prefix_error(self, mock_init_standard, + mock_init_vendor, + mock_get_volume_stats): + + # Error test case: propety does not match vendor prefix + def init_vendor_properties(self): + properties = {} + self._set_property( + properties, + "aaa:minIOPS", + "Minimum IOPS QoS", + "Sets minimum IOPS if QoS is enabled.", + "integer") + self._set_property( + properties, + "abcd:compression_type", + "Compression type", + "Specifies compression type.", + "string") + + return properties, 'abcd' + + expected = { + 'abcd:compression_type': { + 'title': 'Compression type', + 'description': 'Specifies compression type.', + 'type': 'string'}} + + discover = True + mock_get_volume_stats.return_value = {} + mock_init_standard.return_value = {} + mock_init_vendor.return_value = ( + init_vendor_properties(self.volume.driver)) + capabilities = self.volume.get_capabilities(self.context, + discover) + self.assertEqual(expected, capabilities['properties']) + + @mock.patch.object(fake_driver.FakeISCSIDriver, 'get_volume_stats') + @mock.patch.object(driver.BaseVD, '_init_vendor_properties') + @mock.patch.object(driver.BaseVD, '_init_standard_capabilities') + def test_get_capabilities_fail_override(self, mock_init_standard, + mock_init_vendor, + mock_get_volume_stats): + + # Error test case: propety cannot override any standard capabilities + def init_vendor_properties(self): + properties = {} + self._set_property( + properties, + "qos", + "Minimum IOPS QoS", + "Sets minimum IOPS if QoS is enabled.", + "integer") + self._set_property( + properties, + "ab::cd:compression_type", + "Compression type", + "Specifies compression type.", + "string") + + return properties, 'ab::cd' + + expected = { + 'ab__cd:compression_type': { + 'title': 'Compression type', + 'description': 'Specifies compression type.', + 'type': 'string'}} + + discover = True + mock_get_volume_stats.return_value = {} + mock_init_standard.return_value = {} + mock_init_vendor.return_value = ( + init_vendor_properties(self.volume.driver)) + capabilities = self.volume.get_capabilities(self.context, + discover) + self.assertEqual(expected, capabilities['properties']) + class CopyVolumeToImageTestCase(BaseVolumeTestCase): def fake_local_path(self, volume): diff --git a/cinder/tests/unit/test_volume_rpcapi.py b/cinder/tests/unit/test_volume_rpcapi.py index cca2658db..9df7f0c95 100644 --- a/cinder/tests/unit/test_volume_rpcapi.py +++ b/cinder/tests/unit/test_volume_rpcapi.py @@ -402,3 +402,10 @@ class VolumeRpcAPITestCase(test.TestCase): cgsnapshot=None, source_cg=self.fake_src_cg, version='1.26') + + def test_get_capabilities(self): + self._test_volume_api('get_capabilities', + rpc_method='call', + host='fake_host', + discover=True, + version='1.29') diff --git a/cinder/volume/driver.py b/cinder/volume/driver.py index 7b66efcb8..1e933bafc 100644 --- a/cinder/volume/driver.py +++ b/cinder/volume/driver.py @@ -324,6 +324,7 @@ class BaseVD(object): self._stats = {} self.pools = [] + self.capabilities = {} # We set these mappings up in the base driver so they # can be used by children @@ -533,7 +534,179 @@ class BaseVD(object): For replication the following state should be reported: replication = True (None or false disables replication) """ - return None + return + + def get_prefixed_property(self, property): + """Return prefixed property name + + :return a prefixed property name string or None + """ + + if property and self.capabilities.get('vendor_prefix'): + return self.capabilities.get('vendor_prefix') + ':' + property + + def _set_property(self, properties, entry, title, description, + type, **kwargs): + prop = dict(title=title, description=description, type=type) + allowed_keys = ('enum', 'default', 'minimum', 'maximum') + for key in kwargs: + if key in allowed_keys: + prop[key] = kwargs[key] + properties[entry] = prop + + def _init_standard_capabilities(self): + """Create a dictionary of Cinder standard capabilities. + + This method creates a dictionary of Cinder standard capabilities + and returns the created dictionary. + The keys of this dictionary don't contain prefix and separator(:). + """ + + properties = {} + self._set_property( + properties, + "thin_provisioning", + "Thin Provisioning", + _("Sets thin provisioning."), + "boolean") + + self._set_property( + properties, + "compression", + "Compression", + _("Enables compression."), + "boolean") + + self._set_property( + properties, + "qos", + "QoS", + _("Enables QoS."), + "boolean") + + self._set_property( + properties, + "replication", + "Replication", + _("Enables replication."), + "boolean") + + return properties + + def _init_vendor_properties(self): + """Create a dictionary of vendor unique properties. + + This method creates a dictionary of vendor unique properties + and returns both created dictionary and vendor name. + Returned vendor name is used to check for name of vendor + unique properties. + + - Vendor name shouldn't include colon(:) because of the separator + and it is automatically replaced by underscore(_). + ex. abc:d -> abc_d + - Vendor prefix is equal to vendor name. + ex. abcd + - Vendor unique properties must start with vendor prefix + ':'. + ex. abcd:maxIOPS + + Each backend driver needs to override this method to expose + its own properties using _set_property() like this: + + self._set_property( + properties, + "vendorPrefix:specific_property", + "Title of property", + _("Description of property"), + "type") + + : return dictionary of vendor unique properties + : return vendor name + + Example of implementation:: + + properties = {} + self._set_property( + properties, + "abcd:compression_type", + "Compression type", + _("Specifies compression type."), + "string", + enum=["lossy", "lossless", "special"]) + + self._set_property( + properties, + "abcd:minIOPS", + "Minimum IOPS QoS", + _("Sets minimum IOPS if QoS is enabled."), + "integer", + minimum=10, + default=100) + + return properties, 'abcd' + """ + + return {}, None + + def init_capabilities(self): + """Obtain backend volume stats and capabilities list. + + This stores a dictionary which is consisted of two parts. + First part includes static backend capabilities which are + obtained by get_volume_stats(). Second part is properties, + which includes parameters correspond to extra specs. + This properties part is consisted of cinder standard + capabilities and vendor unique properties. + + Using this capabilities list, operator can manage/configure + backend using key/value from capabilities without specific + knowledge of backend. + """ + + # Set static backend capabilities from get_volume_stats() + stats = self.get_volume_stats(True) + if stats: + self.capabilities = stats.copy() + + # Set cinder standard capabilities + self.capabilities['properties'] = self._init_standard_capabilities() + + # Set Vendor unique properties + vendor_prop, vendor_name = self._init_vendor_properties() + if vendor_name and vendor_prop: + updated_vendor_prop = {} + old_name = None + # Replace colon in vendor name to underscore. + if ':' in vendor_name: + old_name = vendor_name + vendor_name = vendor_name.replace(':', '_') + LOG.warning(_LW('The colon in vendor name was replaced ' + 'by underscore. Updated vendor name is ' + '%(name)s".'), {'name': vendor_name}) + + for key in vendor_prop: + # If key has colon in vendor name field, we replace it to + # underscore. + # ex. abc:d:storagetype:provisioning + # -> abc_d:storagetype:provisioning + if old_name and key.startswith(old_name + ':'): + new_key = key.replace(old_name, vendor_name, 1) + updated_vendor_prop[new_key] = vendor_prop[key] + continue + if not key.startswith(vendor_name + ':'): + LOG.warning(_LW('Vendor unique property "%(property)s" ' + 'must start with vendor prefix with colon ' + '"%(prefix)s". The property was ' + 'not registered on capabilities list.'), + {'prefix': vendor_name + ':', + 'property': key}) + continue + updated_vendor_prop[key] = vendor_prop[key] + + # Update vendor unique properties to the dictionary + self.capabilities['vendor_prefix'] = vendor_name + self.capabilities['properties'].update(updated_vendor_prop) + + LOG.debug("Initialized capabilities list: %s.", self.capabilities) def _update_pools_and_stats(self, data): """Updates data for pools and volume stats based on provided data.""" diff --git a/cinder/volume/manager.py b/cinder/volume/manager.py index 3f571c5fc..0bdab3fdc 100644 --- a/cinder/volume/manager.py +++ b/cinder/volume/manager.py @@ -190,7 +190,7 @@ def locked_snapshot_operation(f): class VolumeManager(manager.SchedulerDependentManager): """Manages attachable block storage devices.""" - RPC_API_VERSION = '1.28' + RPC_API_VERSION = '1.29' target = messaging.Target(version=RPC_API_VERSION) @@ -334,6 +334,9 @@ class VolumeManager(manager.SchedulerDependentManager): # to initialize the driver correctly. return + # Initialize backend capabilities list + self.driver.init_capabilities() + volumes = self.db.volume_get_all_by_host(ctxt, self.host) self._sync_provider_info(ctxt, volumes) # FIXME volume count for exporting is wrong @@ -3066,3 +3069,11 @@ class VolumeManager(manager.SchedulerDependentManager): with flow_utils.DynamicLogListener(flow_engine, logger=LOG): flow_engine.run() return snapshot.id + + def get_capabilities(self, context, discover): + """Get capabilities of backend storage.""" + if discover: + self.driver.init_capabilities() + capabilities = self.driver.capabilities + LOG.debug("Obtained capabilities list: %s.", capabilities) + return capabilities diff --git a/cinder/volume/rpcapi.py b/cinder/volume/rpcapi.py index f1b53c711..a2c748cc0 100644 --- a/cinder/volume/rpcapi.py +++ b/cinder/volume/rpcapi.py @@ -74,6 +74,7 @@ class VolumeAPI(object): update_consistencygroup() and delete_consistencygroup(). 1.27 - Adds support for replication V2 1.28 - Adds manage_existing_snapshot + 1.29 - Adds get_capabilities. """ BASE_RPC_API_VERSION = '1.0' @@ -83,7 +84,7 @@ class VolumeAPI(object): target = messaging.Target(topic=CONF.volume_topic, version=self.BASE_RPC_API_VERSION) serializer = objects_base.CinderObjectSerializer() - self.client = rpc.get_client(target, '1.28', serializer=serializer) + self.client = rpc.get_client(target, '1.29', serializer=serializer) def create_consistencygroup(self, ctxt, group, host): new_host = utils.extract_host(host) @@ -295,3 +296,8 @@ class VolumeAPI(object): cctxt.cast(ctxt, 'manage_existing_snapshot', snapshot=snapshot, ref=ref) + + def get_capabilities(self, ctxt, host, discover): + new_host = utils.extract_host(host) + cctxt = self.client.prepare(server=new_host, version='1.29') + return cctxt.call(ctxt, 'get_capabilities', discover=discover) -- 2.45.2