From d61a98d545d74068ae90c931466ac9d89f77bb46 Mon Sep 17 00:00:00 2001 From: Bob Callaway Date: Sun, 19 Apr 2015 21:22:37 -0700 Subject: [PATCH] Add support for volume groups and netapp_raid_type This patch removes a filter constraint on pool information that limited the pools that would be returned to the scheduler based on the raid type for the DDP feature of the E-Series platform. Now that pools may have different RAID types, we return that as an unqualified extra-spec netapp_raid_type. This patch also resolves a defect with the reporting of E-Series extra-specs that was introduced with the removal of the cache functionality. DocImpact Partially-Implements blueprint netapp-eseries-additional-extra-specs Change-Id: I2656c3282270a5ec5256cc12767a09138c4a5371 --- .../volume/drivers/netapp/eseries/fakes.py | 171 +++++++++++++++++- .../drivers/netapp/eseries/test_library.py | 99 ++++++++-- .../volume/drivers/netapp/eseries/library.py | 60 ++++-- 3 files changed, 289 insertions(+), 41 deletions(-) diff --git a/cinder/tests/unit/volume/drivers/netapp/eseries/fakes.py b/cinder/tests/unit/volume/drivers/netapp/eseries/fakes.py index 1e9b5cf2d..6285e41a6 100644 --- a/cinder/tests/unit/volume/drivers/netapp/eseries/fakes.py +++ b/cinder/tests/unit/volume/drivers/netapp/eseries/fakes.py @@ -41,11 +41,170 @@ FOREIGN_HOST_GROUP = { 'label': 'FOREIGN HOST GROUP', } -STORAGE_POOL = { - 'label': 'DDP', - 'volumeGroupRef': 'fakevolgroupref', - 'raidLevel': 'raidDiskPool', -} +STORAGE_POOLS = [ + { + "sequenceNum": 1, + "offline": False, + "raidLevel": "raidDiskPool", + "worldWideName": "60080E50002998A00000A387558A7514", + "volumeGroupRef": "0400000060080E50002998A00000A387558A7514", + "reserved1": "000000000000000000000000", + "reserved2": "", + "trayLossProtection": False, + "label": "DDP", + "state": "complete", + "spindleSpeedMatch": True, + "spindleSpeed": 10033, + "isInaccessible": False, + "securityType": "none", + "drawerLossProtection": False, + "protectionInformationCapable": False, + "protectionInformationCapabilities": { + "protectionInformationCapable": False, + "protectionType": "type0Protection" + }, + "volumeGroupData": { + "type": "diskPool", + "diskPoolData": { + "reconstructionReservedDriveCount": 1, + "reconstructionReservedAmt": "138512695296", + "reconstructionReservedDriveCountCurrent": 1, + "poolUtilizationWarningThreshold": 50, + "poolUtilizationCriticalThreshold": 85, + "poolUtilizationState": "utilizationOptimal", + "unusableCapacity": "0", + "degradedReconstructPriority": "high", + "criticalReconstructPriority": "highest", + "backgroundOperationPriority": "low", + "allocGranularity": "4294967296" + } + }, + "usage": "standard", + "driveBlockFormat": "allNative", + "reservedSpaceAllocated": True, + "usedSpace": "55834574848", + "totalRaidedSpace": "1060856922112", + "extents": [ + { + "sectorOffset": "0", + "rawCapacity": "1005022347264", + "raidLevel": "raidDiskPool", + "volumeGroupRef": "0400000060080E50002998A00000A387558A7514", + "freeExtentRef": "0301000060080E50002998A00000A387558A7514", + "reserved1": "000000000000000000000000", + "reserved2": "" + } + ], + "largestFreeExtentSize": "1005022347264", + "raidStatus": "optimal", + "freeSpace": "1005022347264", + "drivePhysicalType": "sas", + "driveMediaType": "hdd", + "normalizedSpindleSpeed": "spindleSpeed10k", + "id": "0400000060080E50002998A00000A387558A7514", + "diskPool": True + }, + { + "sequenceNum": 1, + "offline": False, + "raidLevel": "raid5", + "worldWideName": "60080E500029E0B4000059A0553E1725", + "volumeGroupRef": "0400000060080E500029E0B4000059A0553E1725", + "reserved1": "000000000000000000000000", + "reserved2": "", + "trayLossProtection": False, + "label": "test_vg_1", + "state": "complete", + "spindleSpeedMatch": False, + "spindleSpeed": 10000, + "isInaccessible": False, + "securityType": "enabled", + "drawerLossProtection": False, + "protectionInformationCapable": False, + "protectionInformationCapabilities": { + "protectionInformationCapable": False, + "protectionType": "type0Protection" + }, + "volumeGroupData": { + "type": "unknown", + "diskPoolData": None + }, + "usage": "standard", + "driveBlockFormat": "allNative", + "reservedSpaceAllocated": False, + "usedSpace": "28366077952", + "totalRaidedSpace": "292557733888", + "extents": [ + { + "sectorOffset": "27701248", + "rawCapacity": "264191655936", + "raidLevel": "raid5", + "volumeGroupRef": "0400000060080E500029E0B4000059A0553E1725", + "freeExtentRef": "0300001B60080E500029E0B4000059A0553E1725", + "reserved1": "000000000000000000000000", + "reserved2": "" + } + ], + "largestFreeExtentSize": "264191655936", + "raidStatus": "optimal", + "freeSpace": "264191655936", + "drivePhysicalType": "sas", + "driveMediaType": "ssd", + "normalizedSpindleSpeed": "spindleSpeed10k", + "id": "0400000060080E500029E0B4000059A0553E1725", + "diskPool": False + }, + { + "sequenceNum": 3, + "offline": False, + "raidLevel": "raid6", + "worldWideName": "60080E500029E0B4000059A2553E1739", + "volumeGroupRef": "0400000060080E500029E0B4000059A2553E1739", + "reserved1": "000000000000000000000000", + "reserved2": "", + "trayLossProtection": False, + "label": "test_vg_2", + "state": "complete", + "spindleSpeedMatch": True, + "spindleSpeed": 10020, + "isInaccessible": False, + "securityType": "enabled", + "drawerLossProtection": False, + "protectionInformationCapable": False, + "protectionInformationCapabilities": { + "protectionInformationCapable": False, + "protectionType": "type2Protection" + }, + "volumeGroupData": { + "type": "unknown", + "diskPoolData": None + }, + "usage": "standard", + "driveBlockFormat": "allNative", + "reservedSpaceAllocated": False, + "usedSpace": "27399710720", + "totalRaidedSpace": "1798769641472", + "extents": [ + { + "sectorOffset": "17839360", + "rawCapacity": "1771369930752", + "raidLevel": "raid6", + "volumeGroupRef": "0400000060080E500029E0B4000059A2553E1739", + "freeExtentRef": "0300001360080E500029E0B4000059A2553E1739", + "reserved1": "000000000000000000000000", + "reserved2": "" + } + ], + "largestFreeExtentSize": "1771369930752", + "raidStatus": "optimal", + "freeSpace": "1771369930752", + "drivePhysicalType": "sas", + "driveMediaType": "hdd", + "normalizedSpindleSpeed": "spindleSpeed10k", + "id": "0400000060080E500029E0B4000059A2553E1739", + "diskPool": False + } +] VOLUME = { 'extremeProtection': False, @@ -484,7 +643,7 @@ class FakeEseriesClient(object): pass def list_storage_pools(self): - return [STORAGE_POOL] + return STORAGE_POOLS def register_storage_system(self, *args, **kwargs): return { diff --git a/cinder/tests/unit/volume/drivers/netapp/eseries/test_library.py b/cinder/tests/unit/volume/drivers/netapp/eseries/test_library.py index d79b152ed..209050827 100644 --- a/cinder/tests/unit/volume/drivers/netapp/eseries/test_library.py +++ b/cinder/tests/unit/volume/drivers/netapp/eseries/test_library.py @@ -70,25 +70,46 @@ class NetAppEseriesLibraryTestCase(test.TestCase): self.assertTrue(mock_check_flags.called) + def test_get_storage_pools(self): + pool_labels = list() + # Retrieve the first pool's label + for pool in eseries_fake.STORAGE_POOLS: + pool_labels.append(pool['label']) + break + self.library.configuration.netapp_storage_pools = ( + ",".join(pool_labels)) + + filtered_pools = self.library._get_storage_pools() + + filtered_pool_labels = [pool['label'] for pool in filtered_pools] + self.assertListEqual(pool_labels, filtered_pool_labels) + def test_update_ssc_info(self): drives = [{'currentVolumeGroupRef': 'test_vg1', 'driveMediaType': 'ssd'}] - - self.library._get_storage_pools = mock.Mock(return_value=['test_vg1']) - self.library._client.list_storage_pools = mock.Mock(return_value=[]) + pools = [{'volumeGroupRef': 'test_vg1', 'label': 'test_vg1', + 'raidLevel': 'raid6', 'securityType': 'enabled'}] + self.library.configuration.netapp_storage_pools = "test_vg1" + self.library._client.list_storage_pools = mock.Mock(return_value=pools) self.library._client.list_drives = mock.Mock(return_value=drives) self.library._update_ssc_info() - self.assertEqual({'test_vg1': {'netapp_disk_type': 'SSD'}}, - self.library._ssc_stats) + self.assertEqual( + {'test_vg1': {'netapp_disk_encryption': 'true', + 'netapp_disk_type': 'SSD', + 'netapp_raid_type': 'raid6'}}, + self.library._ssc_stats) def test_update_ssc_disk_types_ssd(self): drives = [{'currentVolumeGroupRef': 'test_vg1', 'driveMediaType': 'ssd'}] + pools = [{'volumeGroupRef': 'test_vg1'}] + self.library._client.list_drives = mock.Mock(return_value=drives) + self.library._client.get_storage_pool = mock.Mock(return_value=pools) - ssc_stats = self.library._update_ssc_disk_types(['test_vg1']) + ssc_stats = self.library._update_ssc_disk_types(pools) self.assertEqual({'test_vg1': {'netapp_disk_type': 'SSD'}}, ssc_stats) @@ -96,9 +117,12 @@ class NetAppEseriesLibraryTestCase(test.TestCase): def test_update_ssc_disk_types_scsi(self): drives = [{'currentVolumeGroupRef': 'test_vg1', 'interfaceType': {'driveType': 'scsi'}}] + pools = [{'volumeGroupRef': 'test_vg1'}] + self.library._client.list_drives = mock.Mock(return_value=drives) + self.library._client.get_storage_pool = mock.Mock(return_value=pools) - ssc_stats = self.library._update_ssc_disk_types(['test_vg1']) + ssc_stats = self.library._update_ssc_disk_types(pools) self.assertEqual({'test_vg1': {'netapp_disk_type': 'SCSI'}}, ssc_stats) @@ -106,9 +130,12 @@ class NetAppEseriesLibraryTestCase(test.TestCase): def test_update_ssc_disk_types_fcal(self): drives = [{'currentVolumeGroupRef': 'test_vg1', 'interfaceType': {'driveType': 'fibre'}}] + pools = [{'volumeGroupRef': 'test_vg1'}] + self.library._client.list_drives = mock.Mock(return_value=drives) + self.library._client.get_storage_pool = mock.Mock(return_value=pools) - ssc_stats = self.library._update_ssc_disk_types(['test_vg1']) + ssc_stats = self.library._update_ssc_disk_types(pools) self.assertEqual({'test_vg1': {'netapp_disk_type': 'FCAL'}}, ssc_stats) @@ -116,9 +143,12 @@ class NetAppEseriesLibraryTestCase(test.TestCase): def test_update_ssc_disk_types_sata(self): drives = [{'currentVolumeGroupRef': 'test_vg1', 'interfaceType': {'driveType': 'sata'}}] + pools = [{'volumeGroupRef': 'test_vg1'}] + self.library._client.list_drives = mock.Mock(return_value=drives) + self.library._client.get_storage_pool = mock.Mock(return_value=pools) - ssc_stats = self.library._update_ssc_disk_types(['test_vg1']) + ssc_stats = self.library._update_ssc_disk_types(pools) self.assertEqual({'test_vg1': {'netapp_disk_type': 'SATA'}}, ssc_stats) @@ -126,9 +156,12 @@ class NetAppEseriesLibraryTestCase(test.TestCase): def test_update_ssc_disk_types_sas(self): drives = [{'currentVolumeGroupRef': 'test_vg1', 'interfaceType': {'driveType': 'sas'}}] + pools = [{'volumeGroupRef': 'test_vg1'}] + self.library._client.list_drives = mock.Mock(return_value=drives) + self.library._client.get_storage_pool = mock.Mock(return_value=pools) - ssc_stats = self.library._update_ssc_disk_types(['test_vg1']) + ssc_stats = self.library._update_ssc_disk_types(pools) self.assertEqual({'test_vg1': {'netapp_disk_type': 'SAS'}}, ssc_stats) @@ -136,9 +169,12 @@ class NetAppEseriesLibraryTestCase(test.TestCase): def test_update_ssc_disk_types_unknown(self): drives = [{'currentVolumeGroupRef': 'test_vg1', 'interfaceType': {'driveType': 'unknown'}}] + pools = [{'volumeGroupRef': 'test_vg1'}] + self.library._client.list_drives = mock.Mock(return_value=drives) + self.library._client.get_storage_pool = mock.Mock(return_value=pools) - ssc_stats = self.library._update_ssc_disk_types(['test_vg1']) + ssc_stats = self.library._update_ssc_disk_types(pools) self.assertEqual({'test_vg1': {'netapp_disk_type': 'unknown'}}, ssc_stats) @@ -146,9 +182,12 @@ class NetAppEseriesLibraryTestCase(test.TestCase): def test_update_ssc_disk_types_undefined(self): drives = [{'currentVolumeGroupRef': 'test_vg1', 'interfaceType': {'driveType': '__UNDEFINED'}}] + pools = [{'volumeGroupRef': 'test_vg1'}] + self.library._client.list_drives = mock.Mock(return_value=drives) + self.library._client.get_storage_pool = mock.Mock(return_value=pools) - ssc_stats = self.library._update_ssc_disk_types(['test_vg1']) + ssc_stats = self.library._update_ssc_disk_types(pools) self.assertEqual({'test_vg1': {'netapp_disk_type': 'unknown'}}, ssc_stats) @@ -157,7 +196,7 @@ class NetAppEseriesLibraryTestCase(test.TestCase): pools = [{'volumeGroupRef': 'test_vg1', 'securityType': 'enabled'}] self.library._client.list_storage_pools = mock.Mock(return_value=pools) - ssc_stats = self.library._update_ssc_disk_encryption(['test_vg1']) + ssc_stats = self.library._update_ssc_disk_encryption(pools) self.assertEqual({'test_vg1': {'netapp_disk_encryption': 'true'}}, ssc_stats) @@ -166,7 +205,7 @@ class NetAppEseriesLibraryTestCase(test.TestCase): pools = [{'volumeGroupRef': 'test_vg1', 'securityType': 'unknown'}] self.library._client.list_storage_pools = mock.Mock(return_value=pools) - ssc_stats = self.library._update_ssc_disk_encryption(['test_vg1']) + ssc_stats = self.library._update_ssc_disk_encryption(pools) self.assertEqual({'test_vg1': {'netapp_disk_encryption': 'false'}}, ssc_stats) @@ -175,7 +214,7 @@ class NetAppEseriesLibraryTestCase(test.TestCase): pools = [{'volumeGroupRef': 'test_vg1', 'securityType': 'none'}] self.library._client.list_storage_pools = mock.Mock(return_value=pools) - ssc_stats = self.library._update_ssc_disk_encryption(['test_vg1']) + ssc_stats = self.library._update_ssc_disk_encryption(pools) self.assertEqual({'test_vg1': {'netapp_disk_encryption': 'false'}}, ssc_stats) @@ -184,7 +223,7 @@ class NetAppEseriesLibraryTestCase(test.TestCase): pools = [{'volumeGroupRef': 'test_vg1', 'securityType': 'capable'}] self.library._client.list_storage_pools = mock.Mock(return_value=pools) - ssc_stats = self.library._update_ssc_disk_encryption(['test_vg1']) + ssc_stats = self.library._update_ssc_disk_encryption(pools) self.assertEqual({'test_vg1': {'netapp_disk_encryption': 'false'}}, ssc_stats) @@ -193,7 +232,7 @@ class NetAppEseriesLibraryTestCase(test.TestCase): pools = [{'volumeGroupRef': 'test_vg1', 'securityType': 'garbage'}] self.library._client.list_storage_pools = mock.Mock(return_value=pools) - ssc_stats = self.library._update_ssc_disk_encryption(['test_vg1']) + ssc_stats = self.library._update_ssc_disk_encryption(pools) self.assertRaises(TypeError, 'test_vg1', {'netapp_disk_encryption': 'false'}, ssc_stats) @@ -203,8 +242,7 @@ class NetAppEseriesLibraryTestCase(test.TestCase): {'volumeGroupRef': 'test_vg2', 'securityType': 'enabled'}] self.library._client.list_storage_pools = mock.Mock(return_value=pools) - ssc_stats = self.library._update_ssc_disk_encryption(['test_vg1', - 'test_vg2']) + ssc_stats = self.library._update_ssc_disk_encryption(pools) self.assertEqual({'test_vg1': {'netapp_disk_encryption': 'false'}, 'test_vg2': {'netapp_disk_encryption': 'true'}}, @@ -609,6 +647,29 @@ class NetAppEseriesLibraryTestCase(test.TestCase): self.assertDictEqual(eseries_fake.FC_I_T_MAP, initiator_target_map) self.assertEqual(4, num_paths) + @ddt.data(('raid0', 'raid0'), ('raid1', 'raid1'), ('raid3', 'raid5'), + ('raid5', 'raid5'), ('raid6', 'raid6'), ('raidDiskPool', 'DDP')) + @ddt.unpack + def test_update_ssc_raid_type(self, raid_lvl, raid_lvl_mapping): + pools = [{'volumeGroupRef': 'test_vg1', 'raidLevel': raid_lvl}] + self.library._client.list_storage_pools = mock.Mock(return_value=pools) + + ssc_stats = self.library._update_ssc_raid_type(pools) + + self.assertEqual({'test_vg1': {'netapp_raid_type': raid_lvl_mapping}}, + ssc_stats) + + @ddt.data('raidAll', '__UNDEFINED', 'unknown', + 'raidUnsupported', 'garbage') + def test_update_ssc_raid_type_invalid(self, raid_lvl): + pools = [{'volumeGroupRef': 'test_vg1', 'raidLevel': raid_lvl}] + self.library._client.list_storage_pools = mock.Mock(return_value=pools) + + ssc_stats = self.library._update_ssc_raid_type(pools) + + self.assertEqual({'test_vg1': {'netapp_raid_type': 'unknown'}}, + ssc_stats) + class NetAppEseriesLibraryMultiAttachTestCase(test.TestCase): """Test driver when netapp_enable_multiattach is enabled. diff --git a/cinder/volume/drivers/netapp/eseries/library.py b/cinder/volume/drivers/netapp/eseries/library.py index b768cb3fc..2405fa2f9 100644 --- a/cinder/volume/drivers/netapp/eseries/library.py +++ b/cinder/volume/drivers/netapp/eseries/library.py @@ -90,6 +90,15 @@ class NetAppESeriesLibrary(object): 'sas': 'SAS', 'sata': 'SATA', } + SSC_RAID_TYPE_MAPPING = { + 'raidDiskPool': 'DDP', + 'raid0': 'raid0', + 'raid1': 'raid1', + # RAID3 is being deprecated and is actually implemented as RAID5 + 'raid3': 'raid5', + 'raid5': 'raid5', + 'raid6': 'raid6', + } SSC_UPDATE_INTERVAL = 60 # seconds WORLDWIDENAME = 'worldWideName' @@ -879,20 +888,26 @@ class NetAppESeriesLibrary(object): """ LOG.info(_LI("Updating storage service catalog information for " "backend '%s'"), self._backend_name) - self._ssc_stats = \ - self._update_ssc_disk_encryption(self._get_storage_pools()) - self._ssc_stats = \ - self._update_ssc_disk_types(self._get_storage_pools()) - - def _update_ssc_disk_types(self, volume_groups): + relevant_pools = self._get_storage_pools() + self._ssc_stats = ( + self._update_ssc_disk_encryption(relevant_pools)) + self._ssc_stats = ( + self._update_ssc_disk_types(relevant_pools)) + self._ssc_stats = ( + self._update_ssc_raid_type(relevant_pools)) + + def _update_ssc_disk_types(self, storage_pools): """Updates the given ssc dictionary with new disk type information. - :param volume_groups: The volume groups this driver cares about + :param storage_pools: The storage pools this driver cares about """ ssc_stats = copy.deepcopy(self._ssc_stats) all_disks = self._client.list_drives() + + pool_ids = set(pool.get("volumeGroupRef") for pool in storage_pools) + relevant_disks = filter(lambda x: x.get('currentVolumeGroupRef') in - volume_groups, all_disks) + pool_ids, all_disks) for drive in relevant_disks: current_vol_group = drive.get('currentVolumeGroupRef') if current_vol_group not in ssc_stats: @@ -907,16 +922,13 @@ class NetAppESeriesLibrary(object): return ssc_stats - def _update_ssc_disk_encryption(self, volume_groups): + def _update_ssc_disk_encryption(self, storage_pools): """Updates the given ssc dictionary with new disk encryption information. - :param volume_groups: The volume groups this driver cares about + :param storage_pools: The storage pools this driver cares about """ ssc_stats = copy.deepcopy(self._ssc_stats) - all_pools = self._client.list_storage_pools() - relevant_pools = filter(lambda x: x.get('volumeGroupRef') in - volume_groups, all_pools) - for pool in relevant_pools: + for pool in storage_pools: current_vol_group = pool.get('volumeGroupRef') if current_vol_group not in ssc_stats: ssc_stats[current_vol_group] = {} @@ -926,6 +938,23 @@ class NetAppESeriesLibrary(object): return ssc_stats + def _update_ssc_raid_type(self, storage_pools): + """Updates the given ssc dictionary with new RAID type information. + + :param storage_pools: The storage pools this driver cares about + """ + ssc_stats = copy.deepcopy(self._ssc_stats) + for pool in storage_pools: + current_vol_group = pool.get('volumeGroupRef') + if current_vol_group not in ssc_stats: + ssc_stats[current_vol_group] = {} + + raid_type = pool.get('raidLevel') + ssc_stats[current_vol_group]['netapp_raid_type'] = ( + self.SSC_RAID_TYPE_MAPPING.get(raid_type, 'unknown')) + + return ssc_stats + def _get_storage_pools(self): conf_enabled_pools = [] for value in self.configuration.netapp_storage_pools.split(','): @@ -936,8 +965,7 @@ class NetAppESeriesLibrary(object): storage_pools = self._client.list_storage_pools() for storage_pool in storage_pools: # Check if pool can be used - if (storage_pool.get('raidLevel') == 'raidDiskPool' - and storage_pool['label'].lower() in conf_enabled_pools): + if (storage_pool['label'].lower() in conf_enabled_pools): filtered_pools.append(storage_pool) return filtered_pools -- 2.45.2