From: Michael Price Date: Wed, 5 Aug 2015 20:36:59 +0000 (-0500) Subject: Implement thin provisioning support for E-Series X-Git-Url: https://review.fuel-infra.org/gitweb?a=commitdiff_plain;h=137e87efa5a28fa6e3c3f1cd8aafd628ef4df7ea;p=openstack-build%2Fcinder-build.git Implement thin provisioning support for E-Series Implement a new extra spec, 'netapp_thin_provisioned', that will allow users to define thin provisioned Cinder volumes on E-Series storage, alongside pre-existing extra specs such as 'netapp_eseries_data_assurance', 'netapp_eseries_flash_reach_cache', 'netapp_raid_type', etc. We have a followup patch, https://review.openstack.org/#/c/215801/ , that reports 'thin_provisioning_support=True/False' and same for 'thick_provisioning_support', in accord with the scheduler-based over-subscription support added in Kilo. We are not yet attempting to implement the get_capabilities() feature just merged into Liberty (https://review.openstack.org/#/c/201243/) but fully intend to do so in a way that conforms with that plan of record. Partially-Implements: blueprint netapp-eseries-additional-extra-specs DocImpact Change-Id: Ia00b56e6d6a644cff81791bbd04e97f0c02b9e65 --- diff --git a/cinder/tests/unit/test_netapp_eseries_iscsi.py b/cinder/tests/unit/test_netapp_eseries_iscsi.py index 8a8e1ea15..11b383b43 100644 --- a/cinder/tests/unit/test_netapp_eseries_iscsi.py +++ b/cinder/tests/unit/test_netapp_eseries_iscsi.py @@ -2,6 +2,7 @@ # Copyright (c) 2015 Alex Meade. All Rights Reserved. # Copyright (c) 2015 Rushil Chugh. All Rights Reserved. # Copyright (c) 2015 Navneet Singh. All Rights Reserved. +# Copyright (c) 2015 Michael Price. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain @@ -17,6 +18,7 @@ """Tests for NetApp e-series iscsi volume driver.""" import copy +import ddt import json import re import socket @@ -26,7 +28,8 @@ import requests from cinder import exception from cinder import test -from cinder.tests.unit.volume.drivers.netapp.eseries import fakes +from cinder.tests.unit.volume.drivers.netapp.eseries import fakes as \ + fakes from cinder.volume import configuration as conf from cinder.volume.drivers.netapp import common from cinder.volume.drivers.netapp.eseries import client @@ -610,6 +613,7 @@ class FakeEseriesHTTPSession(object): raise exception.Invalid() +@ddt.ddt class NetAppEseriesISCSIDriverTestCase(test.TestCase): """Test case for NetApp e-series iscsi driver.""" @@ -720,14 +724,13 @@ class NetAppEseriesISCSIDriverTestCase(test.TestCase): self.assertTrue(result) def test_create_destroy(self): - FAKE_POOLS = [{'label': 'DDP', 'volumeGroupRef': 'test'}] - self.library._get_storage_pools = mock.Mock(return_value=FAKE_POOLS) - self.library._client.features = mock.Mock() - self.mock_object(self.library._client, '_get_resource_url', mock.Mock( - return_value=fakes.FAKE_ENDPOINT_HTTP)) - self.mock_object(self.library._client, '_eval_response') - self.mock_object(self.library._client, 'list_volumes', mock.Mock( - return_value=FAKE_POOLS)) + self.mock_object(client.RestClient, 'delete_volume', + mock.Mock(return_value='None')) + self.mock_object(self.driver.library, 'create_volume', + mock.Mock(return_value=self.volume)) + self.mock_object(self.library._client, 'list_volume', mock.Mock( + return_value=fakes.VOLUME)) + self.driver.create_volume(self.volume) self.driver.delete_volume(self.volume) @@ -967,72 +970,43 @@ class NetAppEseriesISCSIDriverTestCase(test.TestCase): exception.NoValidHost, driver.library._check_mode_get_or_register_storage_system) - def test_get_vol_with_label_wwn_missing(self): - self.assertRaises(exception.InvalidInput, - self.library._get_volume_with_label_wwn, - None, None) - - def test_get_vol_with_label_wwn_found(self): - fake_vl_list = [{'volumeRef': '1', 'volumeUse': 'standardVolume', - 'label': 'l1', 'volumeGroupRef': 'g1', - 'worlWideName': 'w1ghyu'}, - {'volumeRef': '2', 'volumeUse': 'standardVolume', - 'label': 'l2', 'volumeGroupRef': 'g2', - 'worldWideName': 'w2ghyu'}] - self.library._get_storage_pools = mock.Mock(return_value=['g2', 'g3']) - self.library._client.list_volumes = mock.Mock( - return_value=fake_vl_list) - vol = self.library._get_volume_with_label_wwn('l2', 'w2:gh:yu') - self.assertEqual(1, self.library._client.list_volumes.call_count) - self.assertEqual('2', vol['volumeRef']) - - def test_get_vol_with_label_wwn_unmatched(self): - fake_vl_list = [{'volumeRef': '1', 'volumeUse': 'standardVolume', - 'label': 'l1', 'volumeGroupRef': 'g1', - 'worlWideName': 'w1ghyu'}, - {'volumeRef': '2', 'volumeUse': 'standardVolume', - 'label': 'l2', 'volumeGroupRef': 'g2', - 'worldWideName': 'w2ghyu'}] - self.library._get_storage_pools = mock.Mock(return_value=['g2', 'g3']) - self.library._client.list_volumes = mock.Mock( - return_value=fake_vl_list) - self.assertRaises(KeyError, self.library._get_volume_with_label_wwn, - 'l2', 'abcdef') - self.assertEqual(1, self.library._client.list_volumes.call_count) - def test_manage_existing_get_size(self): self.library._get_existing_vol_with_manage_ref = mock.Mock( return_value=self.fake_ret_vol) size = self.driver.manage_existing_get_size(self.volume, self.fake_ref) self.assertEqual(3, size) self.library._get_existing_vol_with_manage_ref.assert_called_once_with( - self.volume, self.fake_ref) + self.fake_ref) def test_get_exist_vol_source_name_missing(self): + self.library._client.list_volume = mock.Mock( + side_effect=exception.InvalidInput) self.assertRaises(exception.ManageExistingInvalidReference, self.library._get_existing_vol_with_manage_ref, - self.volume, {'id': '1234'}) + {'id': '1234'}) - def test_get_exist_vol_source_not_found(self): - def _get_volume(v_id, v_name): - d = {'id': '1'} + @ddt.data('source-id', 'source-name') + def test_get_exist_vol_source_not_found(self, attr_name): + def _get_volume(v_id): + d = {'id': '1', 'name': 'volume1', 'worldWideName': '0'} return d[v_id] - self.library._get_volume_with_label_wwn = mock.Mock(wraps=_get_volume) + self.library._client.list_volume = mock.Mock(wraps=_get_volume) self.assertRaises(exception.ManageExistingInvalidReference, self.library._get_existing_vol_with_manage_ref, - {'id': 'id2'}, {'source-name': 'name2'}) - self.library._get_volume_with_label_wwn.assert_called_once_with( - 'name2', None) + {attr_name: 'name2'}) + + self.library._client.list_volume.assert_called_once_with( + 'name2') def test_get_exist_vol_with_manage_ref(self): fake_ret_vol = {'id': 'right'} - self.library._get_volume_with_label_wwn = mock.Mock( - return_value=fake_ret_vol) + self.library._client.list_volume = mock.Mock(return_value=fake_ret_vol) + actual_vol = self.library._get_existing_vol_with_manage_ref( - {'id': 'id2'}, {'source-name': 'name2'}) - self.library._get_volume_with_label_wwn.assert_called_once_with( - 'name2', None) + {'source-name': 'name2'}) + + self.library._client.list_volume.assert_called_once_with('name2') self.assertEqual(fake_ret_vol, actual_vol) @mock.patch.object(utils, 'convert_uuid_to_es_fmt') @@ -1042,7 +1016,7 @@ class NetAppEseriesISCSIDriverTestCase(test.TestCase): mock_convert_es_fmt.return_value = 'label' self.driver.manage_existing(self.volume, self.fake_ref) self.library._get_existing_vol_with_manage_ref.assert_called_once_with( - self.volume, self.fake_ref) + self.fake_ref) mock_convert_es_fmt.assert_called_once_with( '114774fb-e15a-4fae-8ee2-c9723e3645ef') @@ -1055,7 +1029,7 @@ class NetAppEseriesISCSIDriverTestCase(test.TestCase): return_value={'id': 'update', 'worldWideName': 'wwn'}) self.driver.manage_existing(self.volume, self.fake_ref) self.library._get_existing_vol_with_manage_ref.assert_called_once_with( - self.volume, self.fake_ref) + self.fake_ref) mock_convert_es_fmt.assert_called_once_with( '114774fb-e15a-4fae-8ee2-c9723e3645ef') self.library._client.update_volume.assert_called_once_with( diff --git a/cinder/tests/unit/volume/drivers/netapp/eseries/fakes.py b/cinder/tests/unit/volume/drivers/netapp/eseries/fakes.py index 42a83e56f..2258c7a62 100644 --- a/cinder/tests/unit/volume/drivers/netapp/eseries/fakes.py +++ b/cinder/tests/unit/volume/drivers/netapp/eseries/fakes.py @@ -1,5 +1,6 @@ # Copyright (c) - 2015, Alex Meade # Copyright (c) - 2015, Yogesh Kshirsagar +# Copyright (c) - 2015, Michael Price # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may @@ -257,44 +258,223 @@ SSC_POOLS = [ STORAGE_POOLS = [ssc_pool['pool'] for ssc_pool in SSC_POOLS] -VOLUME = { - 'extremeProtection': False, - 'pitBaseVolume': True, - 'dssMaxSegmentSize': 131072, - 'totalSizeInBytes': '1073741824', - 'raidLevel': 'raid6', - 'volumeRef': '0200000060080E500023BB34000003FB515C2293', - 'listOfMappings': [], - 'sectorOffset': '15', - 'id': '0200000060080E500023BB34000003FB515C2293', - 'wwn': '60080E500023BB3400001FC352D14CB2', - 'capacity': '2147483648', - 'mgmtClientAttribute': 0, - 'label': 'CFDXJ67BLJH25DXCZFZD4NSF54', - 'volumeFull': False, - 'blkSize': 512, - 'volumeCopyTarget': False, - 'volumeGroupRef': '0400000060080E500023BB3400001F9F52CECC3F', - 'preferredControllerId': '070000000000000000000001', - 'currentManager': '070000000000000000000001', - 'applicationTagOwned': False, - 'status': 'optimal', - 'segmentSize': 131072, - 'volumeUse': 'standardVolume', - 'action': 'none', - 'preferredManager': '070000000000000000000001', - 'volumeHandle': 15, - 'offline': False, - 'preReadRedundancyCheckEnabled': False, - 'dssPreallocEnabled': False, - 'name': 'bdm-vc-test-1', - 'worldWideName': '60080E500023BB3400001FC352D14CB2', - 'currentControllerId': '070000000000000000000001', - 'protectionInformationCapable': False, - 'mapped': False, - 'reconPriority': 1, - 'protectionType': 'type1Protection' -} +VOLUMES = [ + { + "offline": False, + "extremeProtection": False, + "volumeHandle": 2, + "raidLevel": "raid0", + "sectorOffset": "0", + "worldWideName": "60080E50002998A00000945355C37C19", + "label": "1", + "blkSize": 512, + "capacity": "10737418240", + "reconPriority": 1, + "segmentSize": 131072, + "action": "initializing", + "cache": { + "cwob": False, + "enterpriseCacheDump": False, + "mirrorActive": True, + "mirrorEnable": True, + "readCacheActive": True, + "readCacheEnable": True, + "writeCacheActive": True, + "writeCacheEnable": True, + "cacheFlushModifier": "flush10Sec", + "readAheadMultiplier": 1 + }, + "mediaScan": { + "enable": False, + "parityValidationEnable": False + }, + "volumeRef": "0200000060080E50002998A00000945355C37C19", + "status": "optimal", + "volumeGroupRef": "0400000060080E50002998A00000945255C37C14", + "currentManager": "070000000000000000000001", + "preferredManager": "070000000000000000000001", + "perms": { + "mapToLUN": True, + "snapShot": True, + "format": True, + "reconfigure": True, + "mirrorPrimary": True, + "mirrorSecondary": True, + "copySource": True, + "copyTarget": True, + "readable": True, + "writable": True, + "rollback": True, + "mirrorSync": True, + "newImage": True, + "allowDVE": True, + "allowDSS": True, + "concatVolumeMember": True, + "flashReadCache": True, + "asyncMirrorPrimary": True, + "asyncMirrorSecondary": True, + "pitGroup": True, + "cacheParametersChangeable": True, + "allowThinManualExpansion": False, + "allowThinGrowthParametersChange": False, + "allowVaulting": False, + "allowRestore": False + }, + "mgmtClientAttribute": 0, + "dssPreallocEnabled": True, + "dssMaxSegmentSize": 2097152, + "preReadRedundancyCheckEnabled": False, + "protectionInformationCapable": False, + "protectionType": "type1Protection", + "applicationTagOwned": False, + "untrustworthy": 0, + "volumeUse": "standardVolume", + "volumeFull": False, + "volumeCopyTarget": False, + "volumeCopySource": False, + "pitBaseVolume": False, + "asyncMirrorTarget": False, + "asyncMirrorSource": False, + "remoteMirrorSource": False, + "remoteMirrorTarget": False, + "diskPool": False, + "flashCached": False, + "increasingBy": "0", + "metadata": [], + "dataAssurance": True, + "name": "1", + "id": "0200000060080E50002998A00000945355C37C19", + "wwn": "60080E50002998A00000945355C37C19", + "objectType": "volume", + "mapped": False, + "preferredControllerId": "070000000000000000000001", + "totalSizeInBytes": "10737418240", + "onlineVolumeCopy": False, + "listOfMappings": [], + "currentControllerId": "070000000000000000000001", + "cacheSettings": { + "cwob": False, + "enterpriseCacheDump": False, + "mirrorActive": True, + "mirrorEnable": True, + "readCacheActive": True, + "readCacheEnable": True, + "writeCacheActive": True, + "writeCacheEnable": True, + "cacheFlushModifier": "flush10Sec", + "readAheadMultiplier": 1 + }, + "thinProvisioned": False + }, + { + "volumeHandle": 16385, + "worldWideName": "60080E500029347000001D7B55C3791E", + "label": "2", + "allocationGranularity": 128, + "capacity": "53687091200", + "reconPriority": 1, + "volumeRef": "3A00000060080E500029347000001D7B55C3791E", + "status": "optimal", + "repositoryRef": "3600000060080E500029347000001D7955C3791D", + "currentManager": "070000000000000000000002", + "preferredManager": "070000000000000000000002", + "perms": { + "mapToLUN": True, + "snapShot": False, + "format": True, + "reconfigure": False, + "mirrorPrimary": False, + "mirrorSecondary": False, + "copySource": True, + "copyTarget": False, + "readable": True, + "writable": True, + "rollback": True, + "mirrorSync": True, + "newImage": True, + "allowDVE": True, + "allowDSS": True, + "concatVolumeMember": False, + "flashReadCache": True, + "asyncMirrorPrimary": True, + "asyncMirrorSecondary": True, + "pitGroup": True, + "cacheParametersChangeable": True, + "allowThinManualExpansion": False, + "allowThinGrowthParametersChange": False, + "allowVaulting": False, + "allowRestore": False + }, + "mgmtClientAttribute": 0, + "preReadRedundancyCheckEnabled": False, + "protectionType": "type0Protection", + "applicationTagOwned": True, + "maxVirtualCapacity": "69269232549888", + "initialProvisionedCapacity": "4294967296", + "currentProvisionedCapacity": "4294967296", + "provisionedCapacityQuota": "55834574848", + "growthAlertThreshold": 85, + "expansionPolicy": "automatic", + "volumeCache": { + "cwob": False, + "enterpriseCacheDump": False, + "mirrorActive": True, + "mirrorEnable": True, + "readCacheActive": True, + "readCacheEnable": True, + "writeCacheActive": True, + "writeCacheEnable": True, + "cacheFlushModifier": "flush10Sec", + "readAheadMultiplier": 0 + }, + "offline": False, + "volumeFull": False, + "volumeGroupRef": "0400000060080E50002998A00000945155C37C08", + "blkSize": 512, + "storageVolumeRef": "0200000060080E500029347000001D7855C3791D", + "volumeCopyTarget": False, + "volumeCopySource": False, + "pitBaseVolume": False, + "asyncMirrorTarget": False, + "asyncMirrorSource": False, + "remoteMirrorSource": False, + "remoteMirrorTarget": False, + "flashCached": False, + "mediaScan": { + "enable": False, + "parityValidationEnable": False + }, + "metadata": [], + "dataAssurance": False, + "name": "2", + "id": "3A00000060080E500029347000001D7B55C3791E", + "wwn": "60080E500029347000001D7B55C3791E", + "objectType": "thinVolume", + "mapped": False, + "diskPool": True, + "preferredControllerId": "070000000000000000000002", + "totalSizeInBytes": "53687091200", + "onlineVolumeCopy": False, + "listOfMappings": [], + "currentControllerId": "070000000000000000000002", + "segmentSize": 131072, + "cacheSettings": { + "cwob": False, + "enterpriseCacheDump": False, + "mirrorActive": True, + "mirrorEnable": True, + "readCacheActive": True, + "readCacheEnable": True, + "writeCacheActive": True, + "writeCacheEnable": True, + "cacheFlushModifier": "flush10Sec", + "readAheadMultiplier": 0 + }, + "thinProvisioned": True + } +] + +VOLUME = VOLUMES[0] INITIATOR_NAME = 'iqn.1998-01.com.vmware:localhost-28a58148' INITIATOR_NAME_2 = 'iqn.1998-01.com.vmware:localhost-28a58149' @@ -760,6 +940,9 @@ class FakeEseriesClient(object): 'unconfiguredSpace': '0' } + def list_volume(self, volume_id): + return VOLUME + def list_volumes(self): return [VOLUME] diff --git a/cinder/tests/unit/volume/drivers/netapp/eseries/test_client.py b/cinder/tests/unit/volume/drivers/netapp/eseries/test_client.py index 3425ff255..95a05e9b9 100644 --- a/cinder/tests/unit/volume/drivers/netapp/eseries/test_client.py +++ b/cinder/tests/unit/volume/drivers/netapp/eseries/test_client.py @@ -1,5 +1,6 @@ # Copyright (c) 2014 Alex Meade # Copyright (c) 2015 Yogesh Kshirsagar +# Copyright (c) 2015 Michael Price # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may @@ -23,6 +24,8 @@ from cinder import exception from cinder import test from cinder.tests.unit.volume.drivers.netapp.eseries import fakes as \ eseries_fake +from cinder.volume.drivers.netapp.eseries import exception as es_exception + from cinder.volume.drivers.netapp.eseries import client from cinder.volume.drivers.netapp import utils as na_utils @@ -45,19 +48,53 @@ class NetAppEseriesClientDriverTestCase(test.TestCase): 'user', self.fake_password, system_id='fake_sys_id') self.my_client.client._endpoint = eseries_fake.FAKE_ENDPOINT_HTTP - self.mock_object(self.my_client, '_eval_response') fake_response = mock.Mock() fake_response.status_code = 200 self.my_client.invoke_service = mock.Mock(return_value=fake_response) + @ddt.data(200, 201, 203, 204) + def test_eval_response_success(self, status_code): + fake_resp = mock.Mock() + fake_resp.status_code = status_code + + self.assertIsNone(self.my_client._eval_response(fake_resp)) + + @ddt.data(300, 400, 404, 500) + def test_eval_response_failure(self, status_code): + fake_resp = mock.Mock() + fake_resp.status_code = status_code + expected_msg = "Response error code - %s." % status_code + + with self.assertRaisesRegexp(es_exception.WebServiceException, + expected_msg) as exc: + self.my_client._eval_response(fake_resp) + + self.assertEqual(status_code, exc.status_code) + + def test_eval_response_422(self): + status_code = 422 + resp_text = "Fake Error Message" + fake_resp = mock.Mock() + fake_resp.status_code = status_code + fake_resp.text = resp_text + expected_msg = "Response error - %s." % resp_text + + with self.assertRaisesRegexp(es_exception.WebServiceException, + expected_msg) as exc: + self.my_client._eval_response(fake_resp) + + self.assertEqual(status_code, exc.status_code) + def test_register_storage_system_does_not_log_password(self): + self.my_client._eval_response = mock.Mock() self.my_client.register_storage_system([], password=self.fake_password) for call in self.mock_log.debug.mock_calls: __, args, __ = call self.assertNotIn(self.fake_password, args[0]) def test_update_stored_system_password_does_not_log_password(self): + self.my_client._eval_response = mock.Mock() self.my_client.update_stored_system_password( password=self.fake_password) for call in self.mock_log.debug.mock_calls: @@ -405,6 +442,126 @@ class NetAppEseriesClientDriverTestCase(test.TestCase): self.assertEqual(fake_pool, pool) + @ddt.data(('volumes', True), ('volumes', False), + ('volume', True), ('volume', False)) + @ddt.unpack + def test_get_volume_api_path(self, path_key, ssc_available): + self.my_client.features = mock.Mock() + self.my_client.features.SSC_API_V2 = na_utils.FeatureState( + supported=ssc_available) + expected_key = 'ssc_' + path_key if ssc_available else path_key + expected = self.my_client.RESOURCE_PATHS.get(expected_key) + + actual = self.my_client._get_volume_api_path(path_key) + + self.assertEqual(expected, actual) + + @ddt.data(True, False) + def test_get_volume_api_path_invalid(self, ssc_available): + key = 'invalidKey' + self.my_client.features = mock.Mock() + self.my_client.features.SSC_API_V2 = na_utils.FeatureState( + supported=ssc_available) + + self.assertRaises(KeyError, self.my_client._get_volume_api_path, key) + + def test_list_volumes(self): + url = client.RestClient.RESOURCE_PATHS['ssc_volumes'] + self.my_client.features = mock.Mock() + self.my_client.features.SSC_API_V2 = na_utils.FeatureState( + supported=True) + self.my_client._invoke = mock.Mock( + return_value=eseries_fake.VOLUMES) + + volumes = client.RestClient.list_volumes(self.my_client) + + self.assertEqual(eseries_fake.VOLUMES, volumes) + self.my_client._invoke.assert_called_once_with('GET', url) + + @ddt.data(client.RestClient.ID, client.RestClient.WWN, + client.RestClient.NAME) + def test_list_volume_v1(self, uid_field_name): + url = client.RestClient.RESOURCE_PATHS['volumes'] + self.my_client.features = mock.Mock() + self.my_client.features.SSC_API_V2 = na_utils.FeatureState( + supported=False) + fake_volume = copy.deepcopy(eseries_fake.VOLUME) + self.my_client._invoke = mock.Mock( + return_value=eseries_fake.VOLUMES) + + volume = client.RestClient.list_volume(self.my_client, + fake_volume[uid_field_name]) + + self.my_client._invoke.assert_called_once_with('GET', url) + self.assertEqual(fake_volume, volume) + + def test_list_volume_v1_not_found(self): + url = client.RestClient.RESOURCE_PATHS['volumes'] + self.my_client.features = mock.Mock() + self.my_client.features.SSC_API_V2 = na_utils.FeatureState( + supported=False) + self.my_client._invoke = mock.Mock( + return_value=eseries_fake.VOLUMES) + + self.assertRaises(exception.VolumeNotFound, + client.RestClient.list_volume, + self.my_client, 'fakeId') + self.my_client._invoke.assert_called_once_with('GET', url) + + def test_list_volume_v2(self): + url = client.RestClient.RESOURCE_PATHS['ssc_volume'] + self.my_client.features = mock.Mock() + self.my_client.features.SSC_API_V2 = na_utils.FeatureState( + supported=True) + fake_volume = copy.deepcopy(eseries_fake.VOLUME) + self.my_client._invoke = mock.Mock(return_value=fake_volume) + + volume = client.RestClient.list_volume(self.my_client, + fake_volume['id']) + + self.my_client._invoke.assert_called_once_with('GET', url, + **{'object-id': + mock.ANY}) + self.assertEqual(fake_volume, volume) + + def test_list_volume_v2_not_found(self): + status_code = 404 + url = client.RestClient.RESOURCE_PATHS['ssc_volume'] + self.my_client.features = mock.Mock() + self.my_client.features.SSC_API_V2 = na_utils.FeatureState( + supported=True) + msg = "Response error code - %s." % status_code + self.my_client._invoke = mock.Mock( + side_effect=es_exception.WebServiceException(message=msg, + status_code= + status_code)) + + self.assertRaises(exception.VolumeNotFound, + client.RestClient.list_volume, + self.my_client, 'fakeId') + self.my_client._invoke.assert_called_once_with('GET', url, + **{'object-id': + mock.ANY}) + + def test_list_volume_v2_failure(self): + status_code = 422 + url = client.RestClient.RESOURCE_PATHS['ssc_volume'] + self.my_client.features = mock.Mock() + self.my_client.features.SSC_API_V2 = na_utils.FeatureState( + supported=True) + msg = "Response error code - %s." % status_code + self.my_client._invoke = mock.Mock( + side_effect=es_exception.WebServiceException(message=msg, + status_code= + status_code)) + + self.assertRaises(es_exception.WebServiceException, + client.RestClient.list_volume, self.my_client, + 'fakeId') + self.my_client._invoke.assert_called_once_with('GET', url, + **{'object-id': + mock.ANY}) + def test_create_volume_V1(self): self.my_client.features = mock.Mock() self.my_client.features.SSC_API_V2 = na_utils.FeatureState( @@ -448,6 +605,52 @@ class NetAppEseriesClientDriverTestCase(test.TestCase): client.RestClient.create_volume, self.my_client, '1', 'label', 1, read_cache=True) + @ddt.data(True, False) + def test_update_volume(self, ssc_api_enabled): + label = 'updatedName' + fake_volume = copy.deepcopy(eseries_fake.VOLUME) + expected_volume = copy.deepcopy(fake_volume) + expected_volume['name'] = label + self.my_client.features = mock.Mock() + self.my_client.features.SSC_API_V2 = na_utils.FeatureState( + supported=ssc_api_enabled) + self.my_client._invoke = mock.Mock(return_value=expected_volume) + + updated_volume = self.my_client.update_volume(fake_volume['id'], + label) + + if ssc_api_enabled: + url = self.my_client.RESOURCE_PATHS.get('ssc_volume') + else: + url = self.my_client.RESOURCE_PATHS.get('volume') + + self.my_client._invoke.assert_called_once_with('POST', url, + {'name': label}, + **{'object-id': + fake_volume['id']} + ) + self.assertDictMatch(expected_volume, updated_volume) + + @ddt.data(True, False) + def test_delete_volume(self, ssc_api_enabled): + fake_volume = copy.deepcopy(eseries_fake.VOLUME) + self.my_client.features = mock.Mock() + self.my_client.features.SSC_API_V2 = na_utils.FeatureState( + supported=ssc_api_enabled) + self.my_client._invoke = mock.Mock() + + self.my_client.delete_volume(fake_volume['id']) + + if ssc_api_enabled: + url = self.my_client.RESOURCE_PATHS.get('ssc_volume') + else: + url = self.my_client.RESOURCE_PATHS.get('volume') + + self.my_client._invoke.assert_called_once_with('DELETE', url, + **{'object-id': + fake_volume['id']} + ) + @ddt.data('00.00.00.00', '01.52.9000.2', '01.52.9001.2', '01.51.9000.3', '01.51.9001.3', '01.51.9010.5', '0.53.9000.3', '0.53.9001.4') def test_api_version_not_support_asup(self, api_version): @@ -486,7 +689,7 @@ class NetAppEseriesClientDriverTestCase(test.TestCase): self.assertFalse(self.my_client.features.SSC_API_V2.supported) @ddt.data('01.53.9000.1', '01.53.9000.5', '01.53.8999.1', - '01.53.9010.20', '01.53.9010.16', '01.54.9000.1', + '01.53.9010.20', '01.53.9010.17', '01.54.9000.1', '02.51.9000.3', '02.52.8999.3', '02.51.8999.2') def test_api_version_supports_ssc_api(self, api_version): diff --git a/cinder/tests/unit/volume/drivers/netapp/eseries/test_library.py b/cinder/tests/unit/volume/drivers/netapp/eseries/test_library.py index bff644fb4..6ab343b51 100644 --- a/cinder/tests/unit/volume/drivers/netapp/eseries/test_library.py +++ b/cinder/tests/unit/volume/drivers/netapp/eseries/test_library.py @@ -2,6 +2,7 @@ # Copyright (c) 2015 Alex Meade # Copyright (c) 2015 Rushil Chugh # Copyright (c) 2015 Yogesh Kshirsagar +# Copyright (c) 2015 Michael Price # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may @@ -85,6 +86,29 @@ class NetAppEseriesLibraryTestCase(test.TestCase): filtered_pool_labels = [pool['label'] for pool in filtered_pools] self.assertListEqual(pool_labels, filtered_pool_labels) + def test_get_volume(self): + fake_volume = copy.deepcopy(get_fake_volume()) + volume = copy.deepcopy(eseries_fake.VOLUME) + self.library._client.list_volume = mock.Mock(return_value=volume) + + result = self.library._get_volume(fake_volume['id']) + + self.assertEqual(1, self.library._client.list_volume.call_count) + self.assertDictMatch(volume, result) + + def test_get_volume_bad_input(self): + volume = copy.deepcopy(eseries_fake.VOLUME) + self.library._client.list_volume = mock.Mock(return_value=volume) + + self.assertRaises(exception.InvalidInput, self.library._get_volume, + None) + + def test_get_volume_bad_uuid(self): + volume = copy.deepcopy(eseries_fake.VOLUME) + self.library._client.list_volume = mock.Mock(return_value=volume) + + self.assertRaises(ValueError, self.library._get_volume, '1') + def test_update_ssc_info_no_ssc(self): drives = [{'currentVolumeGroupRef': 'test_vg1', 'driveMediaType': 'ssd'}] @@ -139,11 +163,15 @@ class NetAppEseriesLibraryTestCase(test.TestCase): da_enabled = pool['dataAssuranceCapable'] and ( data_assurance_supported) + thin_provisioned = pool['thinProvisioningCapable'] + expected = { 'netapp_disk_encryption': six.text_type(pool['encrypted']).lower(), 'netapp_eseries_flash_read_cache': six.text_type(pool['flashCacheCapable']).lower(), + 'netapp_thin_provisioned': + six.text_type(thin_provisioned).lower(), 'netapp_eseries_data_assurance': six.text_type(da_enabled).lower(), 'netapp_eseries_disk_spindle_speed': pool['spindleSpeed'], @@ -230,6 +258,9 @@ class NetAppEseriesLibraryTestCase(test.TestCase): def test_terminate_connection_iscsi_volume_not_mapped(self): connector = {'initiator': eseries_fake.INITIATOR_NAME} + volume = copy.deepcopy(eseries_fake.VOLUME) + volume['listOfMappings'] = [] + self.library._get_volume = mock.Mock(return_value=volume) self.assertRaises(eseries_exc.VolumeNotMapped, self.library.terminate_connection_iscsi, get_fake_volume(), @@ -241,8 +272,8 @@ class NetAppEseriesLibraryTestCase(test.TestCase): fake_eseries_volume['listOfMappings'] = [ eseries_fake.VOLUME_MAPPING ] - self.mock_object(self.library._client, 'list_volumes', - mock.Mock(return_value=[fake_eseries_volume])) + self.mock_object(self.library._client, 'list_volume', + mock.Mock(return_value=fake_eseries_volume)) self.mock_object(host_mapper, 'unmap_volume_from_host') self.library.terminate_connection_iscsi(get_fake_volume(), connector) @@ -267,6 +298,12 @@ class NetAppEseriesLibraryTestCase(test.TestCase): self.mock_object(host_mapper, 'map_volume_to_single_host', mock.Mock( return_value=eseries_fake.VOLUME_MAPPING)) + fake_eseries_volume = copy.deepcopy(eseries_fake.VOLUME) + fake_eseries_volume['listOfMappings'] = [ + eseries_fake.VOLUME_MAPPING + ] + self.mock_object(self.library._client, 'list_volume', + mock.Mock(return_value=fake_eseries_volume)) self.library.initialize_connection_iscsi(get_fake_volume(), connector) @@ -287,6 +324,12 @@ class NetAppEseriesLibraryTestCase(test.TestCase): self.mock_object(host_mapper, 'map_volume_to_single_host', mock.Mock( return_value=eseries_fake.VOLUME_MAPPING)) + fake_eseries_volume = copy.deepcopy(eseries_fake.VOLUME) + fake_eseries_volume['listOfMappings'] = [ + eseries_fake.VOLUME_MAPPING + ] + self.mock_object(self.library._client, 'list_volume', + mock.Mock(return_value=fake_eseries_volume)) self.library.initialize_connection_iscsi(get_fake_volume(), connector) @@ -303,6 +346,9 @@ class NetAppEseriesLibraryTestCase(test.TestCase): self.mock_object(host_mapper, 'map_volume_to_single_host', mock.Mock( return_value=eseries_fake.VOLUME_MAPPING)) + fake_eseries_volume = copy.deepcopy(eseries_fake.VOLUME) + self.mock_object(self.library._client, 'list_volume', + mock.Mock(return_value=fake_eseries_volume)) self.library.initialize_connection_iscsi(get_fake_volume(), connector) @@ -398,6 +444,10 @@ class NetAppEseriesLibraryTestCase(test.TestCase): 'type': 'fc', 'address': eseries_fake.WWPN }] + volume = copy.deepcopy(eseries_fake.VOLUME) + volume['listOfMappings'] = [] + self.mock_object(self.library, '_get_volume', + mock.Mock(return_value=volume)) self.mock_object(self.library._client, 'list_hosts', mock.Mock(return_value=[fake_host])) @@ -421,8 +471,8 @@ class NetAppEseriesLibraryTestCase(test.TestCase): ] self.mock_object(self.library._client, 'list_hosts', mock.Mock(return_value=[fake_host])) - self.mock_object(self.library._client, 'list_volumes', - mock.Mock(return_value=[fake_eseries_volume])) + self.mock_object(self.library._client, 'list_volume', + mock.Mock(return_value=fake_eseries_volume)) self.mock_object(host_mapper, 'unmap_volume_from_host') self.library.terminate_connection_fc(get_fake_volume(), connector) @@ -447,8 +497,8 @@ class NetAppEseriesLibraryTestCase(test.TestCase): ] self.mock_object(self.library._client, 'list_hosts', mock.Mock(return_value=[fake_host])) - self.mock_object(self.library._client, 'list_volumes', - mock.Mock(return_value=[fake_eseries_volume])) + self.mock_object(self.library._client, 'list_volume', + mock.Mock(return_value=fake_eseries_volume)) self.mock_object(host_mapper, 'unmap_volume_from_host') self.mock_object(self.library._client, 'get_volume_mappings_for_host', mock.Mock(return_value=[copy.deepcopy @@ -484,8 +534,8 @@ class NetAppEseriesLibraryTestCase(test.TestCase): ] self.mock_object(self.library._client, 'list_hosts', mock.Mock(return_value=[fake_host])) - self.mock_object(self.library._client, 'list_volumes', - mock.Mock(return_value=[fake_eseries_volume])) + self.mock_object(self.library._client, 'list_volume', + mock.Mock(return_value=fake_eseries_volume)) self.mock_object(host_mapper, 'unmap_volume_from_host') self.mock_object(self.library._client, 'get_volume_mappings_for_host', mock.Mock(return_value=[])) @@ -738,6 +788,9 @@ class NetAppEseriesLibraryMultiAttachTestCase(test.TestCase): @ddt.data(('netapp_eseries_flash_read_cache', 'flash_cache', 'true'), ('netapp_eseries_flash_read_cache', 'flash_cache', 'false'), ('netapp_eseries_flash_read_cache', 'flash_cache', None), + ('netapp_thin_provisioned', 'thin_provision', 'true'), + ('netapp_thin_provisioned', 'thin_provision', 'false'), + ('netapp_thin_provisioned', 'thin_provision', None), ('netapp_eseries_data_assurance', 'data_assurance', 'true'), ('netapp_eseries_data_assurance', 'data_assurance', 'false'), ('netapp_eseries_data_assurance', 'data_assurance', None), @@ -885,14 +938,6 @@ class NetAppEseriesLibraryMultiAttachTestCase(test.TestCase): # Ensure the volume we created is not cleaned up self.assertEqual(0, self.library._client.delete_volume.call_count) - def test_get_non_existing_volume_raises_keyerror(self): - volume2 = get_fake_volume() - # Change to a nonexistent id. - volume2['name_id'] = '88888888-4444-4444-4444-cccccccccccc' - self.assertRaises(KeyError, - self.library._get_volume, - volume2['name_id']) - def test_delete_non_existing_volume(self): volume2 = get_fake_volume() # Change to a nonexistent id. diff --git a/cinder/volume/drivers/netapp/eseries/client.py b/cinder/volume/drivers/netapp/eseries/client.py index 03891f15c..e045d36cf 100644 --- a/cinder/volume/drivers/netapp/eseries/client.py +++ b/cinder/volume/drivers/netapp/eseries/client.py @@ -3,6 +3,7 @@ # Copyright (c) 2015 Alex Meade # Copyright (c) 2015 Rushil Chugh # Copyright (c) 2015 Yogesh Kshirsagar +# Copyright (c) 2015 Michael Price # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may @@ -32,6 +33,7 @@ from six.moves import urllib from cinder import exception from cinder.i18n import _ import cinder.utils as cinder_utils +from cinder.volume.drivers.netapp.eseries import exception as es_exception from cinder.volume.drivers.netapp.eseries import utils from cinder.volume.drivers.netapp import utils as na_utils @@ -46,9 +48,20 @@ LOG = logging.getLogger(__name__) class RestClient(object): """REST client specific to e-series storage service.""" + ID = 'id' + WWN = 'worldWideName' + NAME = 'label' + ASUP_VALID_VERSION = (1, 52, 9000, 3) # We need to check for both the release and the pre-release versions - SSC_VALID_VERSIONS = ((1, 53, 9000, 1), (1, 53, 9010, 16)) + SSC_VALID_VERSIONS = ((1, 53, 9000, 1), (1, 53, 9010, 17)) + + RESOURCE_PATHS = { + 'volumes': '/storage-systems/{system-id}/volumes', + 'volume': '/storage-systems/{system-id}/volumes/{object-id}', + 'ssc_volumes': '/storage-systems/{system-id}/ssc/volumes', + 'ssc_volume': '/storage-systems/{system-id}/ssc/volumes/{object-id}' + } def __init__(self, scheme, host, port, service_path, username, password, **kwargs): @@ -210,11 +223,22 @@ class RestClient(object): msg = _("Response error - %s.") % response.text else: msg = _("Response error code - %s.") % status_code - raise exception.NetAppDriverException(msg) + raise es_exception.WebServiceException(msg, + status_code=status_code) + + def _get_volume_api_path(self, path_key): + """Retrieve the correct API path based on API availability + + :param path_key: The volume API to request (volume or volumes) + :raise KeyError: If the path_key is not valid + """ + if self.features.SSC_API_V2: + path_key = 'ssc_' + path_key + return self.RESOURCE_PATHS[path_key] def create_volume(self, pool, label, size, unit='gb', seg_size=0, read_cache=None, write_cache=None, flash_cache=None, - data_assurance=None): + data_assurance=None, thin_provision=False): """Creates a volume on array with the configured attributes Note: if read_cache, write_cache, flash_cache, or data_assurance @@ -242,7 +266,8 @@ class RestClient(object): 'size': int(size), 'dataAssuranceEnable': data_assurance, 'flashCacheEnable': flash_cache, 'readCacheEnable': read_cache, - 'writeCacheEnable': write_cache} + 'writeCacheEnable': write_cache, + 'thinProvision': thin_provision} # Use the old API else: # Determine if there are were extra specs provided that are not @@ -267,22 +292,61 @@ class RestClient(object): def delete_volume(self, object_id): """Deletes given volume from array.""" - path = "/storage-systems/{system-id}/volumes/{object-id}" + if self.features.SSC_API_V2: + path = self.RESOURCE_PATHS.get('ssc_volume') + else: + path = self.RESOURCE_PATHS.get('volume') return self._invoke('DELETE', path, **{'object-id': object_id}) def list_volumes(self): """Lists all volumes in storage array.""" - path = "/storage-systems/{system-id}/volumes" + if self.features.SSC_API_V2: + path = self.RESOURCE_PATHS.get('ssc_volumes') + else: + path = self.RESOURCE_PATHS.get('volumes') + return self._invoke('GET', path) def list_volume(self, object_id): - """List given volume from array.""" - path = "/storage-systems/{system-id}/volumes/{object-id}" - return self._invoke('GET', path, **{'object-id': object_id}) + """Retrieve the given volume from array. + + :param object_id: The volume id, label, or wwn + :return The volume identified by object_id + :raise VolumeNotFound if the volume could not be found + """ + + if self.features.SSC_API_V2: + return self._list_volume_v2(object_id) + # The new API is not available, + else: + # Search for the volume with label, id, or wwn. + return self._list_volume_v1(object_id) + + def _list_volume_v1(self, object_id): + # Search for the volume with label, id, or wwn. + for vol in self.list_volumes(): + if (object_id == vol.get(self.NAME) or object_id == vol.get( + self.WWN) or object_id == vol.get(self.ID)): + return vol + # The volume could not be found + raise exception.VolumeNotFound(volume_id=object_id) + + def _list_volume_v2(self, object_id): + path = self.RESOURCE_PATHS.get('ssc_volume') + try: + return self._invoke('GET', path, **{'object-id': object_id}) + except es_exception.WebServiceException as e: + if(404 == e.status_code): + raise exception.VolumeNotFound(volume_id=object_id) + else: + raise def update_volume(self, object_id, label): - """Renames given volume in array.""" - path = "/storage-systems/{system-id}/volumes/{object-id}" + """Renames given volume on array.""" + if self.features.SSC_API_V2: + path = self.RESOURCE_PATHS.get('ssc_volume') + else: + path = self.RESOURCE_PATHS.get('volume') data = {'name': label} return self._invoke('POST', path, data, **{'object-id': object_id}) diff --git a/cinder/volume/drivers/netapp/eseries/exception.py b/cinder/volume/drivers/netapp/eseries/exception.py index a12ff53b2..c2c517127 100644 --- a/cinder/volume/drivers/netapp/eseries/exception.py +++ b/cinder/volume/drivers/netapp/eseries/exception.py @@ -1,4 +1,5 @@ # Copyright (c) 2015 Alex Meade. All Rights Reserved. +# Copyright (c) 2015 Michael Price. All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may # not use this file except in compliance with the License. You may obtain @@ -24,3 +25,9 @@ class VolumeNotMapped(exception.NetAppDriverException): class UnsupportedHostGroup(exception.NetAppDriverException): message = _("Volume %(volume_id)s is currently mapped to unsupported " "host group %(group)s") + + +class WebServiceException(exception.NetAppDriverException): + def __init__(self, message=None, status_code=None): + self.status_code = status_code + super(WebServiceException, self).__init__(message=message) diff --git a/cinder/volume/drivers/netapp/eseries/library.py b/cinder/volume/drivers/netapp/eseries/library.py index 71728de5b..27a3a653c 100644 --- a/cinder/volume/drivers/netapp/eseries/library.py +++ b/cinder/volume/drivers/netapp/eseries/library.py @@ -2,6 +2,7 @@ # Copyright (c) 2015 Rushil Chugh # Copyright (c) 2015 Navneet Singh # Copyright (c) 2015 Yogesh Kshirsagar +# Copyright (c) 2015 Michael Price # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may @@ -110,6 +111,7 @@ class NetAppESeriesLibrary(object): ENCRYPTION_UQ_SPEC = 'netapp_disk_encryption' SPINDLE_SPD_UQ_SPEC = 'netapp_eseries_disk_spindle_speed' RAID_UQ_SPEC = 'netapp_raid_type' + THIN_UQ_SPEC = 'netapp_thin_provisioned' SSC_UPDATE_INTERVAL = 60 # seconds WORLDWIDENAME = 'worldWideName' @@ -280,27 +282,14 @@ class NetAppESeriesLibrary(object): return True def _get_volume(self, uid): - label = utils.convert_uuid_to_es_fmt(uid) - return self._get_volume_with_label_wwn(label) - - def _get_volume_with_label_wwn(self, label=None, wwn=None): - """Searches volume with label or wwn or both.""" - if not (label or wwn): - raise exception.InvalidInput(_('Either volume label or wwn' - ' is required as input.')) - wwn = wwn.replace(':', '').upper() if wwn else None - eseries_volume = None - for vol in self._client.list_volumes(): - if label and vol.get('label') != label: - continue - if wwn and vol.get(self.WORLDWIDENAME).upper() != wwn: - continue - eseries_volume = vol - break - - if not eseries_volume: - raise KeyError() - return eseries_volume + """Retrieve a volume by its label""" + if uid is None: + raise exception.InvalidInput(_('The volume label is required' + ' as input.')) + + uid = utils.convert_uuid_to_es_fmt(uid) + + return self._client.list_volume(uid) def _get_snapshot_group_for_snapshot(self, snapshot_id): label = utils.convert_uuid_to_es_fmt(snapshot_id) @@ -400,6 +389,10 @@ class NetAppESeriesLibrary(object): if data_assurance is not None: data_assurance = na_utils.to_bool(data_assurance) + thin_provision = extra_specs.get(self.THIN_UQ_SPEC) + if(thin_provision is not None): + thin_provision = na_utils.to_bool(thin_provision) + target_pool = None pools = self._get_storage_pools() @@ -418,7 +411,8 @@ class NetAppESeriesLibrary(object): read_cache=read_cache, write_cache=write_cache, flash_cache=flash_cache, - data_assurance=data_assurance) + data_assurance=data_assurance, + thin_provision=thin_provision) LOG.info(_LI("Created volume with " "label %s."), eseries_volume_label) except exception.NetAppDriverException as e: @@ -1018,9 +1012,8 @@ class NetAppESeriesLibrary(object): pool_ssc_info = ssc_stats[poolId] - encrypted = pool['encrypted'] pool_ssc_info[self.ENCRYPTION_UQ_SPEC] = ( - six.text_type(encrypted).lower()) + six.text_type(pool['encrypted']).lower()) pool_ssc_info[self.SPINDLE_SPD_UQ_SPEC] = (pool['spindleSpeed']) @@ -1037,6 +1030,9 @@ class NetAppESeriesLibrary(object): pool_ssc_info[self.RAID_UQ_SPEC] = ( self.SSC_RAID_TYPE_MAPPING.get(pool['raidLevel'], 'unknown')) + pool_ssc_info[self.THIN_UQ_SPEC] = ( + six.text_type(pool['thinProvisioningCapable']).lower()) + if pool['pool'].get("driveMediaType") == 'ssd': pool_ssc_info[self.DISK_TYPE_UQ_SPEC] = 'SSD' else: @@ -1182,7 +1178,7 @@ class NetAppESeriesLibrary(object): @cinder_utils.synchronized('manage_existing') def manage_existing(self, volume, existing_ref): """Brings an existing storage object under Cinder management.""" - vol = self._get_existing_vol_with_manage_ref(volume, existing_ref) + vol = self._get_existing_vol_with_manage_ref(existing_ref) label = utils.convert_uuid_to_es_fmt(volume['id']) if label == vol['label']: LOG.info(_LI("Volume with given ref %s need not be renamed during" @@ -1199,13 +1195,14 @@ class NetAppESeriesLibrary(object): When calculating the size, round up to the next GB. """ - vol = self._get_existing_vol_with_manage_ref(volume, existing_ref) + vol = self._get_existing_vol_with_manage_ref(existing_ref) return int(math.ceil(float(vol['capacity']) / units.Gi)) - def _get_existing_vol_with_manage_ref(self, volume, existing_ref): + def _get_existing_vol_with_manage_ref(self, existing_ref): try: - return self._get_volume_with_label_wwn( - existing_ref.get('source-name'), existing_ref.get('source-id')) + vol_id = existing_ref.get('source-name') or existing_ref.get( + 'source-id') + return self._client.list_volume(vol_id) except exception.InvalidInput: reason = _('Reference must contain either source-name' ' or source-id element.')