VNX Cinder driver Over Subscription Support

Add over subscription support in EMC VNX Cinder driver

Change-Id: I2c9b5c605f9f938e5582a823da1ff2054c8d328a
Implements: blueprint vnx-over-subscription-thin-provisioning

diff --git a/cinder/tests/unit/test_emc_vnxdirect.py b/cinder/tests/unit/test_emc_vnxdirect.py
index 58fdd5f24fa4dcaf70a6fb25c161129ee85d2bee..a1a546da54b7519bed857ae3cd43e48d7e0cbec7 100644 (file)
--- a/cinder/tests/unit/test_emc_vnxdirect.py
+++ b/cinder/tests/unit/test_emc_vnxdirect.py
@@ -376,15 +376,16 @@ class EMCVNXCLIDriverTestData(object):
 
     POOL_PROPERTY_W_FASTCACHE_CMD = ('storagepool', '-list', '-name',
                                      'unit_test_pool', '-availableCap',
-                                     '-userCap', '-fastcache', '-state')
+                                     '-userCap', '-fastcache', '-state',
+                                     '-subscribedCap')
 
     def POOL_GET_ALL_CMD(self, withfastcache=False):
         if withfastcache:
             return ('storagepool', '-list', '-availableCap',
-                    '-userCap', '-fastcache', '-state')
+                    '-userCap', '-fastcache', '-state', '-subscribedCap')
         else:
             return ('storagepool', '-list', '-availableCap',
-                    '-userCap', '-state')
+                    '-userCap', '-state', '-subscribedCap')
 
     def POOL_GET_ALL_RESULT(self, withfastcache=False):
         if withfastcache:
@@ -394,6 +395,7 @@ class EMCVNXCLIDriverTestData(object):
                     "User Capacity (GBs):  3281.146\n"
                     "Available Capacity (Blocks):  6512292864\n"
                     "Available Capacity (GBs):  3105.303\n"
+                    "Total Subscribed Capacity (GBs):  536.140\n"
                     "FAST Cache:  Enabled\n"
                     "State: Ready\n"
                     "\n"
@@ -403,6 +405,7 @@ class EMCVNXCLIDriverTestData(object):
                     "User Capacity (GBs):  4099.992\n"
                     "Available Capacity (Blocks):  8356663296\n"
                     "Available Capacity (GBs):  3984.768\n"
+                    "Total Subscribed Capacity (GBs):  636.240\n"
                     "FAST Cache:  Disabled\n"
                     "State: Ready\n", 0)
         else:
@@ -412,6 +415,7 @@ class EMCVNXCLIDriverTestData(object):
                     "User Capacity (GBs):  3281.146\n"
                     "Available Capacity (Blocks):  6512292864\n"
                     "Available Capacity (GBs):  3105.303\n"
+                    "Total Subscribed Capacity (GBs):  536.140\n"
                     "State: Ready\n"
                     "\n"
                     "Pool Name:  unit test pool 2\n"
@@ -420,6 +424,7 @@ class EMCVNXCLIDriverTestData(object):
                     "User Capacity (GBs):  4099.992\n"
                     "Available Capacity (Blocks):  8356663296\n"
                     "Available Capacity (GBs):  3984.768\n"
+                    "Total Subscribed Capacity (GBs):  636.240\n"
                     "State: Ready\n", 0)
 
     def POOL_GET_ALL_STATES_TEST(self, states=['Ready']):
@@ -673,6 +678,7 @@ State: Ready
         "User Capacity (GBs):  3281.146\n"
         "Available Capacity (Blocks):  6832207872\n"
         "Available Capacity (GBs):  3257.851\n"
+        "Total Subscribed Capacity (GBs):  636.240\n"
         "FAST Cache:  Enabled\n"
         "State: Ready\n\n", 0)
 
@@ -1115,6 +1121,65 @@ class EMCVNXCLIDriverISCSITestCase(DriverTestCaseBase):
                 1))]
         fake_cli.assert_has_calls(expect_cmd)
 
+    @mock.patch(
+        'cinder.openstack.common.loopingcall.FixedIntervalLoopingCall',
+        new=utils.ZeroIntervalLoopingCall)
+    @mock.patch(
+        "cinder.volume.volume_types."
+        "get_volume_type_extra_specs",
+        mock.Mock(return_value={'provisioning:type': 'thin',
+                                'storagetype:provisioning': 'thick'}))
+    def test_create_volume_thin(self):
+        commands = [self.testData.LUN_PROPERTY_ALL_CMD('vol_with_type'),
+                    self.testData.NDU_LIST_CMD]
+        results = [self.testData.LUN_PROPERTY('vol_with_type', True),
+                   self.testData.NDU_LIST_RESULT]
+        fake_cli = self.driverSetup(commands, results)
+        self.driver.cli.enablers = ['-Compression',
+                                    '-Deduplication',
+                                    '-ThinProvisioning',
+                                    '-FAST']
+        # case
+        self.driver.create_volume(self.testData.test_volume_with_type)
+        # verification
+        expect_cmd = [
+            mock.call(*self.testData.LUN_CREATION_CMD(
+                'vol_with_type', 1,
+                'unit_test_pool',
+                'thin', None, False)),
+            mock.call(*self.testData.LUN_PROPERTY_ALL_CMD(
+                'vol_with_type'), poll=False)]
+        fake_cli.assert_has_calls(expect_cmd)
+
+    @mock.patch(
+        'cinder.openstack.common.loopingcall.FixedIntervalLoopingCall',
+        new=utils.ZeroIntervalLoopingCall)
+    @mock.patch(
+        "cinder.volume.volume_types."
+        "get_volume_type_extra_specs",
+        mock.Mock(return_value={'provisioning:type': 'thick'}))
+    def test_create_volume_thick(self):
+        commands = [self.testData.LUN_PROPERTY_ALL_CMD('vol_with_type'),
+                    self.testData.NDU_LIST_CMD]
+        results = [self.testData.LUN_PROPERTY('vol_with_type', False),
+                   self.testData.NDU_LIST_RESULT]
+        fake_cli = self.driverSetup(commands, results)
+        self.driver.cli.enablers = ['-Compression',
+                                    '-Deduplication',
+                                    '-ThinProvisioning',
+                                    '-FAST']
+        # case
+        self.driver.create_volume(self.testData.test_volume_with_type)
+        # verification
+        expect_cmd = [
+            mock.call(*self.testData.LUN_CREATION_CMD(
+                'vol_with_type', 1,
+                'unit_test_pool',
+                'thick', None, False)),
+            mock.call(*self.testData.LUN_PROPERTY_ALL_CMD(
+                'vol_with_type'), poll=False)]
+        fake_cli.assert_has_calls(expect_cmd)
+
     @mock.patch(
         "eventlet.event.Event.wait",
         mock.Mock(return_value=None))
@@ -1276,9 +1341,12 @@ class EMCVNXCLIDriverISCSITestCase(DriverTestCaseBase):
             'reserved_percentage': 3,
             'location_info': 'unit_test_pool|fakeSerial',
             'total_capacity_gb': 3281.146,
+            'provisioned_capacity_gb': 636.240,
             'compression_support': 'True',
             'deduplication_support': 'True',
-            'thinprovisioning_support': 'True',
+            'thin_provisioning_support': True,
+            'thick_provisioning_support': True,
+            'max_over_subscription_ratio': 20.0,
             'consistencygroup_support': 'True',
             'pool_name': 'unit_test_pool',
             'fast_cache_enabled': 'True',
@@ -2831,6 +2899,7 @@ Time Remaining:  0 second(s)
             'lun_nums': 1000,
             'total_capacity_gb': 10,
             'free_capacity_gb': 5,
+            'provisioned_capacity_gb': 8,
             'pool_name': "unit_test_pool",
             'fast_cache_enabled': 'True',
             'state': 'Ready'})
@@ -3223,11 +3292,14 @@ class EMCVNXCLIDArrayBasedDriverTestCase(DriverTestCaseBase):
             'reserved_percentage': 2,
             'location_info': 'unit_test_pool1|fakeSerial',
             'total_capacity_gb': 3281.146,
+            'provisioned_capacity_gb': 536.140,
             'compression_support': 'True',
             'deduplication_support': 'True',
-            'thinprovisioning_support': 'True',
+            'thin_provisioning_support': True,
+            'thick_provisioning_support': True,
             'consistencygroup_support': 'True',
             'pool_name': 'unit_test_pool1',
+            'max_over_subscription_ratio': 20.0,
             'fast_cache_enabled': 'True',
             'fast_support': 'True'}
         self.assertEqual(expected_pool_stats1, pool_stats1)
@@ -3238,11 +3310,14 @@ class EMCVNXCLIDArrayBasedDriverTestCase(DriverTestCaseBase):
             'reserved_percentage': 2,
             'location_info': 'unit test pool 2|fakeSerial',
             'total_capacity_gb': 4099.992,
+            'provisioned_capacity_gb': 636.240,
             'compression_support': 'True',
             'deduplication_support': 'True',
-            'thinprovisioning_support': 'True',
+            'thin_provisioning_support': True,
+            'thick_provisioning_support': True,
             'consistencygroup_support': 'True',
             'pool_name': 'unit test pool 2',
+            'max_over_subscription_ratio': 20.0,
             'fast_cache_enabled': 'False',
             'fast_support': 'True'}
         self.assertEqual(expected_pool_stats2, pool_stats2)
@@ -3263,11 +3338,14 @@ class EMCVNXCLIDArrayBasedDriverTestCase(DriverTestCaseBase):
             'reserved_percentage': 2,
             'location_info': 'unit_test_pool1|fakeSerial',
             'total_capacity_gb': 3281.146,
+            'provisioned_capacity_gb': 536.140,
             'compression_support': 'False',
             'deduplication_support': 'False',
-            'thinprovisioning_support': 'False',
+            'thin_provisioning_support': False,
+            'thick_provisioning_support': True,
             'consistencygroup_support': 'False',
             'pool_name': 'unit_test_pool1',
+            'max_over_subscription_ratio': 20.0,
             'fast_cache_enabled': 'False',
             'fast_support': 'False'}
         self.assertEqual(expected_pool_stats1, pool_stats1)
@@ -3278,11 +3356,14 @@ class EMCVNXCLIDArrayBasedDriverTestCase(DriverTestCaseBase):
             'reserved_percentage': 2,
             'location_info': 'unit test pool 2|fakeSerial',
             'total_capacity_gb': 4099.992,
+            'provisioned_capacity_gb': 636.240,
             'compression_support': 'False',
             'deduplication_support': 'False',
-            'thinprovisioning_support': 'False',
+            'thin_provisioning_support': False,
+            'thick_provisioning_support': True,
             'consistencygroup_support': 'False',
             'pool_name': 'unit test pool 2',
+            'max_over_subscription_ratio': 20.0,
             'fast_cache_enabled': 'False',
             'fast_support': 'False'}
         self.assertEqual(expected_pool_stats2, pool_stats2)
@@ -3670,9 +3751,12 @@ class EMCVNXCLIDriverFCTestCase(DriverTestCaseBase):
             'reserved_percentage': 3,
             'location_info': 'unit_test_pool|fakeSerial',
             'total_capacity_gb': 3281.146,
+            'provisioned_capacity_gb': 636.24,
             'compression_support': 'True',
             'deduplication_support': 'True',
-            'thinprovisioning_support': 'True',
+            'thin_provisioning_support': True,
+            'thick_provisioning_support': True,
+            'max_over_subscription_ratio': 20.0,
             'consistencygroup_support': 'True',
             'pool_name': 'unit_test_pool',
             'fast_cache_enabled': 'True',

diff --git a/cinder/volume/drivers/emc/emc_cli_fc.py b/cinder/volume/drivers/emc/emc_cli_fc.py
index 20b983cc38381eb1f36264136c7742b818314eda..5908b6abbae05cdd40f5e98233d27775f1cf0e9a 100644 (file)
--- a/cinder/volume/drivers/emc/emc_cli_fc.py
+++ b/cinder/volume/drivers/emc/emc_cli_fc.py
@@ -54,6 +54,7 @@ class EMCCLIFCDriver(driver.FibreChannelDriver):
         5.1.0 - iSCSI multipath enhancement
         5.2.0 - Pool-aware scheduler support
         5.3.0 - Consistency group modification support
+        6.0.0 - Over subscription support
     """
 
     def __init__(self, *args, **kwargs):

diff --git a/cinder/volume/drivers/emc/emc_cli_iscsi.py b/cinder/volume/drivers/emc/emc_cli_iscsi.py
index 7055bda3653739da04c3d58bc5d8a138c5a7dbf1..9b2d862a1806eee92f12cc4481f100260f78a988 100644 (file)
--- a/cinder/volume/drivers/emc/emc_cli_iscsi.py
+++ b/cinder/volume/drivers/emc/emc_cli_iscsi.py
@@ -52,6 +52,7 @@ class EMCCLIISCSIDriver(driver.ISCSIDriver):
         5.1.0 - iSCSI multipath enhancement
         5.2.0 - Pool-aware scheduler support
         5.3.0 - Consistency group modification support
+        6.0.0 - Over subscription support
     """
 
     def __init__(self, *args, **kwargs):

diff --git a/cinder/volume/drivers/emc/emc_vnx_cli.py b/cinder/volume/drivers/emc/emc_vnx_cli.py
index e450d5c0c26629f5bd322fc217d124edb77ea177..2048f8e987f5136ccc2f6229d48f46ff17e1e212 100644 (file)
--- a/cinder/volume/drivers/emc/emc_vnx_cli.py
+++ b/cinder/volume/drivers/emc/emc_vnx_cli.py
@@ -240,6 +240,11 @@ class CommandLineHelper(object):
         '-name',
         'Pool Name:\s*(.*)\s*',
         'pool_name')
+    POOL_SUBSCRIBED_CAPACITY = PropertyDescriptor(
+        '-subscribedCap',
+        'Total Subscribed Capacity *\(GBs\) *:\s*(.*)\s*',
+        'provisioned_capacity_gb',
+        float)
 
     POOL_ALL = [POOL_TOTAL_CAPACITY, POOL_FREE_CAPACITY, POOL_STATE]
 
@@ -335,7 +340,9 @@ class CommandLineHelper(object):
 
         # extra spec constants
         self.tiering_spec = 'storagetype:tiering'
-        self.provisioning_spec = 'storagetype:provisioning'
+        self.provisioning_specs = [
+            'provisioning:type',
+            'storagetype:provisioning']
         self.provisioning_values = {
             'thin': ['-type', 'Thin'],
             'thick': ['-type', 'NonThin'],
@@ -1606,7 +1613,7 @@ class CommandLineHelper(object):
 class EMCVnxCliBase(object):
     """This class defines the functions to use the native CLI functionality."""
 
-    VERSION = '05.03.07'
+    VERSION = '06.00.00'
     stats = {'driver_version': VERSION,
              'storage_protocol': None,
              'vendor_name': 'EMC',
@@ -1614,7 +1621,8 @@ class EMCVnxCliBase(object):
              'compression_support': 'False',
              'fast_support': 'False',
              'deduplication_support': 'False',
-             'thinprovisioning_support': 'False'}
+             'thin_provisioning_support': False,
+             'thick_provisioning_support': True}
     enablers = []
 
     def __init__(self, prtcl, configuration=None):
@@ -1653,6 +1661,8 @@ class EMCVnxCliBase(object):
             self.configuration.force_delete_lun_in_storagegroup)
         if self.force_delete_lun_in_sg:
             LOG.warning(_LW("force_delete_lun_in_storagegroup=True"))
+        self.max_over_subscription_ratio = (
+            self.configuration.max_over_subscription_ratio)
 
     def get_target_storagepool(self, volume, source_volume=None):
         raise NotImplementedError
@@ -1763,8 +1773,23 @@ class EMCVnxCliBase(object):
         provisioning = 'thick'
         tiering = None
 
-        if self._client.provisioning_spec in extra_specs:
-            provisioning = extra_specs[self._client.provisioning_spec].lower()
+        if self._client.provisioning_specs[0] in extra_specs:
+            provisioning = (
+                extra_specs[self._client.provisioning_specs[0]].lower())
+            if self._client.provisioning_specs[1] in extra_specs:
+                LOG.warning(_LW("Both 'storagetype:prvosioning' and "
+                                "'provisioning:type' are set in the "
+                                "extra specs, the value of "
+                                "'provisioning:type' will be used. The "
+                                "key 'storagetype:provisioning' may be "
+                                "deprecated in the next release."))
+        elif self._client.provisioning_specs[1] in extra_specs:
+            provisioning = (
+                extra_specs[self._client.provisioning_specs[1]].lower())
+            LOG.warning(_LW("Extra spec key 'storagetype:provisioning' may "
+                            "be deprecated in the next release. It is "
+                            "recommended to use extra spec key "
+                            "'provisioning:type' instead."))
         if self._client.tiering_spec in extra_specs:
             tiering = extra_specs[self._client.tiering_spec].lower()
 
@@ -2005,6 +2030,8 @@ class EMCVnxCliBase(object):
         pool_stats = {}
         pool_stats['pool_name'] = pool['pool_name']
         pool_stats['total_capacity_gb'] = pool['total_capacity_gb']
+        pool_stats['provisioned_capacity_gb'] = (
+            pool['provisioned_capacity_gb'])
         pool_stats['reserved_percentage'] = 0
 
         # Handle pool state Initializing, Ready, Faulted, Offline or Deleting.
@@ -2051,10 +2078,16 @@ class EMCVnxCliBase(object):
         pool_stats['fast_support'] = self.stats['fast_support']
         pool_stats['deduplication_support'] = (
             self.stats['deduplication_support'])
-        pool_stats['thinprovisioning_support'] = (
-            self.stats['thinprovisioning_support'])
+        # Thin provisioning is supported on VNX pools only when
+        # ThinProvisioning Enabler software is installed on VNX,
+        # and thick provisioning is always supported on VNX pools.
+        pool_stats['thin_provisioning_support'] = (
+            self.stats['thin_provisioning_support'])
+        pool_stats['thick_provisioning_support'] = True
         pool_stats['consistencygroup_support'] = (
             self.stats['consistencygroup_support'])
+        pool_stats['max_over_subscription_ratio'] = (
+            self.max_over_subscription_ratio)
 
         return pool_stats
 
@@ -2073,8 +2106,8 @@ class EMCVnxCliBase(object):
         self.stats['deduplication_support'] = (
             'True' if '-Deduplication' in self.enablers else 'False')
 
-        self.stats['thinprovisioning_support'] = (
-            'True' if '-ThinProvisioning' in self.enablers else 'False')
+        self.stats['thin_provisioning_support'] = (
+            True if '-ThinProvisioning' in self.enablers else False)
 
         self.stats['consistencygroup_support'] = (
             'True' if '-VNXSnapshots' in self.enablers else 'False')
@@ -2946,11 +2979,13 @@ class EMCVnxCliPool(EMCVnxCliBase):
             properties = [self._client.POOL_FREE_CAPACITY,
                           self._client.POOL_TOTAL_CAPACITY,
                           self._client.POOL_FAST_CACHE,
-                          self._client.POOL_STATE]
+                          self._client.POOL_STATE,
+                          self._client.POOL_SUBSCRIBED_CAPACITY]
         else:
             properties = [self._client.POOL_FREE_CAPACITY,
                           self._client.POOL_TOTAL_CAPACITY,
-                          self._client.POOL_STATE]
+                          self._client.POOL_STATE,
+                          self._client.POOL_SUBSCRIBED_CAPACITY]
 
         pool = self._client.get_pool(self.storage_pool,
                                      properties=properties,
@@ -3007,11 +3042,13 @@ class EMCVnxCliArray(EMCVnxCliBase):
             properties = [self._client.POOL_FREE_CAPACITY,
                           self._client.POOL_TOTAL_CAPACITY,
                           self._client.POOL_FAST_CACHE,
-                          self._client.POOL_STATE]
+                          self._client.POOL_STATE,
+                          self._client.POOL_SUBSCRIBED_CAPACITY]
         else:
             properties = [self._client.POOL_FREE_CAPACITY,
                           self._client.POOL_TOTAL_CAPACITY,
-                          self._client.POOL_STATE]
+                          self._client.POOL_STATE,
+                          self._client.POOL_SUBSCRIBED_CAPACITY]
         pool_list = self._client.get_pool_list(properties, False)
 
         self.stats['pools'] = map(lambda pool: self._build_pool_stats(pool),