]> review.fuel-infra Code Review - openstack-build/cinder-build.git/commitdiff
NetApp DOT block driver over-subscription support
authorTom Barron <tpb@dyncloud.net>
Mon, 17 Aug 2015 10:36:24 +0000 (06:36 -0400)
committerTom Barron <tpb@dyncloud.net>
Mon, 31 Aug 2015 14:58:45 +0000 (14:58 +0000)
Add support to the NetApp Data ONTAP block drivers for over-
subscription/overprovisioning.  This allows the Cinder scheduler
to more accurately represent the remaining space on a pool when
thin-provisioned Cinder Volumes are in use on a Data ONTAP
backend.

DocImpact
Co-Authored-By: Clinton Knight <cknight@netapp.com>
Partially implements: blueprint netapp-block--over-subscription-thin-provisioning

Change-Id: Ib86512b976e6ba0364b80c536c711c779ca11136

cinder/tests/unit/volume/drivers/netapp/dataontap/client/fakes.py
cinder/tests/unit/volume/drivers/netapp/dataontap/test_block_7mode.py
cinder/tests/unit/volume/drivers/netapp/dataontap/test_block_base.py
cinder/tests/unit/volume/drivers/netapp/dataontap/test_block_cmode.py
cinder/volume/drivers/netapp/dataontap/block_7mode.py
cinder/volume/drivers/netapp/dataontap/block_base.py
cinder/volume/drivers/netapp/dataontap/block_cmode.py
cinder/volume/drivers/netapp/options.py

index 82e94ab5676388614a7f7e09d3a043106b6fe1cb..04428ccc0a10dcc977a21972a3a806389df09e36 100644 (file)
@@ -29,3 +29,74 @@ GET_OPERATIONAL_NETWORK_INTERFACE_ADDRESSES_RESPONSE = etree.XML("""
         </attributes-list>
     </results>
 """ % {"address1": "1.2.3.4", "address2": "99.98.97.96"})
+
+VOLUME_LIST_INFO_RESPONSE = etree.XML("""
+  <results status="passed">
+    <volumes>
+      <volume-info>
+        <name>vol0</name>
+        <block-type>64_bit</block-type>
+        <state>online</state>
+        <size-total>1441193750528</size-total>
+        <size-used>3161096192</size-used>
+        <size-available>1438032654336</size-available>
+        <percentage-used>0</percentage-used>
+        <owning-vfiler>vfiler0</owning-vfiler>
+        <containing-aggregate>aggr0</containing-aggregate>
+        <space-reserve>volume</space-reserve>
+        <space-reserve-enabled>true</space-reserve-enabled>
+        <is-inconsistent>false</is-inconsistent>
+        <is-unrecoverable>false</is-unrecoverable>
+        <is-invalid>false</is-invalid>
+      </volume-info>
+      <volume-info>
+        <name>vol1</name>
+        <block-type>64_bit</block-type>
+        <state>online</state>
+        <size-total>1441193750528</size-total>
+        <size-used>3161096192</size-used>
+        <size-available>1438032654336</size-available>
+        <percentage-used>0</percentage-used>
+        <owning-vfiler>vfiler0</owning-vfiler>
+        <containing-aggregate>aggr0</containing-aggregate>
+        <space-reserve>volume</space-reserve>
+        <space-reserve-enabled>true</space-reserve-enabled>
+        <is-inconsistent>false</is-inconsistent>
+        <is-unrecoverable>false</is-unrecoverable>
+        <is-invalid>false</is-invalid>
+      </volume-info>
+      <volume-info>
+        <name>vol2</name>
+        <block-type>64_bit</block-type>
+        <state>offline</state>
+        <size-total>1441193750528</size-total>
+        <size-used>3161096192</size-used>
+        <size-available>1438032654336</size-available>
+        <percentage-used>0</percentage-used>
+        <owning-vfiler>vfiler0</owning-vfiler>
+        <containing-aggregate>aggr0</containing-aggregate>
+        <space-reserve>volume</space-reserve>
+        <space-reserve-enabled>true</space-reserve-enabled>
+        <is-inconsistent>false</is-inconsistent>
+        <is-unrecoverable>false</is-unrecoverable>
+        <is-invalid>false</is-invalid>
+      </volume-info>
+      <volume-info>
+        <name>vol3</name>
+        <block-type>64_bit</block-type>
+        <state>online</state>
+        <size-total>1441193750528</size-total>
+        <size-used>3161096192</size-used>
+        <size-available>1438032654336</size-available>
+        <percentage-used>0</percentage-used>
+        <owning-vfiler>vfiler0</owning-vfiler>
+        <containing-aggregate>aggr0</containing-aggregate>
+        <space-reserve>volume</space-reserve>
+        <space-reserve-enabled>true</space-reserve-enabled>
+        <is-inconsistent>false</is-inconsistent>
+        <is-unrecoverable>false</is-unrecoverable>
+        <is-invalid>false</is-invalid>
+      </volume-info>
+    </volumes>
+  </results>
+""")
index e14cdae3069b3a7db551e3e899959da19ef953ca..c181ed60b191bf40204896b2bf47f435c82aba4c 100644 (file)
@@ -18,6 +18,7 @@ Mock unit tests for the NetApp block storage 7-mode library
 """
 
 
+import ddt
 from lxml import etree
 import mock
 
@@ -25,6 +26,8 @@ from cinder import exception
 from cinder import test
 from cinder.tests.unit.volume.drivers.netapp.dataontap.client import (
     fake_api as netapp_api)
+import cinder.tests.unit.volume.drivers.netapp.dataontap.client.fakes \
+    as client_fakes
 import cinder.tests.unit.volume.drivers.netapp.dataontap.fakes as fake
 import cinder.tests.unit.volume.drivers.netapp.fakes as na_fakes
 from cinder.volume.drivers.netapp.dataontap import block_7mode
@@ -33,6 +36,7 @@ from cinder.volume.drivers.netapp.dataontap.client import client_base
 from cinder.volume.drivers.netapp import utils as na_utils
 
 
+@ddt.ddt
 class NetAppBlockStorage7modeLibraryTestCase(test.TestCase):
     """Test case for NetApp's 7-Mode iSCSI library."""
 
@@ -454,3 +458,44 @@ class NetAppBlockStorage7modeLibraryTestCase(test.TestCase):
         self.assertEqual(1, self.library._add_lun_to_table.call_count)
         self.zapi_client.move_lun.assert_called_once_with(
             '/vol/vol1/name', '/vol/vol1/volume')
+
+    def test_get_pool_stats_no_volumes(self):
+
+        self.library.vols = []
+
+        result = self.library._get_pool_stats()
+
+        self.assertListEqual([], result)
+
+    @ddt.data({'netapp_lun_space_reservation': 'enabled'},
+              {'netapp_lun_space_reservation': 'disabled'})
+    @ddt.unpack
+    def test_get_pool_stats(self, netapp_lun_space_reservation):
+
+        self.library.volume_list = ['vol0', 'vol1', 'vol2']
+        self.library.root_volume_name = 'vol0'
+        self.library.reserved_percentage = 5
+        self.library.max_over_subscription_ratio = 10.0
+        self.library.configuration.netapp_lun_space_reservation = (
+            netapp_lun_space_reservation)
+        self.library.vols = netapp_api.NaElement(
+            client_fakes.VOLUME_LIST_INFO_RESPONSE).get_child_by_name(
+            'volumes').get_children()
+
+        thick = netapp_lun_space_reservation == 'enabled'
+
+        result = self.library._get_pool_stats()
+
+        expected = [{
+            'pool_name': 'vol1',
+            'QoS_support': False,
+            'thin_provisioned_support': not thick,
+            'thick_provisioned_support': thick,
+            'provisioned_capacity_gb': 2.94,
+            'free_capacity_gb': 1339.27,
+            'total_capacity_gb': 1342.21,
+            'reserved_percentage': 5,
+            'max_over_subscription_ratio': 10.0
+        }]
+
+        self.assertEqual(expected, result)
index 08b844bd5953c45f5b0afd74a0dc8d09cbf4f0be..1008b5f66e627b48d1fbc598f525165a7ffa82ea 100644 (file)
@@ -31,6 +31,7 @@ from cinder import test
 from cinder.tests.unit.volume.drivers.netapp.dataontap.client import (
     fake_api as netapp_api)
 from cinder.tests.unit.volume.drivers.netapp.dataontap import fakes as fake
+import cinder.tests.unit.volume.drivers.netapp.fakes as na_fakes
 from cinder.volume.drivers.netapp.dataontap import block_base
 from cinder.volume.drivers.netapp import utils as na_utils
 from cinder.volume import utils as volume_utils
@@ -44,7 +45,7 @@ class NetAppBlockStorageLibraryTestCase(test.TestCase):
         # Inject fake netapp_lib module classes.
         netapp_api.mock_netapp_lib([block_base])
 
-        kwargs = {'configuration': mock.Mock()}
+        kwargs = {'configuration': self.get_config_base()}
         self.library = block_base.NetAppBlockStorageLibrary(
             'driver', 'protocol', **kwargs)
         self.library.zapi_client = mock.Mock()
@@ -54,6 +55,36 @@ class NetAppBlockStorageLibraryTestCase(test.TestCase):
     def tearDown(self):
         super(NetAppBlockStorageLibraryTestCase, self).tearDown()
 
+    def get_config_base(self):
+        return na_fakes.create_configuration()
+
+    def test_get_reserved_percentage_default_multipler(self):
+
+        default = 1.2
+        reserved_percentage = 20.0
+        self.library.configuration.netapp_size_multiplier = default
+        self.library.configuration.reserved_percentage = reserved_percentage
+        self.mock_object(block_base, 'LOG')
+
+        result = self.library._get_reserved_percentage()
+
+        self.assertEqual(reserved_percentage, result)
+        self.assertFalse(block_base.LOG.warn.called)
+
+    def test_get_reserved_percentage(self):
+
+        multiplier = 2.0
+        self.library.configuration.netapp_size_multiplier = multiplier
+        self.mock_object(block_base, 'LOG')
+
+        result = self.library._get_reserved_percentage()
+
+        reserved_ratio = round(1 - (1 / multiplier), 2)
+        reserved_percentage = 100 * int(reserved_ratio)
+
+        self.assertEqual(reserved_percentage, result)
+        self.assertTrue(block_base.LOG.warn.called)
+
     @mock.patch.object(block_base.NetAppBlockStorageLibrary,
                        '_get_lun_attr',
                        mock.Mock(return_value={'Volume': 'vol1'}))
index 7047643cdc5fd30eb374505e15a1730c1d4242d3..bf8805a19a3ab9adf0a8ce7343fbdcf0e4cc5a8a 100644 (file)
@@ -17,6 +17,7 @@
 Mock unit tests for the NetApp block storage C-mode library
 """
 
+import ddt
 import mock
 from oslo_service import loopingcall
 
@@ -33,6 +34,7 @@ from cinder.volume.drivers.netapp.dataontap import ssc_cmode
 from cinder.volume.drivers.netapp import utils as na_utils
 
 
+@ddt.ddt
 class NetAppBlockStorageCmodeLibraryTestCase(test.TestCase):
     """Test case for NetApp's C-Mode iSCSI library."""
 
@@ -258,6 +260,91 @@ class NetAppBlockStorageCmodeLibraryTestCase(test.TestCase):
 
         self.assertEqual(target_details_list[2], result)
 
+    def test_get_pool_stats_no_volumes(self):
+
+        self.library.ssc_vols = []
+
+        result = self.library._get_pool_stats()
+
+        self.assertListEqual([], result)
+
+    @ddt.data({'thin': True, 'netapp_lun_space_reservation': 'enabled'},
+              {'thin': True, 'netapp_lun_space_reservation': 'disabled'},
+              {'thin': False, 'netapp_lun_space_reservation': 'enabled'},
+              {'thin': False, 'netapp_lun_space_reservation': 'disabled'})
+    @ddt.unpack
+    def test_get_pool_stats(self, thin, netapp_lun_space_reservation):
+
+        class test_volume(object):
+            self.id = None
+            self.aggr = None
+
+        test_volume = test_volume()
+        test_volume.id = {'vserver': 'openstack', 'name': 'vola'}
+        test_volume.aggr = {
+            'disk_type': 'SSD',
+            'ha_policy': 'cfo',
+            'junction': '/vola',
+            'name': 'aggr1',
+            'raid_type': 'raiddp'
+        }
+        test_volume.space = {
+            'size_total_bytes': '10737418240',
+            'space-guarantee': 'file',
+            'size_avl_bytes': '2147483648',
+            'space-guarantee-enabled': False,
+            'thin_provisioned': False
+        }
+        test_volume.sis = {'dedup': False, 'compression': False}
+        test_volume.state = {
+            'status': 'online',
+            'vserver_root': False,
+            'junction_active': True
+        }
+        test_volume.qos = {'qos_policy_group': None}
+
+        ssc_map = {
+            'mirrored': {},
+            'dedup': {},
+            'compression': {},
+            'thin': {test_volume if thin else None},
+            'all': [test_volume]
+        }
+        self.library.ssc_vols = ssc_map
+        self.library.reserved_percentage = 5
+        self.library.max_over_subscription_ratio = 10
+        self.library.configuration.netapp_lun_space_reservation = (
+            netapp_lun_space_reservation)
+
+        netapp_thin = 'true' if thin else 'false'
+        netapp_thick = 'false' if thin else 'true'
+
+        thick = not thin and (netapp_lun_space_reservation == 'enabled')
+
+        result = self.library._get_pool_stats()
+
+        expected = [{'pool_name': 'vola',
+                     'netapp_unmirrored': 'true',
+                     'QoS_support': True,
+                     'thin_provisioned_support': not thick,
+                     'thick_provisioned_support': thick,
+                     'provisioned_capacity_gb': 8.0,
+                     'netapp_thick_provisioned': netapp_thick,
+                     'netapp_nocompression': 'true',
+                     'free_capacity_gb': 2.0,
+                     'netapp_thin_provisioned': netapp_thin,
+                     'total_capacity_gb': 10.0,
+                     'netapp_compression': 'false',
+                     'netapp_mirrored': 'false',
+                     'netapp_dedup': 'false',
+                     'reserved_percentage': 5,
+                     'max_over_subscription_ratio': 10.0,
+                     'netapp_raid_type': 'raiddp',
+                     'netapp_disk_type': 'SSD',
+                     'netapp_nodedup': 'true'}]
+
+        self.assertEqual(expected, result)
+
     def test_delete_volume(self):
         self.mock_object(block_base.NetAppLun, 'get_metadata_property',
                          mock.Mock(return_value=fake.POOL_NAME))
index 03ac32363d2b8a49660da4652b443b7ae6ef0796..d3d6dc0f9c6a1c7571be623146a549848ca07850 100644 (file)
@@ -280,19 +280,28 @@ class NetAppBlockStorage7modeLibrary(block_base.NetAppBlockStorageLibrary):
             pool = dict()
             pool['pool_name'] = volume_name
             pool['QoS_support'] = False
-            pool['reserved_percentage'] = 0
+            pool['reserved_percentage'] = (
+                self.reserved_percentage)
+            pool['max_over_subscription_ratio'] = (
+                self.max_over_subscription_ratio)
 
-            # convert sizes to GB and de-rate by NetApp multiplier
+            # convert sizes to GB
             total = float(vol.get_child_content('size-total') or 0)
-            total /= self.configuration.netapp_size_multiplier
             total /= units.Gi
             pool['total_capacity_gb'] = na_utils.round_down(total, '0.01')
 
             free = float(vol.get_child_content('size-available') or 0)
-            free /= self.configuration.netapp_size_multiplier
             free /= units.Gi
             pool['free_capacity_gb'] = na_utils.round_down(free, '0.01')
 
+            pool['provisioned_capacity_gb'] = (round(
+                pool['total_capacity_gb'] - pool['free_capacity_gb'], 2))
+
+            thick = (
+                self.configuration.netapp_lun_space_reservation == 'enabled')
+            pool['thick_provisioned_support'] = thick
+            pool['thin_provisioned_support'] = not thick
+
             pools.append(pool)
 
         return pools
index 774c9cdf44825f2c02715a406bc1b14f5cf33483..d04cdd5f572466c3a721c228160b854ae688053e 100644 (file)
@@ -27,6 +27,7 @@ import sys
 import uuid
 
 from oslo_log import log as logging
+from oslo_log import versionutils
 from oslo_utils import excutils
 from oslo_utils import importutils
 from oslo_utils import units
@@ -109,6 +110,31 @@ class NetAppBlockStorageLibrary(object):
         self.configuration.append_config_values(
             na_opts.netapp_provisioning_opts)
         self.configuration.append_config_values(na_opts.netapp_san_opts)
+        self.max_over_subscription_ratio = (
+            self.configuration.max_over_subscription_ratio)
+        self.reserved_percentage = self._get_reserved_percentage()
+
+    def _get_reserved_percentage(self):
+        # If the legacy config option if it is set to the default
+        # value, use the more general configuration option.
+        if self.configuration.netapp_size_multiplier == (
+                na_opts.NETAPP_SIZE_MULTIPLIER_DEFAULT):
+            return self.configuration.reserved_percentage
+
+        # If the legacy config option has a non-default value,
+        # honor it for one release.  Note that the "size multiplier"
+        # actually acted as a divisor in the code and didn't apply
+        # to the file size (as the help message for this option suggest),
+        # but rather to total and free size for the pool.
+        divisor = self.configuration.netapp_size_multiplier
+        reserved_ratio = round(1 - (1 / divisor), 2)
+        reserved_percentage = 100 * int(reserved_ratio)
+        msg = _LW('The "netapp_size_multiplier" configuration option is '
+                  'deprecated and will be removed in the Mitaka release. '
+                  'Please set "reserved_percentage = %d" instead.') % (
+                      reserved_percentage)
+        versionutils.report_deprecated_feature(LOG, msg)
+        return reserved_percentage
 
     def do_setup(self, context):
         na_utils.check_flags(self.REQUIRED_FLAGS, self.configuration)
index 3e8b0868dcaaa94a5849a157ea30925d4b430729..7531acaad7dee5c1b36d05569876939a108d5e08 100644 (file)
@@ -198,19 +198,23 @@ class NetAppBlockStorageCmodeLibrary(block_base.NetAppBlockStorageLibrary):
             pool = dict()
             pool['pool_name'] = vol.id['name']
             pool['QoS_support'] = True
-            pool['reserved_percentage'] = 0
+            pool['reserved_percentage'] = (
+                self.reserved_percentage)
+            pool['max_over_subscription_ratio'] = (
+                self.max_over_subscription_ratio)
 
-            # convert sizes to GB and de-rate by NetApp multiplier
+            # convert sizes to GB
             total = float(vol.space['size_total_bytes'])
-            total /= self.configuration.netapp_size_multiplier
             total /= units.Gi
             pool['total_capacity_gb'] = na_utils.round_down(total, '0.01')
 
             free = float(vol.space['size_avl_bytes'])
-            free /= self.configuration.netapp_size_multiplier
             free /= units.Gi
             pool['free_capacity_gb'] = na_utils.round_down(free, '0.01')
 
+            pool['provisioned_capacity_gb'] = (round(
+                pool['total_capacity_gb'] - pool['free_capacity_gb'], 2))
+
             pool['netapp_raid_type'] = vol.aggr['raid_type']
             pool['netapp_disk_type'] = vol.aggr['disk_type']
 
@@ -230,6 +234,11 @@ class NetAppBlockStorageCmodeLibrary(block_base.NetAppBlockStorageLibrary):
             thin = vol in self.ssc_vols['thin']
             pool['netapp_thin_provisioned'] = six.text_type(thin).lower()
             pool['netapp_thick_provisioned'] = six.text_type(not thin).lower()
+            thick = (not thin and
+                     self.configuration.netapp_lun_space_reservation
+                     == 'enabled')
+            pool['thick_provisioned_support'] = thick
+            pool['thin_provisioned_support'] = not thick
 
             pools.append(pool)
 
index f58a6475b7e4c20269c4cc5ff6b4759f051679ec..e21a3e65199d16b3a650d6627432df4d743d6396 100644 (file)
@@ -26,6 +26,8 @@ place to ensure re usability and better management of configuration options.
 
 from oslo_config import cfg
 
+NETAPP_SIZE_MULTIPLIER_DEFAULT = 1.2
+
 netapp_proxy_opts = [
     cfg.StrOpt('netapp_storage_family',
                default='ontap_cluster',
@@ -71,11 +73,13 @@ netapp_basicauth_opts = [
 
 netapp_provisioning_opts = [
     cfg.FloatOpt('netapp_size_multiplier',
-                 default=1.2,
+                 default=NETAPP_SIZE_MULTIPLIER_DEFAULT,
                  help=('The quantity to be multiplied by the requested '
                        'volume size to ensure enough space is available on '
                        'the virtual storage server (Vserver) to fulfill '
-                       'the volume creation request.')),
+                       'the volume creation request.  Note: this option '
+                       'is deprecated and will be removed in favor of '
+                       '"reserved_percentage" in the Mitaka release.')),
     cfg.StrOpt('netapp_volume_list',
                default=None,
                help=('This option is only utilized when the storage protocol '