]> review.fuel-infra Code Review - openstack-build/cinder-build.git/commitdiff
Implement clone_image caching on SolidFire
authorJohn Griffith <john.griffith8@gmail.com>
Thu, 18 Dec 2014 17:55:05 +0000 (10:55 -0700)
committerJohn Griffith <john.griffith@solidfire.com>
Mon, 12 Jan 2015 23:09:07 +0000 (16:09 -0700)
When performing boot from Volume on none FS based devices or devices
that aren't also acting as Glance Image stores, the current process goes
like this:
1. Fetch the image from Glance Store to a temp file on the Cinder Node
2. Perform a qemu-img convert on that file (convert to raw)
3. Transfer the contents of that file to the newly created Volume using dd

This process can take a while and is pretty inefficient. This BP introduces
the option for Operators to enable image-caching on the SolidFire backend device.

When enabled this option will use the existing Cinder clone_image method,
and will create a "Template Volume" of minimum required size on the SolidFire
backend. This Volume will be created with a backend SolidFire account that can
be shared among OpenStack Tenants.

When a request is received we first do a number of checks before proceeding:
1. Is Image/Template caching enabled
2. Inspect the image-metadata and verify that the image is public,
   or that the tenant making the request is the owner
3. Verify that the image-metadata includes the properties['virtual_size']
   parameter

If the preconditions are met, create a minimal sized Volume with the requested
image; then use SolidFires clone operation to create the requested bootable
Volume. During the process we also check and make sure that the image hasn't
been updated since we created our cached copy, if it has we then delete the
copy we have on the system and create a new one.

Also updated those drivers with docstrings duplicating the params info.  I did
not rewrite or modify those that didn't include changed info.  I'm not sure we
should duplicate docstring info for base methods like this anyway, but that's
irrelevant for this work.

Change-Id: I2fac9410f504dff1aaa9fe0b9a274db7b074fa50
Implements: blueprint implement-solidfire-cloneimage

13 files changed:
cinder/tests/test_gpfs.py
cinder/tests/test_netapp_nfs.py
cinder/tests/test_rbd.py
cinder/tests/test_solidfire.py
cinder/tests/test_volume.py
cinder/volume/driver.py
cinder/volume/drivers/ibm/gpfs.py
cinder/volume/drivers/lvm.py
cinder/volume/drivers/netapp/dataontap/nfs_base.py
cinder/volume/drivers/rbd.py
cinder/volume/drivers/scality.py
cinder/volume/drivers/solidfire.py
cinder/volume/flows/manager/create_volume.py

index a994fff8ec962cec9d0c0b10f482ebad96631fbb..0cca0a78944c0e974ef1d6be6b214f6344c4a07e 100644 (file)
@@ -1055,7 +1055,7 @@ class GPFSDriverTestCase(test.TestCase):
 
     @patch('cinder.volume.drivers.ibm.gpfs.GPFSDriver._clone_image')
     def test_clone_image_pub(self, mock_exec):
-        self.driver.clone_image('', '', {'id': 1})
+        self.driver.clone_image('', '', '', {'id': 1}, '')
 
     @patch('cinder.volume.drivers.ibm.gpfs.GPFSDriver._is_gpfs_path')
     def test_is_cloneable_ok(self, mock_is_gpfs_path):
index bdfd3abc5c8d41492344fbe0b28eb4b761516e81..75f0cf0f4f77cd456ff92155c7914620cf7afefe 100644 (file)
@@ -478,7 +478,10 @@ class NetAppCmodeNfsDriverTestCase(test.TestCase):
         drv._post_clone_image(volume)
 
         mox.ReplayAll()
-        drv.clone_image(volume, ('image_location', None), {'id': 'image_id'})
+        drv.clone_image('',
+                        volume,
+                        ('image_location', None),
+                        {'id': 'image_id'}, '')
         mox.VerifyAll()
 
     def get_img_info(self, format):
@@ -501,10 +504,12 @@ class NetAppCmodeNfsDriverTestCase(test.TestCase):
         drv._is_share_vol_compatible(IgnoreArg(), IgnoreArg()).AndReturn(False)
 
         mox.ReplayAll()
-        (prop, cloned) = drv. clone_image(
+        (prop, cloned) = drv.clone_image(
+            '',
             volume,
             ('nfs://127.0.0.1:/share/img-id', None),
-            {'id': 'image_id'})
+            {'id': 'image_id'},
+            '')
         mox.VerifyAll()
         if not cloned and not prop['provider_location']:
             pass
@@ -539,10 +544,12 @@ class NetAppCmodeNfsDriverTestCase(test.TestCase):
         drv._resize_image_file({'name': 'vol'}, IgnoreArg())
 
         mox.ReplayAll()
-        drv. clone_image(
+        drv.clone_image(
+            '',
             volume,
             ('nfs://127.0.0.1:/share/img-id', None),
-            {'id': 'image_id'})
+            {'id': 'image_id'},
+            '')
         mox.VerifyAll()
 
     def test_clone_image_cloneableshare_notraw(self):
@@ -580,7 +587,11 @@ class NetAppCmodeNfsDriverTestCase(test.TestCase):
 
         mox.ReplayAll()
         drv.clone_image(
-            volume, ('nfs://127.0.0.1/share/img-id', None), {'id': 'image_id'})
+            '',
+            volume,
+            ('nfs://127.0.0.1/share/img-id', None),
+            {'id': 'image_id'},
+            '')
         mox.VerifyAll()
 
     def test_clone_image_file_not_discovered(self):
@@ -619,8 +630,12 @@ class NetAppCmodeNfsDriverTestCase(test.TestCase):
         drv._delete_file('/mnt/vol')
 
         mox.ReplayAll()
-        vol_dict, result = drv. clone_image(
-            volume, ('nfs://127.0.0.1/share/img-id', None), {'id': 'image_id'})
+        vol_dict, result = drv.clone_image(
+            '',
+            volume,
+            ('nfs://127.0.0.1/share/img-id', None),
+            {'id': 'image_id'},
+            '')
         mox.VerifyAll()
         self.assertFalse(result)
         self.assertFalse(vol_dict['bootable'])
@@ -667,8 +682,12 @@ class NetAppCmodeNfsDriverTestCase(test.TestCase):
         drv._delete_file('/mnt/vol')
 
         mox.ReplayAll()
-        vol_dict, result = drv. clone_image(
-            volume, ('nfs://127.0.0.1/share/img-id', None), {'id': 'image_id'})
+        vol_dict, result = drv.clone_image(
+            '',
+            volume,
+            ('nfs://127.0.0.1/share/img-id', None),
+            {'id': 'image_id'},
+            '')
         mox.VerifyAll()
         self.assertFalse(result)
         self.assertFalse(vol_dict['bootable'])
index 7b14e18bb5f6108cdece70b26d37ac80408576ab..2dabe8c4a0379aa2f673e55bbec9e754c259dc07 100644 (file)
@@ -239,7 +239,7 @@ class RBDTestCase(test.TestCase):
                           self.driver.manage_existing,
                           self.volume, existing_ref)
 
-        #make sure the exception was raised
+        # Make sure the exception was raised
         self.assertEqual(RAISED_EXCEPTIONS,
                          [self.mock_rbd.ImageExists])
 
@@ -1041,7 +1041,8 @@ class ManagedRBDTestCase(DriverTestCase):
     def test_create_vol_from_image_status_available(self):
         """Clone raw image then verify volume is in available state."""
 
-        def _mock_clone_image(volume, image_location, image_meta):
+        def _mock_clone_image(context, volume, image_location,
+                              image_meta, image_service):
             return {'provider_location': None}, True
 
         with mock.patch.object(self.volume.driver, 'clone_image') as \
@@ -1060,7 +1061,8 @@ class ManagedRBDTestCase(DriverTestCase):
     def test_create_vol_from_non_raw_image_status_available(self):
         """Clone non-raw image then verify volume is in available state."""
 
-        def _mock_clone_image(volume, image_location, image_meta):
+        def _mock_clone_image(context, volume, image_location,
+                              image_meta, image_service):
             return {'provider_location': None}, False
 
         with mock.patch.object(self.volume.driver, 'clone_image') as \
@@ -1096,11 +1098,15 @@ class ManagedRBDTestCase(DriverTestCase):
 
         with mock.patch.object(driver, '_is_cloneable', lambda *args: False):
             image_loc = (mock.Mock(), mock.Mock())
-            actual = driver.clone_image(mock.Mock(), image_loc, {})
+            actual = driver.clone_image(mock.Mock(),
+                                        mock.Mock(),
+                                        image_loc,
+                                        {},
+                                        mock.Mock())
             self.assertEqual(({}, False), actual)
 
         self.assertEqual(({}, False),
-                         driver.clone_image(object(), None, {}))
+                         driver.clone_image('', object(), None, {}, ''))
 
     def test_clone_success(self):
         expected = ({'provider_location': None}, True)
@@ -1116,9 +1122,12 @@ class ManagedRBDTestCase(DriverTestCase):
                     image_loc = ('rbd://fee/fi/fo/fum', None)
 
                     volume = {'name': 'vol1'}
-                    actual = driver.clone_image(volume, image_loc,
+                    actual = driver.clone_image(mock.Mock(),
+                                                volume,
+                                                image_loc,
                                                 {'disk_format': 'raw',
-                                                 'id': 'id.foo'})
+                                                 'id': 'id.foo'},
+                                                mock.Mock())
 
                     self.assertEqual(expected, actual)
                     mock_clone.assert_called_once_with(volume,
index b63c4d28024ef79ed6cbd48d2be5ebc3ae2f94e0..7d45609cb54366f8f670980d69ca4b37affb4850 100644 (file)
@@ -14,6 +14,8 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+import datetime
+
 import mock
 import mox
 from oslo.utils import timeutils
@@ -49,6 +51,8 @@ class SolidFireVolumeTestCase(test.TestCase):
         self.configuration.sf_account_prefix = 'cinder'
         self.configuration.reserved_percentage = 25
         self.configuration.iscsi_helper = None
+        self.configuration.sf_template_account_name = 'openstack-vtemplate'
+        self.configuration.sf_allow_template_caching = False
 
         super(SolidFireVolumeTestCase, self).setUp()
         self.stubs.Set(SolidFireDriver, '_issue_api_request',
@@ -59,6 +63,27 @@ class SolidFireVolumeTestCase(test.TestCase):
         self.expected_qos_results = {'minIOPS': 1000,
                                      'maxIOPS': 10000,
                                      'burstIOPS': 20000}
+        self.mock_stats_data =\
+            {'result':
+                {'clusterCapacity': {'maxProvisionedSpace': 107374182400,
+                                     'usedSpace': 1073741824,
+                                     'compressionPercent': 100,
+                                     'deDuplicationPercent': 100,
+                                     'thinProvisioningPercent': 100}}}
+        self.mock_volume = {'project_id': 'testprjid',
+                            'name': 'testvol',
+                            'size': 1,
+                            'id': 'a720b3c0-d1f0-11e1-9b23-0800200c9a66',
+                            'volume_type_id': 'fast',
+                            'created_at': timeutils.utcnow()}
+        self.fake_image_meta = {'id': '17c550bb-a411-44c0-9aaf-0d96dd47f501',
+                                'updated_at': datetime.datetime(2013, 9,
+                                                                28, 15,
+                                                                27, 36,
+                                                                325355),
+                                'is_public': True,
+                                'owner': 'testprjid'}
+        self.fake_image_service = 'null'
 
     def fake_build_endpoint_info(obj, **kwargs):
         endpoint = {}
@@ -282,23 +307,6 @@ class SolidFireVolumeTestCase(test.TestCase):
         sfv.create_cloned_volume(testvol_b, testvol)
 
     def test_initialize_connector_with_blocksizes(self):
-        expected_iqn = 'iqn.2010-01.com.solidfire:'\
-                       '87hg.uuid-2cc06226-cc74-4cb7-bd55-14aed659a0cc.4060'
-        expected_properties = \
-            {'driver_volume_type': 'iscsi',
-             'data': {'target_discovered': False,
-                      'encrypted': False,
-                      'logical_block_size': '4096',
-                      'physical_block_size': '4096',
-                      'target_iqn': expected_iqn,
-                      'target_portal': '10.10.7.1:3260',
-                      'volume_id': 'a720b3c0-d1f0-11e1-9b23-0800200c9a66',
-                      'target_lun': 0,
-                      'auth_password': '2FE0CQ8J196R',
-                      'auth_username':
-                          'stack-1-a60e2611875f40199931f2c76370d66b',
-                      'auth_method': 'CHAP'}}
-
         connector = {'initiator': 'iqn.2012-07.org.fake:01'}
         testvol = {'project_id': 'testprjid',
                    'name': 'testvol',
@@ -315,62 +323,9 @@ class SolidFireVolumeTestCase(test.TestCase):
                    }
 
         sfv = SolidFireDriver(configuration=self.configuration)
-        self.assertEqual(sfv.initialize_connection(testvol, connector),
-                         expected_properties)
-
-    @mock.patch('cinder.volume.driver.CONF')
-    def test_iscsi_helpers_not_in_base_iscsi_driver(self, mock_conf):
-        # This test is added to check for bug: 1400804
-        # The base iscsi driver should be clean from specifics
-        # regarding tgtadm or LVM driver, this check is here
-        # to make sure nothing regarding specific iscsi_helpers
-        # sneak back in
-        expected_iqn = 'iqn.2010-01.com.solidfire:'\
-                       '87hg.uuid-2cc06226-cc74-4cb7-bd55-14aed659a0cc.4060'
-        expected_properties = \
-            {'driver_volume_type': 'iscsi',
-             'data': {'target_discovered': False,
-                      'encrypted': False,
-                      'logical_block_size': '4096',
-                      'physical_block_size': '4096',
-                      'target_iqn': expected_iqn,
-                      'target_portal': '10.10.7.1:3260',
-                      'volume_id': 'a720b3c0-d1f0-11e1-9b23-0800200c9a66',
-                      'target_lun': 0,
-                      'auth_password': '2FE0CQ8J196R',
-                      'auth_username':
-                          'stack-1-a60e2611875f40199931f2c76370d66b',
-                      'auth_method': 'CHAP'}}
-
-        connector = {'initiator': 'iqn.2012-07.org.fake:01'}
-        testvol = {'project_id': 'testprjid',
-                   'name': 'testvol',
-                   'size': 1,
-                   'id': 'a720b3c0-d1f0-11e1-9b23-0800200c9a66',
-                   'volume_type_id': None,
-                   'provider_location': '10.10.7.1:3260 iqn.2010-01.com.'
-                                        'solidfire:87hg.uuid-2cc06226-cc'
-                                        '74-4cb7-bd55-14aed659a0cc.4060 0',
-                   'provider_auth': 'CHAP stack-1-a60e2611875f40199931f2'
-                                    'c76370d66b 2FE0CQ8J196R',
-                   'provider_geometry': '4096 4096',
-                   'created_at': timeutils.utcnow(),
-                   }
-
-        mock_conf.iscsi_helper = 'lioadm'
-        sfv = SolidFireDriver(configuration=self.configuration)
-        self.assertEqual(sfv.initialize_connection(testvol, connector),
-                         expected_properties)
-
-        mock_conf.iscsi_helper = 'iseradm'
-        sfv = SolidFireDriver(configuration=self.configuration)
-        self.assertEqual(sfv.initialize_connection(testvol, connector),
-                         expected_properties)
-
-        mock_conf.iscsi_helper = 'tgtadm'
-        sfv = SolidFireDriver(configuration=self.configuration)
-        self.assertEqual(sfv.initialize_connection(testvol, connector),
-                         expected_properties)
+        properties = sfv.initialize_connection(testvol, connector)
+        self.assertEqual('4096', properties['data']['physical_block_size'])
+        self.assertEqual('4096', properties['data']['logical_block_size'])
 
     def test_create_volume_with_qos(self):
         preset_qos = {}
@@ -732,3 +687,155 @@ class SolidFireVolumeTestCase(test.TestCase):
                          sf_vol_object['attributes']['migration_uuid'])
         self.assertEqual('UUID-a720b3c0-d1f0-11e1-9b23-0800200c9a66',
                          sf_vol_object['name'])
+
+    @mock.patch.object(SolidFireDriver, '_issue_api_request')
+    @mock.patch.object(SolidFireDriver, '_get_sfaccount')
+    @mock.patch.object(SolidFireDriver, '_get_sf_volume')
+    @mock.patch.object(SolidFireDriver, '_create_image_volume')
+    def test_verify_image_volume_out_of_date(self,
+                                             _mock_create_image_volume,
+                                             _mock_get_sf_volume,
+                                             _mock_get_sfaccount,
+                                             _mock_issue_api_request):
+        fake_sf_vref = {
+            'status': 'active', 'volumeID': 1,
+            'attributes': {
+                'image_info':
+                    {'image_updated_at': '2014-12-17T00:16:23+00:00',
+                     'image_id': '17c550bb-a411-44c0-9aaf-0d96dd47f501',
+                     'image_name': 'fake-image',
+                     'image_created_at': '2014-12-17T00:16:23+00:00'}}}
+
+        stats_data =\
+            {'result':
+                {'clusterCapacity': {'maxProvisionedSpace': 107374182400,
+                                     'usedSpace': 1073741824,
+                                     'compressionPercent': 100,
+                                     'deDuplicationPercent': 100,
+                                     'thinProvisioningPercent': 100}}}
+
+        _mock_issue_api_request.return_value = stats_data
+        _mock_get_sfaccount.return_value = {'username': 'openstack-vtemplate',
+                                            'accountID': 7777}
+        _mock_get_sf_volume.return_value = fake_sf_vref
+        _mock_create_image_volume.return_value = fake_sf_vref
+
+        image_meta = {'id': '17c550bb-a411-44c0-9aaf-0d96dd47f501',
+                      'updated_at': datetime.datetime(2013, 9, 28,
+                                                      15, 27, 36,
+                                                      325355)}
+        image_service = 'null'
+
+        sfv = SolidFireDriver(configuration=self.configuration)
+        _mock_issue_api_request.return_value = {'result': 'ok'}
+        sfv._verify_image_volume(self.ctxt, image_meta, image_service)
+        self.assertTrue(_mock_create_image_volume.called)
+
+    @mock.patch.object(SolidFireDriver, '_issue_api_request')
+    @mock.patch.object(SolidFireDriver, '_get_sfaccount')
+    @mock.patch.object(SolidFireDriver, '_get_sf_volume')
+    @mock.patch.object(SolidFireDriver, '_create_image_volume')
+    def test_verify_image_volume_ok(self,
+                                    _mock_create_image_volume,
+                                    _mock_get_sf_volume,
+                                    _mock_get_sfaccount,
+                                    _mock_issue_api_request):
+
+        _mock_issue_api_request.return_value = self.mock_stats_data
+        _mock_get_sfaccount.return_value = {'username': 'openstack-vtemplate',
+                                            'accountID': 7777}
+        _mock_get_sf_volume.return_value =\
+            {'status': 'active', 'volumeID': 1,
+             'attributes': {
+                 'image_info':
+                     {'image_updated_at': '2013-09-28T15:27:36.325355',
+                      'image_id': '17c550bb-a411-44c0-9aaf-0d96dd47f501',
+                      'image_name': 'fake-image',
+                      'image_created_at': '2014-12-17T00:16:23+00:00'}}}
+        _mock_create_image_volume.return_value = None
+
+        image_meta = {'id': '17c550bb-a411-44c0-9aaf-0d96dd47f501',
+                      'updated_at': datetime.datetime(2013, 9, 28,
+                                                      15, 27, 36,
+                                                      325355)}
+        image_service = 'null'
+
+        sfv = SolidFireDriver(configuration=self.configuration)
+        _mock_issue_api_request.return_value = {'result': 'ok'}
+
+        sfv._verify_image_volume(self.ctxt, image_meta, image_service)
+        self.assertFalse(_mock_create_image_volume.called)
+
+    @mock.patch.object(SolidFireDriver, '_issue_api_request')
+    def test_clone_image_not_configured(self, _mock_issue_api_request):
+        _mock_issue_api_request.return_value = self.mock_stats_data
+
+        sfv = SolidFireDriver(configuration=self.configuration)
+        self.assertEqual((None, False),
+                         sfv.clone_image(self.ctxt,
+                                         self.mock_volume,
+                                         'fake',
+                                         self.fake_image_meta,
+                                         'fake'))
+
+    @mock.patch.object(SolidFireDriver, '_issue_api_request')
+    def test_clone_image_authorization(self, _mock_issue_api_request):
+        _mock_issue_api_request.return_value = self.mock_stats_data
+        self.configuration.sf_allow_template_caching = True
+        sfv = SolidFireDriver(configuration=self.configuration)
+
+        # Make sure if it's NOT public and we're NOT the owner it
+        # doesn't try and cache
+        _fake_image_meta = {'id': '17c550bb-a411-44c0-9aaf-0d96dd47f501',
+                            'updated_at': datetime.datetime(2013, 9,
+                                                            28, 15,
+                                                            27, 36,
+                                                            325355),
+                            'properties': {'virtual_size': 1},
+                            'is_public': False,
+                            'owner': 'wrong-owner'}
+        self.assertEqual((None, False),
+                         sfv.clone_image(self.ctxt,
+                                         self.mock_volume,
+                                         'fake',
+                                         _fake_image_meta,
+                                         'fake'))
+
+        # And is_public False, but the correct owner does work
+        # expect raise AccountNotFound as that's the next call after
+        # auth checks
+        _fake_image_meta['owner'] = 'testprjid'
+        self.assertRaises(exception.SolidFireAccountNotFound,
+                          sfv.clone_image, self.ctxt,
+                          self.mock_volume, 'fake',
+                          _fake_image_meta, 'fake')
+
+        # And is_public True, even if not the correct owner
+        _fake_image_meta['is_public'] = True
+        _fake_image_meta['owner'] = 'wrong-owner'
+        self.assertRaises(exception.SolidFireAccountNotFound,
+                          sfv.clone_image, self.ctxt,
+                          self.mock_volume, 'fake',
+                          _fake_image_meta, 'fake')
+
+    @mock.patch.object(SolidFireDriver, '_issue_api_request')
+    def test_clone_image_virt_size_not_set(self, _mock_issue_api_request):
+        _mock_issue_api_request.return_value = self.mock_stats_data
+        self.configuration.sf_allow_template_caching = True
+        sfv = SolidFireDriver(configuration=self.configuration)
+
+        # Don't run clone_image if virtual_size property not on image
+        _fake_image_meta = {'id': '17c550bb-a411-44c0-9aaf-0d96dd47f501',
+                            'updated_at': datetime.datetime(2013, 9,
+                                                            28, 15,
+                                                            27, 36,
+                                                            325355),
+                            'is_public': True,
+                            'owner': 'testprjid'}
+
+        self.assertEqual((None, False),
+                         sfv.clone_image(self.ctxt,
+                                         self.mock_volume,
+                                         'fake',
+                                         _fake_image_meta,
+                                         'fake'))
index 54e99cde2226d84320a60c7fec119fd9fe15e05f..70da0876d74d845f4b83ae34c54ede92f3e0eab8 100644 (file)
@@ -106,7 +106,6 @@ class BaseVolumeTestCase(test.TestCase):
             mock_trace_cls.return_value = mock_decorator
             self.volume = importutils.import_object(CONF.volume_manager)
         self.configuration = mock.Mock(conf.Configuration)
-        #self.configuration = conf.Configuration(fake_opts, 'fake_group')
         self.context = context.get_admin_context()
         self.context.user_id = 'fake'
         self.context.project_id = 'fake'
@@ -2178,7 +2177,9 @@ class VolumeTestCase(BaseVolumeTestCase):
                               size=None):
             pass
 
-        def fake_clone_image(volume_ref, image_location, image_meta):
+        def fake_clone_image(ctx, volume_ref,
+                             image_location, image_meta,
+                             image_service):
             return {'provider_location': None}, True
 
         dst_fd, dst_path = tempfile.mkstemp()
index 78f13885db1c847376f7010a25a47cfd48360ec8..97c28a6d92077af846f30c418f3e06e82ff1c82f 100644 (file)
@@ -532,22 +532,24 @@ class VolumeDriver(object):
                                                       {'path': host_device}))
         return {'conn': conn, 'device': device, 'connector': connector}
 
-    def clone_image(self, volume, image_location, image_meta):
+    def clone_image(self, context, volume,
+                    image_location, image_meta,
+                    image_service):
         """Create a volume efficiently from an existing image.
 
         image_location is a string whose format depends on the
         image service backend in use. The driver should use it
         to determine whether cloning is possible.
 
-        image_id is a string which represents id of the image.
-        It can be used by the driver to introspect internal
-        stores or registry to do an efficient image clone.
-
         image_meta is a dictionary that includes 'disk_format' (e.g.
         raw, qcow2) and other image attributes that allow drivers to
         decide whether they can clone the image without first requiring
         conversion.
 
+        image_service is the reference of the image_service to use.
+        Note that this is needed to be passed here for drivers that
+        will want to fetch images from the image service directly.
+
         Returns a dict of volume properties eg. provider_location,
         boolean indicating whether cloning occurred
         """
index c5732eba4834afce27e17f3396ec7151c389cf3a..5f9456f83cf84f7c43bab9605033f8ac2551516f 100644 (file)
@@ -703,7 +703,9 @@ class GPFSDriver(driver.VolumeDriver):
         data['reserved_percentage'] = 0
         self._stats = data
 
-    def clone_image(self, volume, image_location, image_meta):
+    def clone_image(self, context, volume,
+                    image_location, image_meta,
+                    image_service):
         """Create a volume from the specified image."""
         return self._clone_image(volume, image_location, image_meta['id'])
 
index b0409a7c01d14dc95b9cbd1d3b0ee53f97f84e46..8b6aa17976a8b6376ed424a1871f72ce8d6afa8b 100644 (file)
@@ -386,7 +386,9 @@ class LVMVolumeDriver(driver.VolumeDriver):
         finally:
             self.delete_snapshot(temp_snapshot)
 
-    def clone_image(self, volume, image_location, image_meta):
+    def clone_image(self, context, volume,
+                    image_location, image_meta,
+                    image_service):
         return None, False
 
     def backup_volume(self, context, backup, backup_service):
index 8c441f088af53032213c7a79ca5bf9f8d2e19ad7..61bbc9b3c68817dcd6878062bc397ae7f431312b 100644 (file)
@@ -366,17 +366,15 @@ class NetAppNfsDriver(nfs.NfsDriver):
             LOG.warning(_LW('Exception during deleting %s'), ex.__str__())
             return False
 
-    def clone_image(self, volume, image_location, image_meta):
+    def clone_image(self, context, volume,
+                    image_location, image_meta,
+                    image_service):
         """Create a volume efficiently from an existing image.
 
         image_location is a string whose format depends on the
         image service backend in use. The driver should use it
         to determine whether cloning is possible.
 
-        image_id is a string which represents id of the image.
-        It can be used by the driver to introspect internal
-        stores or registry to do an efficient image clone.
-
         Returns a dict of volume properties eg. provider_location,
         boolean indicating whether cloning occurred.
         """
index bf191614902c95b0ef422780ab41d950203cd7c1..7c8260990caa1121072b8fe529fe1381b8972ea9 100644 (file)
@@ -805,7 +805,9 @@ class RBDDriver(driver.VolumeDriver):
                       dict(loc=image_location, err=e))
             return False
 
-    def clone_image(self, volume, image_location, image_meta):
+    def clone_image(self, context, volume,
+                    image_location, image_meta,
+                    image_service):
         image_location = image_location[0] if image_location else None
         if image_location is None or not self._is_cloneable(
                 image_location, image_meta):
index a16bb53db97af473e516f1c59f80f0f9cb166feb..c6f61a3b4ada17ad3e51b6f6288c52d9719ebbaf 100644 (file)
@@ -256,16 +256,22 @@ class ScalityDriver(driver.VolumeDriver):
                                   image_meta,
                                   self.local_path(volume))
 
-    def clone_image(self, volume, image_location, image_meta):
+    def clone_image(self, context, volume,
+                    image_location, image_meta,
+                    image_service):
         """Create a volume efficiently from an existing image.
 
         image_location is a string whose format depends on the
         image service backend in use. The driver should use it
         to determine whether cloning is possible.
 
-        image_id is a string which represents id of the image.
-        It can be used by the driver to introspect internal
-        stores or registry to do an efficient image clone.
+        image_meta is the metadata associated with the image and
+        includes properties like the image id, size, virtual-size
+        etc.
+
+        image_service is the reference of the image_service to use.
+        Note that this is needed to be passed here for drivers that
+        will want to fetch images from the image service directly.
 
         Returns a dict of volume properties eg. provider_location,
         boolean indicating whether cloning occurred
index 2684d27089ca2fe080e54027bcd38a14eed2079f..01532205df6199db27f742b9290277b8a40a6b5e 100644 (file)
@@ -14,6 +14,7 @@
 #    under the License.
 
 import json
+import math
 import random
 import socket
 import string
@@ -27,7 +28,8 @@ from six import wraps
 
 from cinder import context
 from cinder import exception
-from cinder.i18n import _, _LE, _LW
+from cinder.i18n import _, _LE, _LI, _LW
+from cinder.image import image_utils
 from cinder.openstack.common import log as logging
 from cinder.volume.drivers.san.san import SanISCSIDriver
 from cinder.volume import qos_specs
@@ -51,6 +53,17 @@ sf_opts = [
                     'and will create a prefix using the cinder node hostsname '
                     '(previous default behavior).  The default is NO prefix.'),
 
+    cfg.StrOpt('sf_template_account_name',
+               default='openstack-vtemplate',
+               help='Account name on the SolidFire Cluster to use as owner of '
+                    'template/cache volumes (created if doesnt exist).'),
+
+    cfg.BoolOpt('sf_allow_template_caching',
+                default=True,
+                help='Create an internal cache of copy of images when '
+                     'a bootable volume is created to eliminate fetch from '
+                     'glance and qemu-conversion on subsequent calls.'),
+
     cfg.IntOpt('sf_api_port',
                default=443,
                help='SolidFire API port. Useful if the device api is behind '
@@ -328,6 +341,7 @@ class SolidFireDriver(SanISCSIDriver):
         params = {'accountID': sfaccount['accountID']}
 
         sf_vol = self._get_sf_volume(src_uuid, params)
+
         if sf_vol is None:
             raise exception.VolumeNotFound(volume_id=src_uuid)
 
@@ -381,6 +395,15 @@ class SolidFireDriver(SanISCSIDriver):
             mesg = _('Failed to get model update from clone')
             raise exception.SolidFireAPIException(mesg)
 
+        # Increment the usage count, just for data collection
+        cloned_count = sf_vol['attributes'].get('cloned_count', 0)
+        cloned_count += 1
+        attributes = sf_vol['attributes']
+        attributes['cloned_count'] = cloned_count
+
+        params = {'volumeID': int(sf_vol['volumeID'])}
+        params['attributes'] = attributes
+        data = self._issue_api_request('ModifyVolume', params)
         return (data, sfaccount, model_update)
 
     def _do_volume_create(self, project_id, params):
@@ -477,6 +500,160 @@ class SolidFireDriver(SanISCSIDriver):
 
         return sf_volref
 
+    def _create_image_volume(self, context,
+                             image_meta, image_service,
+                             image_id):
+        # NOTE(jdg): It's callers responsibility to ensure that
+        # the optional properties.virtual_size is set on the image
+        # before we get here
+        virt_size = int(image_meta['properties'].get('virtual_size'))
+        min_sz_in_bytes =\
+            math.ceil(virt_size / float(units.Gi)) * float(units.Gi)
+        min_sz_in_gig = math.ceil(min_sz_in_bytes / float(units.Gi))
+
+        attributes = {}
+        attributes['image_info'] = {}
+        attributes['image_info']['image_updated_at'] =\
+            image_meta['updated_at'].isoformat()
+        attributes['image_info']['image_name'] =\
+            image_meta['name']
+        attributes['image_info']['image_created_at'] =\
+            image_meta['created_at'].isoformat()
+        attributes['image_info']['image_id'] = image_meta['id']
+
+        params = {'name': 'OpenStackIMG-%s' % image_id,
+                  'accountID': None,
+                  'sliceCount': 1,
+                  'totalSize': int(min_sz_in_bytes),
+                  'enable512e': self.configuration.sf_emulate_512,
+                  'attributes': attributes,
+                  'qos': {}}
+
+        account = self.configuration.sf_template_account_name
+        template_vol = self._do_volume_create(account, params)
+        tvol = {}
+        tvol['id'] = image_id
+        tvol['provider_location'] = template_vol['provider_location']
+        tvol['provider_auth'] = template_vol['provider_auth']
+
+        connector = 'na'
+        conn = self.initialize_connection(tvol, connector)
+        attach_info = super(SolidFireDriver, self)._connect_device(conn)
+
+        sfaccount = self._get_sfaccount(account)
+        params = {'accountID': sfaccount['accountID']}
+        properties = 'na'
+
+        try:
+            image_utils.fetch_to_raw(context,
+                                     image_service,
+                                     image_id,
+                                     attach_info['device']['path'],
+                                     self.configuration.volume_dd_blocksize,
+                                     size=min_sz_in_gig)
+        except Exception as exc:
+            params['volumeID'] = template_vol['volumeID']
+            LOG.error(_LE('Failed image conversion during cache creation: %s'),
+                      exc)
+            LOG.debug('Removing SolidFire Cache Volume (SF ID): %s',
+                      template_vol['volumeID'])
+
+            self._detach_volume(context, attach_info, tvol, properties)
+            self._issue_api_request('DeleteVolume', params)
+            return
+
+        self._detach_volume(context, attach_info, tvol, properties)
+        sf_vol = self._get_sf_volume(image_id, params)
+        LOG.debug('Successfully created SolidFire Image Template ',
+                  'for image-id: %s', image_id)
+        return sf_vol
+
+    def _verify_image_volume(self, context, image_meta, image_service):
+        # This method just verifies that IF we have a cache volume that
+        # it's still up to date and current WRT the image in Glance
+        # ie an image-update hasn't occured since we grabbed it
+
+        # If it's out of date, just delete it and we'll create a new one
+        # Any other case we don't care and just return without doing anything
+
+        account = self.configuration.sf_template_account_name
+        sfaccount = self._get_sfaccount(account)
+        params = {'accountID': sfaccount['accountID']}
+        sf_vol = self._get_sf_volume(image_meta['id'], params)
+        if sf_vol is None:
+            return
+
+        # Check updated_at field, delete copy and update if needed
+        if sf_vol['attributes']['image_info']['image_updated_at'] ==\
+                image_meta['updated_at'].isoformat():
+            return
+        else:
+            # Bummer, it's been updated, delete it
+            params = {'accountID': sfaccount['accountID']}
+            params = {'volumeID': sf_vol['volumeID']}
+            data = self._issue_api_request('DeleteVolume', params)
+            if 'result' not in data:
+                msg = _("Failed to delete SolidFire Image-Volume: %s") % data
+                raise exception.SolidFireAPIException(msg)
+
+            if not self._create_image_volume(context,
+                                             image_meta,
+                                             image_service,
+                                             image_meta['id']):
+                msg = _("Failed to create SolidFire Image-Volume")
+                raise exception.SolidFireAPIException(msg)
+
+    def clone_image(self, context,
+                    volume, image_location,
+                    image_meta, image_service):
+
+        # Check out pre-requisites:
+        # Is template caching enabled?
+        if not self.configuration.sf_allow_template_caching:
+            return None, False
+
+        # Is the image owned by this tenant or public?
+        if ((not image_meta.get('is_public', False)) and
+                (image_meta['owner'] != volume['project_id'])):
+                LOG.warning(_LW("Requested image is not "
+                                "accesible by current Tenant."))
+                return None, False
+
+        # Is virtual_size property set on the image?
+        if ((not image_meta.get('properties', None)) or
+                (not image_meta['properties'].get('virtual_size', None))):
+            LOG.info(_LI('Unable to create cache volume because image: %s '
+                         'does not include properties.virtual_size'),
+                     image_meta['id'])
+            return None, False
+
+        try:
+            self._verify_image_volume(context,
+                                      image_meta,
+                                      image_service)
+        except exception.SolidFireAPIException:
+            return None, False
+
+        account = self.configuration.sf_template_account_name
+        try:
+            (data, sfaccount, model) = self._do_clone_volume(image_meta['id'],
+                                                             account,
+                                                             volume)
+        except exception.VolumeNotFound:
+            if self._create_image_volume(context,
+                                         image_meta,
+                                         image_service,
+                                         image_meta['id']) is None:
+                # We failed, dump out
+                return None, False
+
+            # Ok, should be good to go now, try it again
+            (data, sfaccount, model) = self._do_clone_volume(image_meta['id'],
+                                                             account,
+                                                             volume)
+
+        return model, True
+
     def create_volume(self, volume):
         """Create volume on SolidFire device.
 
index 02d2dc5dd04818d182ffe36f9e4d90be4e3731d3..9037cd95d58529ef11a03709ea8b6eddde94e358 100644 (file)
@@ -567,8 +567,11 @@ class CreateVolumeFromSpecTask(flow_utils.CinderTask):
         # NOTE (singn): two params need to be returned
         # dict containing provider_location for cloned volume
         # and clone status.
-        model_update, cloned = self.driver.clone_image(
-            volume_ref, image_location, image_meta)
+        model_update, cloned = self.driver.clone_image(context,
+                                                       volume_ref,
+                                                       image_location,
+                                                       image_meta,
+                                                       image_service)
         if not cloned:
             # TODO(harlowja): what needs to be rolled back in the clone if this
             # volume create fails?? Likely this should be a subflow or broken