From: wanghao Date: Wed, 30 Dec 2015 06:18:09 +0000 (+0800) Subject: Add pagination support to consistency group X-Git-Url: https://review.fuel-infra.org/gitweb?a=commitdiff_plain;h=7d60f39911b24ae926f78775a26457e6aef8f89a;p=openstack-build%2Fcinder-build.git Add pagination support to consistency group In liberty release, we have added pagination to backups and snapshots. There are still some work that hasn't been done yet. This patch adds pagination support to consistency groups. APIImpact Add pagination args like limit, marker, sort to query consistency group. DocImpact Change-Id: I37602069e06cc99c9adbf45c2a981b0513a1be26 Implements: blueprint add-pagination-to-other-resource --- diff --git a/cinder/api/contrib/consistencygroups.py b/cinder/api/contrib/consistencygroups.py index 6d885192f..dbf11e4fc 100644 --- a/cinder/api/contrib/consistencygroups.py +++ b/cinder/api/contrib/consistencygroups.py @@ -195,15 +195,20 @@ class ConsistencyGroupsController(wsgi.Controller): def _get_consistencygroups(self, req, is_detail): """Returns a list of consistency groups through view builder.""" context = req.environ['cinder.context'] - consistencygroups = self.consistencygroup_api.get_all(context) - limited_list = common.limited(consistencygroups, req) + filters = req.params.copy() + marker, limit, offset = common.get_pagination_params(filters) + sort_keys, sort_dirs = common.get_sort_params(filters) + + consistencygroups = self.consistencygroup_api.get_all( + context, filters=filters, marker=marker, limit=limit, + offset=offset, sort_keys=sort_keys, sort_dirs=sort_dirs) if is_detail: - consistencygroups = self._view_builder.detail_list(req, - limited_list) + consistencygroups = self._view_builder.detail_list( + req, consistencygroups) else: - consistencygroups = self._view_builder.summary_list(req, - limited_list) + consistencygroups = self._view_builder.summary_list( + req, consistencygroups) return consistencygroups @wsgi.response(202) diff --git a/cinder/api/views/consistencygroups.py b/cinder/api/views/consistencygroups.py index 306790032..05d4c3a3b 100644 --- a/cinder/api/views/consistencygroups.py +++ b/cinder/api/views/consistencygroups.py @@ -72,6 +72,11 @@ class ViewBuilder(common.ViewBuilder): consistencygroups_list = [ func(request, consistencygroup)['consistencygroup'] for consistencygroup in consistencygroups] + cg_links = self._get_collection_links(request, + consistencygroups, + self._collection_name) consistencygroups_dict = dict(consistencygroups=consistencygroups_list) + if cg_links: + consistencygroups_dict['consistencygroup_links'] = cg_links return consistencygroups_dict diff --git a/cinder/consistencygroup/api.py b/cinder/consistencygroup/api.py index ef16314cf..171d43ed2 100644 --- a/cinder/consistencygroup/api.py +++ b/cinder/consistencygroup/api.py @@ -687,30 +687,25 @@ class API(base.Base): check_policy(context, 'get', group) return group - def get_all(self, context, marker=None, limit=None, sort_key='created_at', - sort_dir='desc', filters=None): + def get_all(self, context, filters=None, marker=None, limit=None, + offset=None, sort_keys=None, sort_dirs=None): check_policy(context, 'get_all') if filters is None: filters = {} - try: - if limit is not None: - limit = int(limit) - if limit < 0: - msg = _('limit param must be positive') - raise exception.InvalidInput(reason=msg) - except ValueError: - msg = _('limit param must be an integer') - raise exception.InvalidInput(reason=msg) - if filters: LOG.debug("Searching by: %s", filters) if (context.is_admin and 'all_tenants' in filters): - groups = objects.ConsistencyGroupList.get_all(context) + del filters['all_tenants'] + groups = objects.ConsistencyGroupList.get_all( + context, filters=filters, marker=marker, limit=limit, + offset=offset, sort_keys=sort_keys, sort_dirs=sort_dirs) else: groups = objects.ConsistencyGroupList.get_all_by_project( - context, context.project_id) + context, context.project_id, filters=filters, marker=marker, + limit=limit, offset=offset, sort_keys=sort_keys, + sort_dirs=sort_dirs) return groups def create_cgsnapshot(self, context, group, name, description): diff --git a/cinder/db/api.py b/cinder/db/api.py index 7b9907340..d223e865e 100644 --- a/cinder/db/api.py +++ b/cinder/db/api.py @@ -946,9 +946,13 @@ def consistencygroup_get(context, consistencygroup_id): return IMPL.consistencygroup_get(context, consistencygroup_id) -def consistencygroup_get_all(context): +def consistencygroup_get_all(context, filters=None, marker=None, limit=None, + offset=None, sort_keys=None, sort_dirs=None): """Get all consistencygroups.""" - return IMPL.consistencygroup_get_all(context) + return IMPL.consistencygroup_get_all(context, filters=filters, + marker=marker, limit=limit, + offset=offset, sort_keys=sort_keys, + sort_dirs=sort_dirs) def consistencygroup_create(context, values): @@ -956,9 +960,16 @@ def consistencygroup_create(context, values): return IMPL.consistencygroup_create(context, values) -def consistencygroup_get_all_by_project(context, project_id): +def consistencygroup_get_all_by_project(context, project_id, filters=None, + marker=None, limit=None, offset=None, + sort_keys=None, sort_dirs=None): """Get all consistencygroups belonging to a project.""" - return IMPL.consistencygroup_get_all_by_project(context, project_id) + return IMPL.consistencygroup_get_all_by_project(context, project_id, + filters=filters, + marker=marker, limit=limit, + offset=offset, + sort_keys=sort_keys, + sort_dirs=sort_dirs) def consistencygroup_update(context, consistencygroup_id, values): diff --git a/cinder/db/sqlalchemy/api.py b/cinder/db/sqlalchemy/api.py index 9efab63d4..dad187105 100644 --- a/cinder/db/sqlalchemy/api.py +++ b/cinder/db/sqlalchemy/api.py @@ -3792,17 +3792,99 @@ def consistencygroup_get(context, consistencygroup_id): return _consistencygroup_get(context, consistencygroup_id) +def _consistencygroups_get_query(context, session=None, project_only=False): + return model_query(context, models.ConsistencyGroup, session=session, + project_only=project_only) + + +def _process_consistencygroups_filters(query, filters): + if filters: + # Ensure that filters' keys exist on the model + if not is_valid_model_filters(models.ConsistencyGroup, filters): + return + query = query.filter_by(**filters) + return query + + +def _consistencygroup_get_all(context, filters=None, marker=None, limit=None, + offset=None, sort_keys=None, sort_dirs=None): + if filters and not is_valid_model_filters(models.ConsistencyGroup, + filters): + return [] + + session = get_session() + with session.begin(): + # Generate the paginate query + query = _generate_paginate_query(context, session, marker, + limit, sort_keys, sort_dirs, filters, + offset, models.ConsistencyGroup) + if query is None: + return [] + return query.all() + + @require_admin_context -def consistencygroup_get_all(context): - return model_query(context, models.ConsistencyGroup).all() +def consistencygroup_get_all(context, filters=None, marker=None, limit=None, + offset=None, sort_keys=None, sort_dirs=None): + """Retrieves all consistency groups. + + If no sort parameters are specified then the returned cgs are sorted + first by the 'created_at' key and then by the 'id' key in descending + order. + + :param context: context to query under + :param marker: the last item of the previous page, used to determine the + next page of results to return + :param limit: maximum number of items to return + :param sort_keys: list of attributes by which results should be sorted, + paired with corresponding item in sort_dirs + :param sort_dirs: list of directions in which results should be sorted, + paired with corresponding item in sort_keys + :param filters: dictionary of filters; values that are in lists, tuples, + or sets cause an 'IN' operation, while exact matching + is used for other values, see + _process_consistencygroups_filters function for more + information + :returns: list of matching consistency groups + """ + return _consistencygroup_get_all(context, filters, marker, limit, offset, + sort_keys, sort_dirs) @require_context -def consistencygroup_get_all_by_project(context, project_id): +def consistencygroup_get_all_by_project(context, project_id, filters=None, + marker=None, limit=None, offset=None, + sort_keys=None, sort_dirs=None): + """Retrieves all consistency groups in a project. + + If no sort parameters are specified then the returned cgs are sorted + first by the 'created_at' key and then by the 'id' key in descending + order. + + :param context: context to query under + :param marker: the last item of the previous page, used to determine the + next page of results to return + :param limit: maximum number of items to return + :param sort_keys: list of attributes by which results should be sorted, + paired with corresponding item in sort_dirs + :param sort_dirs: list of directions in which results should be sorted, + paired with corresponding item in sort_keys + :param filters: dictionary of filters; values that are in lists, tuples, + or sets cause an 'IN' operation, while exact matching + is used for other values, see + _process_consistencygroups_filters function for more + information + :returns: list of matching consistency groups + """ authorize_project_context(context, project_id) + if not filters: + filters = {} + else: + filters = filters.copy() - return model_query(context, models.ConsistencyGroup).\ - filter_by(project_id=project_id).all() + filters['project_id'] = project_id + return _consistencygroup_get_all(context, filters, marker, limit, offset, + sort_keys, sort_dirs) @require_context @@ -4065,7 +4147,10 @@ PAGINATION_HELPERS = { models.QualityOfServiceSpecs: (_qos_specs_get_query, _process_qos_specs_filters, _qos_specs_get), models.VolumeTypes: (_volume_type_get_query, _process_volume_types_filters, - _volume_type_get_db_object) + _volume_type_get_db_object), + models.ConsistencyGroup: (_consistencygroups_get_query, + _process_consistencygroups_filters, + _consistencygroup_get) } diff --git a/cinder/objects/consistencygroup.py b/cinder/objects/consistencygroup.py index 07e41d2c4..1f90c7077 100644 --- a/cinder/objects/consistencygroup.py +++ b/cinder/objects/consistencygroup.py @@ -136,26 +136,35 @@ class ConsistencyGroup(base.CinderPersistentObject, base.CinderObject, @base.CinderObjectRegistry.register class ConsistencyGroupList(base.ObjectListBase, base.CinderObject): - VERSION = '1.0' + # Version 1.0: Initial version + # Version 1.1: Add pagination support to consistency group + VERSION = '1.1' fields = { 'objects': fields.ListOfObjectsField('ConsistencyGroup') } child_version = { - '1.0': '1.0' + '1.0': '1.0', + '1.1': '1.1', } @base.remotable_classmethod - def get_all(cls, context): - consistencygroups = db.consistencygroup_get_all(context) + def get_all(cls, context, filters=None, marker=None, limit=None, + offset=None, sort_keys=None, sort_dirs=None): + consistencygroups = db.consistencygroup_get_all( + context, filters=filters, marker=marker, limit=limit, + offset=offset, sort_keys=sort_keys, sort_dirs=sort_dirs) return base.obj_make_list(context, cls(context), objects.ConsistencyGroup, consistencygroups) @base.remotable_classmethod - def get_all_by_project(cls, context, project_id): - consistencygroups = db.consistencygroup_get_all_by_project(context, - project_id) + def get_all_by_project(cls, context, project_id, filters=None, marker=None, + limit=None, offset=None, sort_keys=None, + sort_dirs=None): + consistencygroups = db.consistencygroup_get_all_by_project( + context, project_id, filters=filters, marker=marker, limit=limit, + offset=offset, sort_keys=sort_keys, sort_dirs=sort_dirs) return base.obj_make_list(context, cls(context), objects.ConsistencyGroup, consistencygroups) diff --git a/cinder/tests/unit/api/contrib/test_consistencygroups.py b/cinder/tests/unit/api/contrib/test_consistencygroups.py index 4ea93c1b0..d63d758bc 100644 --- a/cinder/tests/unit/api/contrib/test_consistencygroups.py +++ b/cinder/tests/unit/api/contrib/test_consistencygroups.py @@ -17,6 +17,7 @@ Tests for consistency group code. """ +import ddt import json from xml.dom import minidom @@ -37,6 +38,7 @@ from cinder.tests.unit import utils from cinder.volume import api as volume_api +@ddt.ddt class ConsistencyGroupsAPITestCase(test.TestCase): """Test Case for consistency groups API.""" @@ -153,7 +155,7 @@ class ConsistencyGroupsAPITestCase(test.TestCase): res_dict = json.loads(res.body) self.assertEqual(200, res.status_int) - self.assertEqual(consistencygroup1.id, + self.assertEqual(consistencygroup3.id, res_dict['consistencygroups'][0]['id']) self.assertEqual('test_consistencygroup', res_dict['consistencygroups'][0]['name']) @@ -161,7 +163,7 @@ class ConsistencyGroupsAPITestCase(test.TestCase): res_dict['consistencygroups'][1]['id']) self.assertEqual('test_consistencygroup', res_dict['consistencygroups'][1]['name']) - self.assertEqual(consistencygroup3.id, + self.assertEqual(consistencygroup1.id, res_dict['consistencygroups'][2]['id']) self.assertEqual('test_consistencygroup', res_dict['consistencygroups'][2]['name']) @@ -185,17 +187,146 @@ class ConsistencyGroupsAPITestCase(test.TestCase): dom = minidom.parseString(res.body) consistencygroup_list = dom.getElementsByTagName('consistencygroup') - self.assertEqual(consistencygroup1.id, + self.assertEqual(consistencygroup3.id, consistencygroup_list.item(0).getAttribute('id')) self.assertEqual(consistencygroup2.id, consistencygroup_list.item(1).getAttribute('id')) - self.assertEqual(consistencygroup3.id, + self.assertEqual(consistencygroup1.id, consistencygroup_list.item(2).getAttribute('id')) consistencygroup3.destroy() consistencygroup2.destroy() consistencygroup1.destroy() + @ddt.data(False, True) + def test_list_consistencygroups_with_limit(self, is_detail): + consistencygroup1 = self._create_consistencygroup() + consistencygroup2 = self._create_consistencygroup() + consistencygroup3 = self._create_consistencygroup() + url = '/v2/fake/consistencygroups?limit=1' + if is_detail: + url = '/v2/fake/consistencygroups/detail?limit=1' + req = webob.Request.blank(url) + req.method = 'GET' + req.headers['Content-Type'] = 'application/json' + res = req.get_response(fakes.wsgi_app()) + res_dict = json.loads(res.body) + + self.assertEqual(200, res.status_int) + self.assertEqual(1, len(res_dict['consistencygroups'])) + self.assertEqual(consistencygroup3.id, + res_dict['consistencygroups'][0]['id']) + next_link = ('http://localhost/v2/fake/consistencygroups?limit=' + '1&marker=%s') % res_dict['consistencygroups'][0]['id'] + self.assertEqual(next_link, + res_dict['consistencygroup_links'][0]['href']) + consistencygroup1.destroy() + consistencygroup2.destroy() + consistencygroup3.destroy() + + @ddt.data(False, True) + def test_list_consistencygroups_with_offset(self, is_detail): + consistencygroup1 = self._create_consistencygroup() + consistencygroup2 = self._create_consistencygroup() + consistencygroup3 = self._create_consistencygroup() + url = '/v2/fake/consistencygroups?offset=1' + if is_detail: + url = '/v2/fake/consistencygroups/detail?offset=1' + req = webob.Request.blank(url) + req.method = 'GET' + req.headers['Content-Type'] = 'application/json' + res = req.get_response(fakes.wsgi_app()) + res_dict = json.loads(res.body) + + self.assertEqual(200, res.status_int) + self.assertEqual(2, len(res_dict['consistencygroups'])) + self.assertEqual(consistencygroup2.id, + res_dict['consistencygroups'][0]['id']) + self.assertEqual(consistencygroup1.id, + res_dict['consistencygroups'][1]['id']) + consistencygroup1.destroy() + consistencygroup2.destroy() + consistencygroup3.destroy() + + @ddt.data(False, True) + def test_list_consistencygroups_with_limit_and_offset(self, is_detail): + consistencygroup1 = self._create_consistencygroup() + consistencygroup2 = self._create_consistencygroup() + consistencygroup3 = self._create_consistencygroup() + url = '/v2/fake/consistencygroups?limit=2&offset=1' + if is_detail: + url = '/v2/fake/consistencygroups/detail?limit=2&offset=1' + req = webob.Request.blank(url) + req.method = 'GET' + req.headers['Content-Type'] = 'application/json' + res = req.get_response(fakes.wsgi_app()) + res_dict = json.loads(res.body) + + self.assertEqual(200, res.status_int) + self.assertEqual(2, len(res_dict['consistencygroups'])) + self.assertEqual(consistencygroup2.id, + res_dict['consistencygroups'][0]['id']) + self.assertEqual(consistencygroup1.id, + res_dict['consistencygroups'][1]['id']) + consistencygroup1.destroy() + consistencygroup2.destroy() + consistencygroup3.destroy() + + @ddt.data(False, True) + def test_list_consistencygroups_with_filter(self, is_detail): + consistencygroup1 = self._create_consistencygroup() + consistencygroup2 = self._create_consistencygroup() + common_ctxt = context.RequestContext('fake', 'fake', auth_token=True, + is_admin=False) + consistencygroup3 = self._create_consistencygroup(ctxt=common_ctxt) + url = ('/v2/fake/consistencygroups?' + 'all_tenants=True&id=%s') % consistencygroup3.id + if is_detail: + url = ('/v2/fake/consistencygroups/detail?' + 'all_tenants=True&id=%s') % consistencygroup3.id + req = webob.Request.blank(url) + req.method = 'GET' + req.headers['Content-Type'] = 'application/json' + res = req.get_response(fakes.wsgi_app(fake_auth_context=self.ctxt)) + res_dict = json.loads(res.body) + + self.assertEqual(200, res.status_int) + self.assertEqual(1, len(res_dict['consistencygroups'])) + self.assertEqual(consistencygroup3.id, + res_dict['consistencygroups'][0]['id']) + consistencygroup1.destroy() + consistencygroup2.destroy() + consistencygroup3.destroy() + + @ddt.data(False, True) + def test_list_consistencygroups_with_sort(self, is_detail): + consistencygroup1 = self._create_consistencygroup() + consistencygroup2 = self._create_consistencygroup() + consistencygroup3 = self._create_consistencygroup() + url = '/v2/fake/consistencygroups?sort=id:asc' + if is_detail: + url = '/v2/fake/consistencygroups/detail?sort=id:asc' + req = webob.Request.blank(url) + req.method = 'GET' + req.headers['Content-Type'] = 'application/json' + res = req.get_response(fakes.wsgi_app()) + res_dict = json.loads(res.body) + expect_result = [consistencygroup1.id, consistencygroup2.id, + consistencygroup3.id] + expect_result.sort() + + self.assertEqual(200, res.status_int) + self.assertEqual(3, len(res_dict['consistencygroups'])) + self.assertEqual(expect_result[0], + res_dict['consistencygroups'][0]['id']) + self.assertEqual(expect_result[1], + res_dict['consistencygroups'][1]['id']) + self.assertEqual(expect_result[2], + res_dict['consistencygroups'][2]['id']) + consistencygroup1.destroy() + consistencygroup2.destroy() + consistencygroup3.destroy() + def test_list_consistencygroups_detail_json(self): consistencygroup1 = self._create_consistencygroup() consistencygroup2 = self._create_consistencygroup() @@ -216,11 +347,11 @@ class ConsistencyGroupsAPITestCase(test.TestCase): res_dict['consistencygroups'][0]['description']) self.assertEqual('test_consistencygroup', res_dict['consistencygroups'][0]['name']) - self.assertEqual(consistencygroup1.id, + self.assertEqual(consistencygroup3.id, res_dict['consistencygroups'][0]['id']) self.assertEqual('creating', res_dict['consistencygroups'][0]['status']) - self.assertEqual(['123456'], + self.assertEqual(['uuid1', 'uuid2'], res_dict['consistencygroups'][0]['volume_types']) self.assertEqual('az1', @@ -242,11 +373,11 @@ class ConsistencyGroupsAPITestCase(test.TestCase): res_dict['consistencygroups'][2]['description']) self.assertEqual('test_consistencygroup', res_dict['consistencygroups'][2]['name']) - self.assertEqual(consistencygroup3.id, + self.assertEqual(consistencygroup1.id, res_dict['consistencygroups'][2]['id']) self.assertEqual('creating', res_dict['consistencygroups'][2]['status']) - self.assertEqual(['uuid1', 'uuid2'], + self.assertEqual(['123456'], res_dict['consistencygroups'][2]['volume_types']) consistencygroup1.destroy() @@ -278,7 +409,7 @@ class ConsistencyGroupsAPITestCase(test.TestCase): 'test_consistencygroup', consistencygroup_detail.item(0).getAttribute('name')) self.assertEqual( - consistencygroup1.id, + consistencygroup3.id, consistencygroup_detail.item(0).getAttribute('id')) self.assertEqual( 'creating', @@ -310,7 +441,7 @@ class ConsistencyGroupsAPITestCase(test.TestCase): 'test_consistencygroup', consistencygroup_detail.item(2).getAttribute('name')) self.assertEqual( - consistencygroup3.id, + consistencygroup1.id, consistencygroup_detail.item(2).getAttribute('id')) self.assertEqual( 'creating', diff --git a/cinder/tests/unit/objects/test_consistencygroup.py b/cinder/tests/unit/objects/test_consistencygroup.py index 2188508d1..c5125c031 100644 --- a/cinder/tests/unit/objects/test_consistencygroup.py +++ b/cinder/tests/unit/objects/test_consistencygroup.py @@ -184,3 +184,30 @@ class TestConsistencyGroupList(test_objects.BaseObjectsTestCase): self.assertEqual(1, len(consistencygroups)) TestConsistencyGroup._compare(self, fake_consistencygroup, consistencygroups[0]) + + @mock.patch('cinder.db.consistencygroup_get_all', + return_value=[fake_consistencygroup]) + def test_get_all_with_pagination(self, consistencygroup_get_all): + consistencygroups = objects.ConsistencyGroupList.get_all( + self.context, filters={'id': 'fake'}, marker=None, limit=1, + offset=None, sort_keys='id', sort_dirs='asc') + self.assertEqual(1, len(consistencygroups)) + consistencygroup_get_all.assert_called_once_with( + self.context, filters={'id': 'fake'}, marker=None, limit=1, + offset=None, sort_keys='id', sort_dirs='asc') + TestConsistencyGroup._compare(self, fake_consistencygroup, + consistencygroups[0]) + + @mock.patch('cinder.db.consistencygroup_get_all_by_project', + return_value=[fake_consistencygroup]) + def test_get_all_by_project_with_pagination( + self, consistencygroup_get_all_by_project): + consistencygroups = objects.ConsistencyGroupList.get_all_by_project( + self.context, self.project_id, filters={'id': 'fake'}, marker=None, + limit=1, offset=None, sort_keys='id', sort_dirs='asc') + self.assertEqual(1, len(consistencygroups)) + consistencygroup_get_all_by_project.assert_called_once_with( + self.context, self.project_id, filters={'id': 'fake'}, marker=None, + limit=1, offset=None, sort_keys='id', sort_dirs='asc') + TestConsistencyGroup._compare(self, fake_consistencygroup, + consistencygroups[0]) diff --git a/cinder/tests/unit/objects/test_objects.py b/cinder/tests/unit/objects/test_objects.py index e6527d782..bda91f30b 100644 --- a/cinder/tests/unit/objects/test_objects.py +++ b/cinder/tests/unit/objects/test_objects.py @@ -27,7 +27,7 @@ object_data = { 'CGSnapshot': '1.0-190da2a2aa9457edc771d888f7d225c4', 'CGSnapshotList': '1.0-e8c3f4078cd0ee23487b34d173eec776', 'ConsistencyGroup': '1.2-ed7f90a6871991a19af716ade7337fc9', - 'ConsistencyGroupList': '1.0-09d0aad5491e762ecfdf66bef02ceb8d', + 'ConsistencyGroupList': '1.1-73916823b697dfa0c7f02508d87e0f28', 'Service': '1.0-64baeb4911dbab1153064dd1c87edb9f', 'ServiceList': '1.0-d242d3384b68e5a5a534e090ff1d5161', 'Snapshot': '1.0-a6c33eefeadefb324d79f72f66c54e9a',