From 71d9f1170a5fe087de5a957e46d8c9e0fe9fb574 Mon Sep 17 00:00:00 2001 From: John Griffith Date: Tue, 13 Nov 2012 19:03:50 +0000 Subject: [PATCH] Add hosts extension to Cinder. This adds the hosts extension to Cinder which is similar to the extension by the same name in Nova. This extension allows an admin user to check the status of Cinder services on different hosts. In addition it also provides the capability to get host and project specific information regarding volume count and space allocated for volumes on a specified host. Implements blueprint cinder-hosts-extension Change-Id: Ic3679cfcd900baecdc2da17e42194cac99c9d422 --- cinder/api/openstack/volume/contrib/hosts.py | 261 ++++++++++++++++++ cinder/db/api.py | 14 + cinder/db/sqlalchemy/api.py | 34 ++- .../openstack/volume/contrib/test_hosts.py | 187 +++++++++++++ cinder/tests/policy.json | 3 +- cinder/volume/api.py | 22 ++ etc/cinder/policy.json | 3 +- 7 files changed, 521 insertions(+), 3 deletions(-) create mode 100644 cinder/api/openstack/volume/contrib/hosts.py create mode 100644 cinder/tests/api/openstack/volume/contrib/test_hosts.py diff --git a/cinder/api/openstack/volume/contrib/hosts.py b/cinder/api/openstack/volume/contrib/hosts.py new file mode 100644 index 000000000..cb83f57b5 --- /dev/null +++ b/cinder/api/openstack/volume/contrib/hosts.py @@ -0,0 +1,261 @@ +# Copyright (c) 2011 OpenStack, LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""The hosts admin extension.""" + +import webob.exc +from xml.dom import minidom +from xml.parsers import expat + +from cinder.api.openstack import extensions +from cinder.api.openstack import wsgi +from cinder.api.openstack import xmlutil +from cinder.volume import api as volume_api +from cinder import db +from cinder import exception +from cinder import flags +from cinder.openstack.common import log as logging +from cinder.openstack.common import timeutils +from cinder import utils + +FLAGS = flags.FLAGS +LOG = logging.getLogger(__name__) +authorize = extensions.extension_authorizer('volume', 'hosts') + + +class HostIndexTemplate(xmlutil.TemplateBuilder): + def construct(self): + def shimmer(obj, do_raise=False): + # A bare list is passed in; we need to wrap it in a dict + return dict(hosts=obj) + + root = xmlutil.TemplateElement('hosts', selector=shimmer) + elem = xmlutil.SubTemplateElement(root, 'host', selector='hosts') + elem.set('host') + elem.set('topic') + + return xmlutil.MasterTemplate(root, 1) + + +class HostUpdateTemplate(xmlutil.TemplateBuilder): + def construct(self): + root = xmlutil.TemplateElement('host') + root.set('host') + root.set('status') + + return xmlutil.MasterTemplate(root, 1) + + +class HostActionTemplate(xmlutil.TemplateBuilder): + def construct(self): + root = xmlutil.TemplateElement('host') + root.set('host') + + return xmlutil.MasterTemplate(root, 1) + + +class HostShowTemplate(xmlutil.TemplateBuilder): + def construct(self): + root = xmlutil.TemplateElement('host') + elem = xmlutil.make_flat_dict('resource', selector='host', + subselector='resource') + root.append(elem) + + return xmlutil.MasterTemplate(root, 1) + + +class HostDeserializer(wsgi.XMLDeserializer): + def default(self, string): + try: + node = minidom.parseString(string) + except expat.ExpatError: + msg = _("cannot understand XML") + raise exception.MalformedRequestBody(reason=msg) + + updates = {} + for child in node.childNodes[0].childNodes: + updates[child.tagName] = self.extract_text(child) + + return dict(body=updates) + + +def _list_hosts(req, service=None): + """Returns a summary list of hosts.""" + curr_time = timeutils.utcnow() + context = req.environ['cinder.context'] + services = db.service_get_all(context, False) + zone = '' + if 'zone' in req.GET: + zone = req.GET['zone'] + if zone: + services = [s for s in services if s['availability_zone'] == zone] + hosts = [] + for host in services: + delta = curr_time - (host['updated_at'] or host['created_at']) + alive = abs(utils.total_seconds(delta)) <= FLAGS.service_down_time + status = (alive and "available") or "unavailable" + active = 'enabled' + if host['disabled']: + active = 'disabled' + LOG.debug('status, active and update: %s, %s, %s' % + (status, active, host['updated_at'])) + hosts.append({'host_name': host['host'], + 'service': host['topic'], + 'zone': host['availability_zone'], + 'service-status': status, + 'service-state': active, + 'last-update': host['updated_at']}) + if service: + hosts = [host for host in hosts + if host["service"] == service] + return hosts + + +def check_host(fn): + """Makes sure that the host exists.""" + def wrapped(self, req, id, service=None, *args, **kwargs): + listed_hosts = _list_hosts(req, service) + hosts = [h["host_name"] for h in listed_hosts] + if id in hosts: + return fn(self, req, id, *args, **kwargs) + else: + message = _("Host '%s' could not be found.") % id + raise webob.exc.HTTPNotFound(explanation=message) + return wrapped + + +class HostController(object): + """The Hosts API controller for the OpenStack API.""" + def __init__(self): + self.api = volume_api.HostAPI() + super(HostController, self).__init__() + + @wsgi.serializers(xml=HostIndexTemplate) + def index(self, req): + authorize(req.environ['cinder.context']) + return {'hosts': _list_hosts(req)} + + @wsgi.serializers(xml=HostUpdateTemplate) + @wsgi.deserializers(xml=HostDeserializer) + @check_host + def update(self, req, id, body): + authorize(req.environ['cinder.context']) + update_values = {} + for raw_key, raw_val in body.iteritems(): + key = raw_key.lower().strip() + val = raw_val.lower().strip() + if key == "status": + if val in ("enable", "disable"): + update_values['status'] = val.startswith("enable") + else: + explanation = _("Invalid status: '%s'") % raw_val + raise webob.exc.HTTPBadRequest(explanation=explanation) + else: + explanation = _("Invalid update setting: '%s'") % raw_key + raise webob.exc.HTTPBadRequest(explanation=explanation) + update_setters = {'status': self._set_enabled_status} + result = {} + for key, value in update_values.iteritems(): + result.update(update_setters[key](req, id, value)) + return result + + def _set_enabled_status(self, req, host, enabled): + """Sets the specified host's ability to accept new volumes.""" + context = req.environ['cinder.context'] + state = "enabled" if enabled else "disabled" + LOG.audit(_("Setting host %(host)s to %(state)s.") % locals()) + result = self.api.set_host_enabled(context, host=host, + enabled=enabled) + if result not in ("enabled", "disabled"): + # An error message was returned + raise webob.exc.HTTPBadRequest(explanation=result) + return {"host": host, "status": result} + + #@wsgi.serializers(xml=HostShowTemplate) + def show(self, req, id): + """Shows the volume usage info given by hosts. + + :param context: security context + :param host: hostname + :returns: expected to use HostShowTemplate. + ex.:: + + {'host': {'resource':D},..} + D: {'host': 'hostname','project': 'admin', + 'volume_count': 1, 'total_volume_gb': 2048} + """ + host = id + context = req.environ['cinder.context'] + if not context.is_admin: + msg = _("Describe-resource is admin only functionality") + raise webob.exc.HTTPForbidden(explanation=msg) + + try: + host_ref = db.service_get_by_host_and_topic(context, + host, + 'cinder-volume') + except exception.ServiceNotFound: + raise webob.exc.HTTPNotFound(explanation=_("Host not found")) + + # Getting total available/used resource + # TODO(jdg): Add summary info for Snapshots + volume_refs = db.volume_get_all_by_host(context, host_ref['host']) + (count, sum) = db.volume_data_get_for_host(context, + host_ref['host']) + + snap_count_total = 0 + snap_sum_total = 0 + resources = [{'resource': {'host': host, 'project': '(total)', + 'volume_count': str(count), + 'total_volume_gb': str(sum)}, + 'snapshot_count': str(snap_count_total), + 'total_snapshot_gb': str(snap_sum_total)}] + + project_ids = [v['project_id'] for v in volume_refs] + project_ids = list(set(project_ids)) + for project_id in project_ids: + (count, sum) = db.volume_data_get_for_project(context, project_id) + (snap_count, snap_sum) = db.snapshot_data_get_for_project( + context, + project_id) + resources.append({'resource': + {'host': host, + 'project': project_id, + 'volume_count': str(count), + 'total_volume_gb': str(sum), + 'snapshot_count': str(snap_count), + 'total_snapshot_gb': str(snap_sum)}}) + snap_count_total += int(snap_count) + snap_sum_total += int(snap_sum) + resources[0]['resource']['snapshot_count'] = str(snap_count_total) + resources[0]['resource']['total_snapshot_gb'] = str(snap_sum_total) + return {"host": resources} + + +class Hosts(extensions.ExtensionDescriptor): + """Admin-only host administration""" + + name = "Hosts" + alias = "os-hosts" + namespace = "http://docs.openstack.org/volume/ext/hosts/api/v1.1" + updated = "2011-06-29T00:00:00+00:00" + + def get_resources(self): + resources = [extensions.ResourceExtension('os-hosts', + HostController(), + collection_actions={'update': 'PUT'}, + member_actions={"startup": "GET", "shutdown": "GET", + "reboot": "GET"})] + return resources diff --git a/cinder/db/api.py b/cinder/db/api.py index 7ac9acffa..6cebfb428 100644 --- a/cinder/db/api.py +++ b/cinder/db/api.py @@ -200,6 +200,13 @@ def volume_create(context, values): return IMPL.volume_create(context, values) +def volume_data_get_for_host(context, host, session=None): + """Get (volume_count, gigabytes) for project.""" + return IMPL.volume_data_get_for_host(context, + host, + session) + + def volume_data_get_for_project(context, project_id, session=None): """Get (volume_count, gigabytes) for project.""" return IMPL.volume_data_get_for_project(context, @@ -298,6 +305,13 @@ def snapshot_update(context, snapshot_id, values): return IMPL.snapshot_update(context, snapshot_id, values) +def snapshot_data_get_for_project(context, project_id, session=None): + """Get count and gigabytes used for snapshots for specified project.""" + return IMPL.snapshot_data_get_for_project(context, + project_id, + session=None) + + #################### diff --git a/cinder/db/sqlalchemy/api.py b/cinder/db/sqlalchemy/api.py index bfc9303a8..4a3ecb95d 100644 --- a/cinder/db/sqlalchemy/api.py +++ b/cinder/db/sqlalchemy/api.py @@ -257,11 +257,14 @@ def service_get_all_by_topic(context, topic): @require_admin_context def service_get_by_host_and_topic(context, host, topic): - return model_query(context, models.Service, read_deleted="no").\ + result = model_query(context, models.Service, read_deleted="no").\ filter_by(disabled=False).\ filter_by(host=host).\ filter_by(topic=topic).\ first() + if not result: + raise exception.ServiceNotFound(host=host, topic=topic) + return result @require_admin_context @@ -925,6 +928,20 @@ def volume_create(context, values): return volume_get(context, values['id'], session=session) +@require_admin_context +def volume_data_get_for_host(context, host, session=None): + result = model_query(context, + func.count(models.Volume.id), + func.sum(models.Volume.size), + read_deleted="no", + session=session).\ + filter_by(host=host).\ + first() + + # NOTE(vish): convert None to 0 + return (result[0] or 0, result[1] or 0) + + @require_admin_context def volume_data_get_for_project(context, project_id, session=None): result = model_query(context, @@ -1188,6 +1205,21 @@ def snapshot_get_all_by_project(context, project_id): all() +@require_context +def snapshot_data_get_for_project(context, project_id, session=None): + authorize_project_context(context, project_id) + result = model_query(context, + func.count(models.Snapshot.id), + func.sum(models.Snapshot.volume_size), + read_deleted="no", + session=session).\ + filter_by(project_id=project_id).\ + first() + + # NOTE(vish): convert None to 0 + return (result[0] or 0, result[1] or 0) + + @require_context def snapshot_update(context, snapshot_id, values): session = get_session() diff --git a/cinder/tests/api/openstack/volume/contrib/test_hosts.py b/cinder/tests/api/openstack/volume/contrib/test_hosts.py new file mode 100644 index 000000000..fc71a1be1 --- /dev/null +++ b/cinder/tests/api/openstack/volume/contrib/test_hosts.py @@ -0,0 +1,187 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright (c) 2011 OpenStack, LLC. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from lxml import etree +import webob.exc + +from cinder.api.openstack.volume.contrib import hosts as os_hosts +from cinder import context +import datetime +from cinder import db +from cinder import flags +from cinder.openstack.common import log as logging +from cinder.openstack.common import timeutils +from cinder import test + + +FLAGS = flags.FLAGS +LOG = logging.getLogger(__name__) +created_time = datetime.datetime(2012, 11, 14, 1, 20, 41, 95099) +curr_time = timeutils.utcnow() + +SERVICE_LIST = [ + {'created_at': created_time, 'updated_at': curr_time, + 'host': 'test.host.1', 'topic': 'cinder-volume', 'disabled': 0, + 'availability_zone': 'cinder'}, + {'created_at': created_time, 'updated_at': curr_time, + 'host': 'test.host.1', 'topic': 'cinder-volume', 'disabled': 0, + 'availability_zone': 'cinder'}, + {'created_at': created_time, 'updated_at': curr_time, + 'host': 'test.host.1', 'topic': 'cinder-volume', 'disabled': 0, + 'availability_zone': 'cinder'}, + {'created_at': created_time, 'updated_at': curr_time, + 'host': 'test.host.1', 'topic': 'cinder-volume', 'disabled': 0, + 'availability_zone': 'cinder'}] + +LIST_RESPONSE = [{'service-status': 'available', 'service': 'cinder-volume', + 'zone': 'cinder', 'service-state': 'enabled', + 'host_name': 'test.host.1', 'last-update': curr_time}, + {'service-status': 'available', 'service': 'cinder-volume', + 'zone': 'cinder', 'service-state': 'enabled', + 'host_name': 'test.host.1', 'last-update': curr_time}, + {'service-status': 'available', 'service': 'cinder-volume', + 'zone': 'cinder', 'service-state': 'enabled', + 'host_name': 'test.host.1', 'last-update': curr_time}, + {'service-status': 'available', 'service': 'cinder-volume', + 'zone': 'cinder', 'service-state': 'enabled', + 'host_name': 'test.host.1', 'last-update': curr_time}] + + +def stub_service_get_all(self, req): + return SERVICE_LIST + + +class FakeRequest(object): + environ = {'cinder.context': context.get_admin_context()} + GET = {} + + +class FakeRequestWithcinderZone(object): + environ = {'cinder.context': context.get_admin_context()} + GET = {'zone': 'cinder'} + + +class HostTestCase(test.TestCase): + """Test Case for hosts.""" + + def setUp(self): + super(HostTestCase, self).setUp() + self.controller = os_hosts.HostController() + self.req = FakeRequest() + self.stubs.Set(db, 'service_get_all', + stub_service_get_all) + + def _test_host_update(self, host, key, val, expected_value): + body = {key: val} + result = self.controller.update(self.req, host, body=body) + self.assertEqual(result[key], expected_value) + + def test_list_hosts(self): + """Verify that the volume hosts are returned.""" + hosts = os_hosts._list_hosts(self.req) + self.assertEqual(hosts, LIST_RESPONSE) + + cinder_hosts = os_hosts._list_hosts(self.req, 'cinder-volume') + expected = [host for host in LIST_RESPONSE + if host['service'] == 'cinder-volume'] + self.assertEqual(cinder_hosts, expected) + + def test_list_hosts_with_zone(self): + req = FakeRequestWithcinderZone() + hosts = os_hosts._list_hosts(req) + self.assertEqual(hosts, LIST_RESPONSE) + + def test_bad_status_value(self): + self.assertRaises(webob.exc.HTTPBadRequest, self.controller.update, + self.req, 'test.host.1', body={'status': 'bad'}) + self.assertRaises(webob.exc.HTTPBadRequest, self.controller.update, + self.req, 'test.host.1', body={'status': 'disablabc'}) + + def test_bad_update_key(self): + bad_body = {'crazy': 'bad'} + self.assertRaises(webob.exc.HTTPBadRequest, self.controller.update, + self.req, 'test.host.1', body=bad_body) + + def test_bad_update_key_and_correct_udpate_key(self): + bad_body = {'status': 'disable', 'crazy': 'bad'} + self.assertRaises(webob.exc.HTTPBadRequest, self.controller.update, + self.req, 'test.host.1', body=bad_body) + + def test_good_udpate_keys(self): + body = {'status': 'disable'} + self.assertRaises(NotImplementedError, self.controller.update, + self.req, 'test.host.1', body=body) + + def test_bad_host(self): + self.assertRaises(webob.exc.HTTPNotFound, self.controller.update, + self.req, 'bogus_host_name', body={'disabled': 0}) + + def test_show_forbidden(self): + self.req.environ['cinder.context'].is_admin = False + dest = 'dummydest' + self.assertRaises(webob.exc.HTTPForbidden, + self.controller.show, + self.req, dest) + self.req.environ['cinder.context'].is_admin = True + + def test_show_host_not_exist(self): + """A host given as an argument does not exists.""" + self.req.environ['cinder.context'].is_admin = True + dest = 'dummydest' + self.assertRaises(webob.exc.HTTPNotFound, + self.controller.show, + self.req, dest) + + +class HostSerializerTest(test.TestCase): + def setUp(self): + super(HostSerializerTest, self).setUp() + self.deserializer = os_hosts.HostDeserializer() + + def test_index_serializer(self): + serializer = os_hosts.HostIndexTemplate() + text = serializer.serialize(SERVICE_LIST) + + tree = etree.fromstring(text) + + self.assertEqual('hosts', tree.tag) + self.assertEqual(len(SERVICE_LIST), len(tree)) + for i in range(len(SERVICE_LIST)): + self.assertEqual('host', tree[i].tag) + self.assertEqual(SERVICE_LIST[i]['host'], + tree[i].get('host')) + self.assertEqual(SERVICE_LIST[i]['topic'], + tree[i].get('topic')) + + def test_update_serializer_with_status(self): + exemplar = dict(host='test.host.1', status='enabled') + serializer = os_hosts.HostUpdateTemplate() + text = serializer.serialize(exemplar) + + tree = etree.fromstring(text) + + self.assertEqual('host', tree.tag) + for key, value in exemplar.items(): + self.assertEqual(value, tree.get(key)) + + def test_update_deserializer(self): + exemplar = dict(status='enabled', foo='bar') + intext = ("\n" + 'enabledbar') + result = self.deserializer.deserialize(intext) + + self.assertEqual(dict(body=exemplar), result) diff --git a/cinder/tests/policy.json b/cinder/tests/policy.json index e12565822..3576076b8 100644 --- a/cinder/tests/policy.json +++ b/cinder/tests/policy.json @@ -36,5 +36,6 @@ "volume_extension:types_extra_specs": [], "volume_extension:extended_snapshot_attributes": [], "volume_extension:volume_host_attribute": [["rule:admin_api"]], - "volume_extension:volume_tenant_attribute": [["rule:admin_api"]] + "volume_extension:volume_tenant_attribute": [["rule:admin_api"]], + "volume_extension:hosts": [["rule:admin_api"]] } diff --git a/cinder/volume/api.py b/cinder/volume/api.py index faed8a35b..062900178 100644 --- a/cinder/volume/api.py +++ b/cinder/volume/api.py @@ -518,3 +518,25 @@ class API(base.Base): "image_name": recv_metadata.get('name', None) } return response + + +class HostAPI(base.Base): + def __init__(self): + super(HostAPI, self).__init__() + + """Sub-set of the Volume Manager API for managing host operations.""" + def set_host_enabled(self, context, host, enabled): + """Sets the specified host's ability to accept new volumes.""" + raise NotImplementedError() + + def get_host_uptime(self, context, host): + """Returns the result of calling "uptime" on the target host.""" + raise NotImplementedError() + + def host_power_action(self, context, host, action): + raise NotImplementedError() + + def set_host_maintenance(self, context, host, mode): + """Start/Stop host maintenance window. On start, it triggers + volume evacuation.""" + raise NotImplementedError() diff --git a/etc/cinder/policy.json b/etc/cinder/policy.json index a49931607..9b088780a 100644 --- a/etc/cinder/policy.json +++ b/etc/cinder/policy.json @@ -26,5 +26,6 @@ "volume_extension:snapshot_admin_actions:force_delete": [["rule:admin_api"]], "volume_extension:volume_host_attribute": [["rule:admin_api"]], - "volume_extension:volume_tenant_attribute": [["rule:admin_api"]] + "volume_extension:volume_tenant_attribute": [["rule:admin_api"]], + "volume_extension:hosts": [["rule:admin_api"]] } -- 2.45.2