From 5d46ab040dbccf004adcebe67854a267e9240653 Mon Sep 17 00:00:00 2001 From: Steve Baker Date: Tue, 22 Jan 2013 12:10:15 +1300 Subject: [PATCH] A native Swift container resource type. Implements blueprint swift-resource-type The properties schema maps to the swift headers that are set during container creation. The schema allows any arbitrary X-Container-Meta-* header to be set in the template, which will be useful for users who store arbitrary header keys. This will be used during the presentation demo at LCA next week. Change-Id: I35e594cbe3cc251d5b48b41ea78ed2edda8aa807 --- heat/engine/resources/swift.py | 129 ++++++++++++++++ heat/tests/test_swift.py | 262 +++++++++++++++++++++++++++++++++ templates/Swift.template | 35 +++++ 3 files changed, 426 insertions(+) create mode 100644 heat/engine/resources/swift.py create mode 100644 heat/tests/test_swift.py create mode 100644 templates/Swift.template diff --git a/heat/engine/resources/swift.py b/heat/engine/resources/swift.py new file mode 100644 index 00000000..907b287e --- /dev/null +++ b/heat/engine/resources/swift.py @@ -0,0 +1,129 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import binascii +import itertools +import os +from urlparse import urlparse + +from heat.common import exception +from heat.engine import resource +from heat.openstack.common import log as logging +from heat.engine import clients + +logger = logging.getLogger(__name__) + + +class SwiftContainer(resource.Resource): + properties_schema = { + 'name': {'Type': 'String'}, + 'X-Container-Read': {'Type': 'String'}, + 'X-Container-Write': {'Type': 'String'}, + 'X-Container-Meta': {'Type': 'Map', 'Default': {}}, + 'DeletionPolicy': { + 'Type': 'String', + 'AllowedValues': ['Delete', 'Retain']}} + + def __init__(self, name, json_snippet, stack): + super(SwiftContainer, self).__init__(name, json_snippet, stack) + + def validate(self): + ''' + Validate any of the provided params + ''' + #check if swiftclient is installed + if clients.swiftclient is None: + return {'Error': + 'SwiftContainer unavailable due to missing swiftclient.'} + + @staticmethod + def _create_container_name(resource_name, name=None): + return name or 'heat-%s-%s' % (resource_name, + binascii.hexlify(os.urandom(10))) + + @staticmethod + def _build_meta_headers(meta_props): + ''' + Returns a new dict where each key is prepended with: + X-Container-Meta- + ''' + if meta_props is None: + return {} + return dict( + ('X-Container-Meta-' + k, v) for (k, v) in meta_props.items()) + + def handle_create(self): + """Create a container.""" + container = SwiftContainer._create_container_name( + self.physical_resource_name(), self.properties['name']) + headers = SwiftContainer._build_meta_headers( + self.properties['X-Container-Meta']) + if 'X-Container-Read' in self.properties.keys(): + headers['X-Container-Read'] = self.properties['X-Container-Read'] + if 'X-Container-Write' in self.properties.keys(): + headers['X-Container-Write'] = self.properties['X-Container-Write'] + logger.debug('SwiftContainer create container %s with headers %s' % + (container, headers)) + + self.swift().put_container(container, headers) + self.resource_id_set(container) + + def handle_update(self): + return self.UPDATE_REPLACE + + def handle_delete(self): + """Perform specified delete policy""" + if self.properties['DeletionPolicy'] == 'Retain': + return + logger.debug('SwiftContainer delete container %s' % self.resource_id) + if self.resource_id is not None: + try: + self.swift().delete_container(self.resource_id) + except clients.swiftclient.ClientException as ex: + logger.warn("Delete container failed: %s" % str(ex)) + + def FnGetRefId(self): + return unicode(self.resource_id) + + def FnGetAtt(self, key): + url, token_id = self.swift().get_auth() + if self.resource_id: + headers = self.swift().head_container(self.resource_id) + parsed = list(urlparse(url)) + if key == 'DomainName': + return parsed[1].split(':')[0] + elif key == 'WebsiteURL': + return '%s://%s%s/%s' % (parsed[0], parsed[1], parsed[2], + self.resource_id) + elif key == 'RootURL': + return '%s://%s%s' % (parsed[0], parsed[1], parsed[2]) + elif key == 'ObjectCount': + return headers['x-container-object-count'] + elif key == 'BytesUsed': + return headers['x-container-bytes-used'] + elif key == 'HeadContainer': + return headers + else: + raise exception.InvalidTemplateAttribute(resource=self.name, + key=key) + + +def resource_mapping(): + if clients.swiftclient is None: + return {} + + return { + 'OS::Swift::Container': SwiftContainer, + } diff --git a/heat/tests/test_swift.py b/heat/tests/test_swift.py new file mode 100644 index 00000000..24f9be3f --- /dev/null +++ b/heat/tests/test_swift.py @@ -0,0 +1,262 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + + +import os +import re + +import unittest +import mox + +from nose.plugins.attrib import attr + +from heat.common import context +from heat.common import template_format +from heat.openstack.common.importutils import try_import +from heat.engine.resources import swift +from heat.engine import parser +from heat.tests.utils import skip_if + +swiftclient = try_import('swiftclient.client') + + +@attr(tag=['unit', 'resource']) +@attr(speed='fast') +class swiftTest(unittest.TestCase): + @skip_if(swiftclient is None, 'unable to import swiftclient') + def setUp(self): + self.m = mox.Mox() + self.m.CreateMock(swiftclient.Connection) + self.m.StubOutWithMock(swiftclient.Connection, 'put_container') + self.m.StubOutWithMock(swiftclient.Connection, 'delete_container') + self.m.StubOutWithMock(swiftclient.Connection, 'head_container') + self.m.StubOutWithMock(swiftclient.Connection, 'get_auth') + + self.container_pattern = 'heat-test_stack.test_resource-[0-9a-f]+' + + def tearDown(self): + self.m.UnsetStubs() + print "swiftTest teardown complete" + + def load_template(self): + self.path = os.path.dirname(os.path.realpath(__file__)).\ + replace('heat/tests', 'templates') + f = open("%s/Swift.template" % self.path) + t = template_format.parse(f.read()) + f.close() + return t + + def parse_stack(self, t): + ctx = context.RequestContext.from_dict({ + 'tenant': 'test_tenant', + 'username': 'test_username', + 'password': 'password', + 'auth_url': 'http://localhost:5000/v2.0'}) + stack = parser.Stack(ctx, 'test_stack', parser.Template(t)) + + return stack + + def create_resource(self, t, stack, resource_name): + resource = swift.SwiftContainer( + 'test_resource', + t['Resources'][resource_name], + stack) + self.assertEqual(None, resource.create()) + self.assertEqual(swift.SwiftContainer.CREATE_COMPLETE, resource.state) + return resource + + @skip_if(swiftclient is None, 'unable to import swiftclient') + def test_create_container_name(self): + self.m.UnsetStubs() + self.assertTrue(re.match(self.container_pattern, + swift.SwiftContainer._create_container_name( + 'test_stack.test_resource'))) + self.assertEqual( + 'the_name', + swift.SwiftContainer._create_container_name( + 'test_stack.test_resource', + 'the_name')) + + @skip_if(swiftclient is None, 'unable to import swiftclient') + def test_build_meta_headers(self): + self.m.UnsetStubs() + self.assertEqual({}, swift.SwiftContainer._build_meta_headers({})) + self.assertEqual({}, swift.SwiftContainer._build_meta_headers(None)) + meta = { + 'X-Container-Meta-Web-Index': 'index.html', + 'X-Container-Meta-Web-Error': 'error.html' + } + self.assertEqual(meta, swift.SwiftContainer._build_meta_headers({ + "Web-Index": "index.html", + "Web-Error": "error.html" + })) + + @skip_if(swiftclient is None, 'unable to import swiftclient') + def test_attributes(self): + headers = { + "content-length": "0", + "x-container-object-count": "82", + "x-container-write": "None", + "accept-ranges": "bytes", + "x-trans-id": "tx08ea48ef2fa24e6da3d2f5c188fd938b", + "date": "Wed, 23 Jan 2013 22:48:05 GMT", + "x-timestamp": "1358980499.84298", + "x-container-read": ".r:*", + "x-container-bytes-used": "17680980", + "content-type": "text/plain; charset=utf-8"} + + swiftclient.Connection.put_container( + mox.Regex(self.container_pattern), + {'X-Container-Write': None, + 'X-Container-Read': None} + ).AndReturn(None) + swiftclient.Connection.get_auth().MultipleTimes().AndReturn( + ('http://localhost:8080/v_2', None)) + swiftclient.Connection.head_container( + mox.IgnoreArg()).MultipleTimes().AndReturn(headers) + swiftclient.Connection.delete_container( + mox.Regex(self.container_pattern)).AndReturn(None) + + self.m.ReplayAll() + t = self.load_template() + stack = self.parse_stack(t) + resource = self.create_resource(t, stack, 'SwiftContainer') + + ref_id = resource.FnGetRefId() + self.assertTrue(re.match(self.container_pattern, + ref_id)) + + self.assertEqual('localhost', resource.FnGetAtt('DomainName')) + url = 'http://localhost:8080/v_2/%s' % ref_id + + self.assertEqual(url, resource.FnGetAtt('WebsiteURL')) + self.assertEqual('82', resource.FnGetAtt('ObjectCount')) + self.assertEqual('17680980', resource.FnGetAtt('BytesUsed')) + self.assertEqual(headers, resource.FnGetAtt('HeadContainer')) + + try: + resource.FnGetAtt('Foo') + raise Exception('Expected InvalidTemplateAttribute') + except swift.exception.InvalidTemplateAttribute: + pass + + self.assertEqual(swift.SwiftContainer.UPDATE_REPLACE, + resource.handle_update()) + + resource.delete() + self.m.VerifyAll() + + @skip_if(swiftclient is None, 'unable to import swiftclient') + def test_public_read(self): + swiftclient.Connection.put_container( + mox.Regex(self.container_pattern), + {'X-Container-Write': None, + 'X-Container-Read': '.r:*'}).AndReturn(None) + swiftclient.Connection.delete_container( + mox.Regex(self.container_pattern)).AndReturn(None) + + self.m.ReplayAll() + t = self.load_template() + properties = t['Resources']['SwiftContainer']['Properties'] + properties['X-Container-Read'] = '.r:*' + stack = self.parse_stack(t) + resource = self.create_resource(t, stack, 'SwiftContainer') + resource.delete() + self.m.VerifyAll() + + @skip_if(swiftclient is None, 'unable to import swiftclient') + def test_public_read_write(self): + swiftclient.Connection.put_container( + mox.Regex(self.container_pattern), + {'X-Container-Write': '.r:*', + 'X-Container-Read': '.r:*'}).AndReturn(None) + swiftclient.Connection.delete_container( + mox.Regex(self.container_pattern)).AndReturn(None) + + self.m.ReplayAll() + t = self.load_template() + properties = t['Resources']['SwiftContainer']['Properties'] + properties['X-Container-Read'] = '.r:*' + properties['X-Container-Write'] = '.r:*' + stack = self.parse_stack(t) + resource = self.create_resource(t, stack, 'SwiftContainer') + resource.delete() + self.m.VerifyAll() + + @skip_if(swiftclient is None, 'unable to import swiftclient') + def test_website(self): + + swiftclient.Connection.put_container( + mox.Regex(self.container_pattern), + {'X-Container-Meta-Web-Error': 'error.html', + 'X-Container-Meta-Web-Index': 'index.html', + 'X-Container-Write': None, + 'X-Container-Read': '.r:*'}).AndReturn(None) + swiftclient.Connection.delete_container( + mox.Regex(self.container_pattern)).AndReturn(None) + + self.m.ReplayAll() + t = self.load_template() + stack = self.parse_stack(t) + resource = self.create_resource(t, stack, 'SwiftContainerWebsite') + resource.delete() + self.m.VerifyAll() + + @skip_if(swiftclient is None, 'unable to import swiftclient') + def test_delete_exception(self): + + swiftclient.Connection.put_container( + mox.Regex(self.container_pattern), + {'X-Container-Write': None, + 'X-Container-Read': None}).AndReturn(None) + swiftclient.Connection.delete_container( + mox.Regex(self.container_pattern)).AndRaise( + swiftclient.ClientException('Test delete failure')) + + self.m.ReplayAll() + t = self.load_template() + stack = self.parse_stack(t) + resource = self.create_resource(t, stack, 'SwiftContainer') + resource.delete() + + self.m.VerifyAll() + + @skip_if(swiftclient is None, 'unable to import swiftclient') + def test_delete_retain(self): + + # first run, with retain policy + swiftclient.Connection.put_container( + mox.Regex(self.container_pattern), + {'X-Container-Write': None, + 'X-Container-Read': None}).AndReturn(None) + # This should not be called + swiftclient.Connection.delete_container( + mox.Regex(self.container_pattern)).AndReturn(None) + + self.m.ReplayAll() + t = self.load_template() + + properties = t['Resources']['SwiftContainer']['Properties'] + properties['DeletionPolicy'] = 'Retain' + stack = self.parse_stack(t) + resource = self.create_resource(t, stack, 'SwiftContainer') + # if delete_container is called, mox verify will succeed + resource.delete() + + try: + self.m.VerifyAll() + except mox.ExpectedMethodCallsError: + return + + raise Exception('delete_container was called despite Retain policy') diff --git a/templates/Swift.template b/templates/Swift.template new file mode 100644 index 00000000..2ec17aed --- /dev/null +++ b/templates/Swift.template @@ -0,0 +1,35 @@ +{ + "AWSTemplateFormatVersion" : "2010-09-09", + + "Description" : "Template to test OS::Swift::Container resources", + + "Resources" : { + "SwiftContainerWebsite" : { + "Type" : "OS::Swift::Container", + "Properties" : { + "X-Container-Read" : ".r:*", + "X-Container-Meta" : { + "Web-Index" : "index.html", + "Web-Error" : "error.html" + }, + "DeletionPolicy" : "Delete" + } + }, + "SwiftContainer" : { + "Type" : "OS::Swift::Container", + "Properties" : { + } + } + }, + + "Outputs" : { + "WebsiteURL" : { + "Value" : { "Fn::GetAtt" : [ "SwiftContainer", "WebsiteURL" ] }, + "Description" : "URL for website hosted on S3" + }, + "DomainName" : { + "Value" : { "Fn::GetAtt" : [ "SwiftContainer", "DomainName" ] }, + "Description" : "Domain of Swift host" + } + } +} -- 2.45.2