From: Steven Hardy Date: Wed, 20 Feb 2013 10:24:06 +0000 (+0000) Subject: heat engine : fail update immediately on resource update failure X-Git-Tag: 2014.1~880^2 X-Git-Url: https://review.fuel-infra.org/gitweb?a=commitdiff_plain;h=24a47ad695c5913d558f00ce050ba3543592d980;p=openstack-build%2Fheat-build.git heat engine : fail update immediately on resource update failure When doing an update, we want to fail immediately, instead of collecting the resource update errors and continuing. This is particularly important for update rollback, where we want to declare failure ASAP to simplify and speed up the rollback blueprint update-rollback Change-Id: I56133254036d8aac592dcf3cda2ca7928cc02fa9 --- diff --git a/heat/engine/parser.py b/heat/engine/parser.py index 963e15fc..b2e3af45 100644 --- a/heat/engine/parser.py +++ b/heat/engine/parser.py @@ -283,7 +283,6 @@ class Stack(object): self.state_set(self.UPDATE_IN_PROGRESS, 'Stack update started') # Now make the resources match the new stack definition - failures = [] with eventlet.Timeout(self.timeout_mins * 60) as tmo: try: # First delete any resources which are not in newstack @@ -293,8 +292,10 @@ class Stack(object): % res.name + " definition, deleting") result = res.destroy() if result: - failures.append('Resource %s delete failed' - % res.name) + logger.error("Failed to remove %s : %s" % + (res.name, result)) + raise exception.ResourceUpdateFailed( + resource_name=res.name) else: del self.resources[res.name] @@ -307,8 +308,10 @@ class Stack(object): self[res.name] = res result = self[res.name].create() if result: - failures.append('Resource %s create failed' - % res.name) + logger.error("Failed to add %s : %s" % + (res.name, result)) + raise exception.ResourceUpdateFailed( + resource_name=res.name) # Now (the hard part :) update existing resources # The Resource base class allows equality-test of resources, @@ -331,8 +334,8 @@ class Stack(object): # of the existing stack (which is the stack being updated) old_snippet = self.resolve_runtime_data(self[res.name].t) new_snippet = self.resolve_runtime_data(res.t) - if old_snippet != new_snippet: + if old_snippet != new_snippet: # Can fail if underlying resource class does not # implement update logic or update requires replacement retval = self[res.name].update(new_snippet) @@ -346,37 +349,35 @@ class Stack(object): # Resource requires replacement for update result = self[res.name].destroy() if result: - failures.append('Resource %s delete failed' - % res.name) + logger.error("Failed to delete %s : %s" % + (res.name, result)) + raise exception.ResourceUpdateFailed( + resource_name=res.name) else: res.stack = self self[res.name] = res result = self[res.name].create() if result: - failures.append('Resource %s create failed' - % res.name) + logger.error("Failed to create %s : %s" % + (res.name, result)) + raise exception.ResourceUpdateFailed( + resource_name=res.name) else: - logger.warning("Cannot update resource %s," % - res.name + " reason %s" % retval) - failures.append('Resource %s update failed' - % res.name) - - # Set stack status values - if not failures: - # flip the template & parameters to the newstack values - self.t = newstack.t - self.parameters = newstack.parameters - template_outputs = self.t[template.OUTPUTS] - self.outputs = self.resolve_static_data(template_outputs) - self.dependencies = self._get_dependencies( - self.resources.itervalues()) - self.store() - - stack_status = self.UPDATE_COMPLETE - reason = 'Stack successfully updated' - else: - stack_status = self.UPDATE_FAILED - reason = ",".join(failures) + logger.error("Failed to update %s" % res.name) + raise exception.ResourceUpdateFailed( + resource_name=res.name) + + # flip the template & parameters to the newstack values + self.t = newstack.t + self.parameters = newstack.parameters + template_outputs = self.t[template.OUTPUTS] + self.outputs = self.resolve_static_data(template_outputs) + self.dependencies = self._get_dependencies( + self.resources.itervalues()) + self.store() + + stack_status = self.UPDATE_COMPLETE + reason = 'Stack successfully updated' except eventlet.Timeout as t: if t is tmo: @@ -385,6 +386,9 @@ class Stack(object): else: # not my timeout raise + except exception.ResourceUpdateFailed as e: + stack_status = self.UPDATE_FAILED + reason = str(e) or "Error : %s" % type(e) self.state_set(stack_status, reason) diff --git a/heat/tests/test_parser.py b/heat/tests/test_parser.py index b0a899b0..83b458cc 100644 --- a/heat/tests/test_parser.py +++ b/heat/tests/test_parser.py @@ -20,10 +20,10 @@ import mox from heat.common import context from heat.common import exception from heat.common import template_format +from heat.engine import resource from heat.engine import parser from heat.engine import parameters from heat.engine import template -from heat.engine.resource import Resource from heat.tests.utils import stack_delete_after import heat.db as db_api @@ -188,8 +188,8 @@ class TemplateTest(unittest.TestCase): snippet, params) def test_resource_refs(self): - resources = {'foo': self.m.CreateMock(Resource), - 'blarg': self.m.CreateMock(Resource)} + resources = {'foo': self.m.CreateMock(resource.Resource), + 'blarg': self.m.CreateMock(resource.Resource)} resources['foo'].FnGetRefId().AndReturn('bar') self.m.ReplayAll() @@ -404,3 +404,209 @@ class StackTest(unittest.TestCase): db_s = db_api.stack_get(self.ctx, stack_id) self.assertNotEqual(db_s, None) self.assertEqual(self.stack.state, self.stack.DELETE_FAILED) + + @stack_delete_after + def test_update_badstate(self): + self.stack = parser.Stack(self.ctx, 'test_stack', parser.Template({}), + state=parser.Stack.CREATE_FAILED) + stack_id = self.stack.store() + self.assertEqual(self.stack.state, parser.Stack.CREATE_FAILED) + self.stack.update({}) + self.assertEqual(self.stack.state, parser.Stack.UPDATE_FAILED) + + @stack_delete_after + def test_update_add(self): + tmpl = {'Resources': {'AResource': {'Type': 'GenericResourceType'}}} + + self.stack = parser.Stack(self.ctx, 'update_test_stack', + template.Template(tmpl)) + self.stack.store() + self.stack.create() + self.assertEqual(self.stack.state, parser.Stack.CREATE_COMPLETE) + + tmpl2 = {'Resources': { + 'AResource': {'Type': 'GenericResourceType'}, + 'BResource': {'Type': 'GenericResourceType'}}} + updated_stack = parser.Stack(self.ctx, 'updated_stack', + template.Template(tmpl2)) + self.stack.update(updated_stack) + self.assertEqual(self.stack.state, parser.Stack.UPDATE_COMPLETE) + self.assertTrue('BResource' in self.stack) + + @stack_delete_after + def test_update_remove(self): + tmpl = {'Resources': { + 'AResource': {'Type': 'GenericResourceType'}, + 'BResource': {'Type': 'GenericResourceType'}}} + + self.stack = parser.Stack(self.ctx, 'update_test_stack', + template.Template(tmpl)) + self.stack.store() + self.stack.create() + self.assertEqual(self.stack.state, parser.Stack.CREATE_COMPLETE) + + tmpl2 = {'Resources': {'AResource': {'Type': 'GenericResourceType'}}} + + updated_stack = parser.Stack(self.ctx, 'updated_stack', + template.Template(tmpl2)) + self.stack.update(updated_stack) + self.assertEqual(self.stack.state, parser.Stack.UPDATE_COMPLETE) + self.assertFalse('BResource' in self.stack) + + @stack_delete_after + def test_update_description(self): + tmpl = {'Description': 'ATemplate', + 'Resources': {'AResource': {'Type': 'GenericResourceType'}}} + + self.stack = parser.Stack(self.ctx, 'update_test_stack', + template.Template(tmpl)) + self.stack.store() + self.stack.create() + self.assertEqual(self.stack.state, parser.Stack.CREATE_COMPLETE) + + tmpl2 = {'Description': 'BTemplate', + 'Resources': {'AResource': {'Type': 'GenericResourceType'}}} + + updated_stack = parser.Stack(self.ctx, 'updated_stack', + template.Template(tmpl2)) + self.stack.update(updated_stack) + self.assertEqual(self.stack.state, parser.Stack.UPDATE_COMPLETE) + self.assertEqual(self.stack.t[template.DESCRIPTION], 'BTemplate') + + @stack_delete_after + def test_update_modify_ok_replace(self): + # patch in a dummy property schema for GenericResource + dummy_schema = {'Foo': {'Type': 'String'}} + resource.GenericResource.properties_schema = dummy_schema + + tmpl = {'Resources': {'AResource': {'Type': 'GenericResourceType', + 'Properties': {'Foo': 'abc'}}}} + + self.stack = parser.Stack(self.ctx, 'update_test_stack', + template.Template(tmpl)) + self.stack.store() + self.stack.create() + self.assertEqual(self.stack.state, parser.Stack.CREATE_COMPLETE) + + tmpl2 = {'Resources': {'AResource': {'Type': 'GenericResourceType', + 'Properties': {'Foo': 'xyz'}}}} + + updated_stack = parser.Stack(self.ctx, 'updated_stack', + template.Template(tmpl2)) + # patch in a dummy handle_update + self.m.StubOutWithMock(resource.GenericResource, 'handle_update') + resource.GenericResource.handle_update( + tmpl2['Resources']['AResource']).AndReturn( + resource.Resource.UPDATE_REPLACE) + self.m.ReplayAll() + + self.stack.update(updated_stack) + self.assertEqual(self.stack.state, parser.Stack.UPDATE_COMPLETE) + self.assertEqual(self.stack['AResource'].properties['Foo'], 'xyz') + self.m.VerifyAll() + + @stack_delete_after + def test_update_modify_update_failed(self): + # patch in a dummy property schema for GenericResource + dummy_schema = {'Foo': {'Type': 'String'}} + resource.GenericResource.properties_schema = dummy_schema + + tmpl = {'Resources': {'AResource': {'Type': 'GenericResourceType', + 'Properties': {'Foo': 'abc'}}}} + + self.stack = parser.Stack(self.ctx, 'update_test_stack', + template.Template(tmpl)) + self.stack.store() + self.stack.create() + self.assertEqual(self.stack.state, parser.Stack.CREATE_COMPLETE) + + tmpl2 = {'Resources': {'AResource': {'Type': 'GenericResourceType', + 'Properties': {'Foo': 'xyz'}}}} + + updated_stack = parser.Stack(self.ctx, 'updated_stack', + template.Template(tmpl2)) + + # patch in a dummy handle_update + self.m.StubOutWithMock(resource.GenericResource, 'handle_update') + resource.GenericResource.handle_update( + tmpl2['Resources']['AResource']).AndReturn( + resource.Resource.UPDATE_FAILED) + self.m.ReplayAll() + + self.stack.update(updated_stack) + self.assertEqual(self.stack.state, parser.Stack.UPDATE_FAILED) + self.m.VerifyAll() + + @stack_delete_after + def test_update_modify_replace_failed_delete(self): + # patch in a dummy property schema for GenericResource + dummy_schema = {'Foo': {'Type': 'String'}} + resource.GenericResource.properties_schema = dummy_schema + + tmpl = {'Resources': {'AResource': {'Type': 'GenericResourceType', + 'Properties': {'Foo': 'abc'}}}} + + self.stack = parser.Stack(self.ctx, 'update_test_stack', + template.Template(tmpl)) + self.stack.store() + self.stack.create() + self.assertEqual(self.stack.state, parser.Stack.CREATE_COMPLETE) + + tmpl2 = {'Resources': {'AResource': {'Type': 'GenericResourceType', + 'Properties': {'Foo': 'xyz'}}}} + + updated_stack = parser.Stack(self.ctx, 'updated_stack', + template.Template(tmpl2)) + + # patch in a dummy handle_update + self.m.StubOutWithMock(resource.GenericResource, 'handle_update') + resource.GenericResource.handle_update( + tmpl2['Resources']['AResource']).AndReturn( + resource.Resource.UPDATE_REPLACE) + + # make the update fail deleting the existing resource + self.m.StubOutWithMock(resource.Resource, 'destroy') + resource.Resource.destroy().AndReturn("Error") + self.m.ReplayAll() + + self.stack.update(updated_stack) + self.assertEqual(self.stack.state, parser.Stack.UPDATE_FAILED) + self.m.VerifyAll() + # Unset here so destroy() is not stubbed for stack.delete cleanup + self.m.UnsetStubs() + + @stack_delete_after + def test_update_modify_replace_failed_create(self): + # patch in a dummy property schema for GenericResource + dummy_schema = {'Foo': {'Type': 'String'}} + resource.GenericResource.properties_schema = dummy_schema + + tmpl = {'Resources': {'AResource': {'Type': 'GenericResourceType', + 'Properties': {'Foo': 'abc'}}}} + + self.stack = parser.Stack(self.ctx, 'update_test_stack', + template.Template(tmpl)) + self.stack.store() + self.stack.create() + self.assertEqual(self.stack.state, parser.Stack.CREATE_COMPLETE) + + tmpl2 = {'Resources': {'AResource': {'Type': 'GenericResourceType', + 'Properties': {'Foo': 'xyz'}}}} + + updated_stack = parser.Stack(self.ctx, 'updated_stack', + template.Template(tmpl2)) + + # patch in a dummy handle_update + self.m.StubOutWithMock(resource.GenericResource, 'handle_update') + resource.GenericResource.handle_update( + tmpl2['Resources']['AResource']).AndReturn( + resource.Resource.UPDATE_REPLACE) + + # patch in a dummy handle_create making the replace fail creating + self.m.StubOutWithMock(resource.GenericResource, 'handle_create') + resource.GenericResource.handle_create().AndRaise(Exception) + self.m.ReplayAll() + + self.stack.update(updated_stack) + self.assertEqual(self.stack.state, parser.Stack.UPDATE_FAILED) + self.m.VerifyAll()