From aa2e3d013d42244d3fbe7b770e4b167e37b9e6d3 Mon Sep 17 00:00:00 2001 From: Steven Hardy Date: Wed, 20 Feb 2013 15:34:48 +0000 Subject: [PATCH] heat engine : Add support rollback support for stack updates Adds support for rolling back failed stack updates blueprint update-rollback Change-Id: Ie90e37751365a8ef0ed24757c15f4020ee33ab8a --- heat/engine/parser.py | 57 +++++++++++--- heat/tests/test_parser.py | 151 +++++++++++++++++++++++++++++++++++++- 2 files changed, 196 insertions(+), 12 deletions(-) diff --git a/heat/engine/parser.py b/heat/engine/parser.py index 274d9ab5..e1df1a92 100644 --- a/heat/engine/parser.py +++ b/heat/engine/parser.py @@ -264,7 +264,7 @@ class Stack(object): if stack_status == self.CREATE_FAILED and not self.disable_rollback: self.delete(action=self.ROLLBACK) - def update(self, newstack): + def update(self, newstack, action=UPDATE): ''' Compare the current stack with newstack, and where necessary create/update/delete the resources until @@ -276,11 +276,29 @@ class Stack(object): Update will fail if it exceeds the specified timeout. The default is 60 minutes, set in the constructor ''' - if self.state not in (self.CREATE_COMPLETE, self.UPDATE_COMPLETE): - self.state_set(self.UPDATE_FAILED, 'State invalid for update') + if action not in (self.UPDATE, self.ROLLBACK): + logger.error("Unexpected action %s passed to update!" % action) + self.state_set(self.UPDATE_FAILED, "Invalid action %s" % action) return - else: + + if self.state not in (self.CREATE_COMPLETE, self.UPDATE_COMPLETE, + self.ROLLBACK_COMPLETE): + if (action == self.ROLLBACK and + self.state == self.UPDATE_IN_PROGRESS): + logger.debug("Starting update rollback for %s" % self.name) + else: + if action == self.UPDATE: + self.state_set(self.UPDATE_FAILED, + 'State invalid for update') + else: + self.state_set(self.ROLLBACK_FAILED, + 'State invalid for rollback') + return + + if action == self.UPDATE: self.state_set(self.UPDATE_IN_PROGRESS, 'Stack update started') + else: + self.state_set(self.ROLLBACK_IN_PROGRESS, 'Stack rollback started') # Now make the resources match the new stack definition with eventlet.Timeout(self.timeout_mins * 60) as tmo: @@ -369,7 +387,8 @@ class Stack(object): raise exception.ResourceUpdateFailed( resource_name=res.name) else: - logger.error("Failed to update %s" % res.name) + logger.error("Failed to %s %s" % + (action, res.name)) raise exception.ResourceUpdateFailed( resource_name=res.name) @@ -380,8 +399,12 @@ class Stack(object): self.outputs = self.resolve_static_data(template_outputs) self.store() - stack_status = self.UPDATE_COMPLETE - reason = 'Stack successfully updated' + if action == self.UPDATE: + stack_status = self.UPDATE_COMPLETE + reason = 'Stack successfully updated' + else: + stack_status = self.ROLLBACK_COMPLETE + reason = 'Stack rollback completed' except eventlet.Timeout as t: if t is tmo: @@ -391,10 +414,26 @@ class Stack(object): # not my timeout raise except exception.ResourceUpdateFailed as e: - stack_status = self.UPDATE_FAILED reason = str(e) or "Error : %s" % type(e) - self.state_set(stack_status, reason) + if action == self.UPDATE: + stack_status = self.UPDATE_FAILED + # If rollback is enabled, we do another update, with the + # existing template, so we roll back to the original state + # Note - ensure nothing after the "flip the template..." + # section above can raise ResourceUpdateFailed or this + # will not work ;) + if self.disable_rollback: + stack_status = self.UPDATE_FAILED + else: + oldstack = Stack(self.context, self.name, self.t, + self.parameters) + self.update(oldstack, action=self.ROLLBACK) + return + else: + stack_status = self.ROLLBACK_FAILED + + self.state_set(stack_status, reason) def delete(self, action=DELETE): ''' diff --git a/heat/tests/test_parser.py b/heat/tests/test_parser.py index 83b458cc..6162d881 100644 --- a/heat/tests/test_parser.py +++ b/heat/tests/test_parser.py @@ -515,7 +515,8 @@ class StackTest(unittest.TestCase): 'Properties': {'Foo': 'abc'}}}} self.stack = parser.Stack(self.ctx, 'update_test_stack', - template.Template(tmpl)) + template.Template(tmpl), + disable_rollback=True) self.stack.store() self.stack.create() self.assertEqual(self.stack.state, parser.Stack.CREATE_COMPLETE) @@ -547,7 +548,8 @@ class StackTest(unittest.TestCase): 'Properties': {'Foo': 'abc'}}}} self.stack = parser.Stack(self.ctx, 'update_test_stack', - template.Template(tmpl)) + template.Template(tmpl), + disable_rollback=True) self.stack.store() self.stack.create() self.assertEqual(self.stack.state, parser.Stack.CREATE_COMPLETE) @@ -585,7 +587,8 @@ class StackTest(unittest.TestCase): 'Properties': {'Foo': 'abc'}}}} self.stack = parser.Stack(self.ctx, 'update_test_stack', - template.Template(tmpl)) + template.Template(tmpl), + disable_rollback=True) self.stack.store() self.stack.create() self.assertEqual(self.stack.state, parser.Stack.CREATE_COMPLETE) @@ -610,3 +613,145 @@ class StackTest(unittest.TestCase): self.stack.update(updated_stack) self.assertEqual(self.stack.state, parser.Stack.UPDATE_FAILED) self.m.VerifyAll() + + @stack_delete_after + def test_update_rollback(self): + # patch in a dummy property schema for GenericResource + dummy_schema = {'Foo': {'Type': 'String'}} + resource.GenericResource.properties_schema = dummy_schema + + tmpl = {'Resources': {'AResource': {'Type': 'GenericResourceType', + 'Properties': {'Foo': 'abc'}}}} + + self.stack = parser.Stack(self.ctx, 'update_test_stack', + template.Template(tmpl)) + self.stack.store() + self.stack.create() + self.assertEqual(self.stack.state, parser.Stack.CREATE_COMPLETE) + + tmpl2 = {'Resources': {'AResource': {'Type': 'GenericResourceType', + 'Properties': {'Foo': 'xyz'}}}} + + updated_stack = parser.Stack(self.ctx, 'updated_stack', + template.Template(tmpl2)) + + # There will be two calls to handle_update, one for the new template + # then another (with the initial template) for rollback + self.m.StubOutWithMock(resource.GenericResource, 'handle_update') + resource.GenericResource.handle_update( + tmpl2['Resources']['AResource']).AndReturn( + resource.Resource.UPDATE_REPLACE) + resource.GenericResource.handle_update( + tmpl['Resources']['AResource']).AndReturn( + resource.Resource.UPDATE_REPLACE) + + # patch in a dummy handle_create making the replace fail when creating + # the replacement resource, but succeed the second call (rollback) + self.m.StubOutWithMock(resource.GenericResource, 'handle_create') + resource.GenericResource.handle_create().AndRaise(Exception) + resource.GenericResource.handle_create().AndReturn(None) + self.m.ReplayAll() + + self.stack.update(updated_stack) + self.assertEqual(self.stack.state, parser.Stack.ROLLBACK_COMPLETE) + self.assertEqual(self.stack['AResource'].properties['Foo'], 'abc') + self.m.VerifyAll() + + @stack_delete_after + def test_update_rollback_fail(self): + # patch in a dummy property schema for GenericResource + dummy_schema = {'Foo': {'Type': 'String'}} + resource.GenericResource.properties_schema = dummy_schema + + tmpl = {'Resources': {'AResource': {'Type': 'GenericResourceType', + 'Properties': {'Foo': 'abc'}}}} + + self.stack = parser.Stack(self.ctx, 'update_test_stack', + template.Template(tmpl)) + self.stack.store() + self.stack.create() + self.assertEqual(self.stack.state, parser.Stack.CREATE_COMPLETE) + + tmpl2 = {'Resources': {'AResource': {'Type': 'GenericResourceType', + 'Properties': {'Foo': 'xyz'}}}} + + updated_stack = parser.Stack(self.ctx, 'updated_stack', + template.Template(tmpl2)) + + # There will be two calls to handle_update, one for the new template + # then another (with the initial template) for rollback + self.m.StubOutWithMock(resource.GenericResource, 'handle_update') + resource.GenericResource.handle_update( + tmpl2['Resources']['AResource']).AndReturn( + resource.Resource.UPDATE_REPLACE) + resource.GenericResource.handle_update( + tmpl['Resources']['AResource']).AndReturn( + resource.Resource.UPDATE_REPLACE) + + # patch in a dummy handle_create making the replace fail when creating + # the replacement resource, and again on the second call (rollback) + self.m.StubOutWithMock(resource.GenericResource, 'handle_create') + resource.GenericResource.handle_create().AndRaise(Exception) + resource.GenericResource.handle_create().AndRaise(Exception) + self.m.ReplayAll() + + self.stack.update(updated_stack) + self.assertEqual(self.stack.state, parser.Stack.ROLLBACK_FAILED) + self.m.VerifyAll() + + @stack_delete_after + def test_update_rollback_add(self): + tmpl = {'Resources': {'AResource': {'Type': 'GenericResourceType'}}} + + self.stack = parser.Stack(self.ctx, 'update_test_stack', + template.Template(tmpl)) + self.stack.store() + self.stack.create() + self.assertEqual(self.stack.state, parser.Stack.CREATE_COMPLETE) + + tmpl2 = {'Resources': { + 'AResource': {'Type': 'GenericResourceType'}, + 'BResource': {'Type': 'GenericResourceType'}}} + + updated_stack = parser.Stack(self.ctx, 'updated_stack', + template.Template(tmpl2)) + + # patch in a dummy handle_create making the replace fail when creating + # the replacement resource, and succeed on the second call (rollback) + self.m.StubOutWithMock(resource.GenericResource, 'handle_create') + resource.GenericResource.handle_create().AndRaise(Exception) + self.m.ReplayAll() + + self.stack.update(updated_stack) + self.assertEqual(self.stack.state, parser.Stack.ROLLBACK_COMPLETE) + self.assertFalse('BResource' in self.stack) + self.m.VerifyAll() + + @stack_delete_after + def test_update_rollback_remove(self): + tmpl = {'Resources': { + 'AResource': {'Type': 'GenericResourceType'}, + 'BResource': {'Type': 'GenericResourceType'}}} + + self.stack = parser.Stack(self.ctx, 'update_test_stack', + template.Template(tmpl)) + self.stack.store() + self.stack.create() + self.assertEqual(self.stack.state, parser.Stack.CREATE_COMPLETE) + + tmpl2 = {'Resources': {'AResource': {'Type': 'GenericResourceType'}}} + + updated_stack = parser.Stack(self.ctx, 'updated_stack', + template.Template(tmpl2)) + + # patch in a dummy destroy making the delete fail + self.m.StubOutWithMock(resource.Resource, 'destroy') + resource.Resource.destroy().AndReturn('Error') + self.m.ReplayAll() + + self.stack.update(updated_stack) + self.assertEqual(self.stack.state, parser.Stack.ROLLBACK_COMPLETE) + self.assertTrue('BResource' in self.stack) + self.m.VerifyAll() + # Unset here so destroy() is not stubbed for stack.delete cleanup + self.m.UnsetStubs() -- 2.45.2