From 46ae6848896a24dece79771037b86cc6f4b53292 Mon Sep 17 00:00:00 2001 From: Zane Bitter Date: Tue, 27 Aug 2013 20:15:08 +0200 Subject: [PATCH] Implement interruption-free update and rollback During an update, where a resource must be replaced in its entirety, create the replacement resource before deleting the old resource. Also, allow rollback to the previous version of the resource without replacing it, where that is possible. Fixes bug #1176142 Change-Id: Id89654bad297815bdbcc86f666367772889b5df4 --- heat/engine/resource.py | 1 + heat/engine/update.py | 72 ++++++++++++++++++++++++++++++++++----- heat/tests/test_parser.py | 54 +++++++++++++++++++++-------- 3 files changed, 104 insertions(+), 23 deletions(-) diff --git a/heat/engine/resource.py b/heat/engine/resource.py index 302693ff..cd82d88a 100644 --- a/heat/engine/resource.py +++ b/heat/engine/resource.py @@ -602,6 +602,7 @@ class Resource(object): rs.update_and_save({'action': self.action, 'status': self.status, 'status_reason': reason, + 'stack_id': self.stack.id, 'nova_instance': self.resource_id}) self.stack.updated_time = datetime.utcnow() diff --git a/heat/engine/update.py b/heat/engine/update.py index 9ccbef5d..2b1ac7b2 100644 --- a/heat/engine/update.py +++ b/heat/engine/update.py @@ -13,6 +13,8 @@ # License for the specific language governing permissions and limitations # under the License. +from heat.db import api as db_api + from heat.engine import resource from heat.engine import scheduler @@ -51,6 +53,10 @@ class StackUpdate(object): existing_deps = self.existing_stack.dependencies new_deps = self.new_stack.dependencies + cleanup_prev = scheduler.DependencyTaskGroup( + self.previous_stack.dependencies, + self._remove_backup_resource, + reverse=True) cleanup = scheduler.DependencyTaskGroup(existing_deps, self._remove_old_resource, reverse=True) @@ -59,13 +65,32 @@ class StackUpdate(object): update = scheduler.DependencyTaskGroup(new_deps, self._update_resource) - yield cleanup() + if not self.rollback: + yield cleanup_prev() + yield create_new() - yield update() + try: + yield update() + finally: + prev_deps = self.previous_stack._get_dependencies( + self.previous_stack.resources.itervalues()) + self.previous_stack.dependencies = prev_deps + yield cleanup() + + @scheduler.wrappertask + def _remove_backup_resource(self, prev_res): + if prev_res.state not in ((prev_res.INIT, prev_res.COMPLETE), + (prev_res.DELETE, prev_res.COMPLETE)): + logger.debug("Deleting backup resource %s" % prev_res.name) + yield prev_res.destroy() @scheduler.wrappertask def _remove_old_resource(self, existing_res): res_name = existing_res.name + + if res_name in self.previous_stack: + yield self._remove_backup_resource(self.previous_stack[res_name]) + if res_name not in self.new_stack: logger.debug("resource %s not found in updated stack" % res_name + " definition, deleting") @@ -78,14 +103,45 @@ class StackUpdate(object): if res_name not in self.existing_stack: logger.debug("resource %s not found in current stack" % res_name + " definition, adding") - new_res.stack = self.existing_stack - self.existing_stack[res_name] = new_res - yield new_res.create() + yield self._create_resource(new_res) + + @staticmethod + def _exchange_stacks(existing_res, prev_res): + db_api.resource_exchange_stacks(existing_res.stack.context, + existing_res.id, prev_res.id) + existing_res.stack, prev_res.stack = prev_res.stack, existing_res.stack + existing_res.stack[existing_res.name] = existing_res + prev_res.stack[prev_res.name] = prev_res @scheduler.wrappertask - def _replace_resource(self, new_res): + def _create_resource(self, new_res): res_name = new_res.name - yield self.existing_stack[res_name].destroy() + + # Clean up previous resource + if res_name in self.previous_stack: + prev_res = self.previous_stack[res_name] + + if prev_res.state not in ((prev_res.INIT, prev_res.COMPLETE), + (prev_res.DELETE, prev_res.COMPLETE)): + # Swap in the backup resource if it is in a valid state, + # instead of creating a new resource + if prev_res.status == prev_res.COMPLETE: + logger.debug("Swapping in backup Resource %s" % res_name) + self._exchange_stacks(self.existing_stack[res_name], + prev_res) + return + + logger.debug("Deleting backup Resource %s" % res_name) + yield prev_res.destroy() + + # Back up existing resource + if res_name in self.existing_stack: + logger.debug("Backing up existing Resource %s" % res_name) + existing_res = self.existing_stack[res_name] + existing_res.stack = self.previous_stack + self.previous_stack[res_name] = existing_res + existing_res.state_set(existing_res.UPDATE, existing_res.COMPLETE) + new_res.stack = self.existing_stack self.existing_stack[res_name] = new_res yield new_res.create() @@ -108,7 +164,7 @@ class StackUpdate(object): yield self.existing_stack[res_name].update(new_snippet, existing_snippet) except resource.UpdateReplace: - yield self._replace_resource(new_res) + yield self._create_resource(new_res) else: logger.info("Resource %s for stack %s updated" % (res_name, self.existing_stack.name)) diff --git a/heat/tests/test_parser.py b/heat/tests/test_parser.py index d0419d1d..ac2a71fa 100644 --- a/heat/tests/test_parser.py +++ b/heat/tests/test_parser.py @@ -1179,10 +1179,9 @@ class StackTest(HeatTestCase): # key/property in update_allowed_keys/update_allowed_properties # patch in a dummy handle_create making the replace fail when creating - # the replacement rsrc, but succeed the second call (rollback) + # the replacement rsrc self.m.StubOutWithMock(generic_rsrc.ResourceWithProps, 'handle_create') generic_rsrc.ResourceWithProps.handle_create().AndRaise(Exception) - generic_rsrc.ResourceWithProps.handle_create().AndReturn(None) self.m.ReplayAll() self.stack.update(updated_stack) @@ -1217,8 +1216,9 @@ class StackTest(HeatTestCase): # patch in a dummy handle_create making the replace fail when creating # the replacement rsrc, and again on the second call (rollback) self.m.StubOutWithMock(generic_rsrc.ResourceWithProps, 'handle_create') + self.m.StubOutWithMock(generic_rsrc.ResourceWithProps, 'handle_delete') generic_rsrc.ResourceWithProps.handle_create().AndRaise(Exception) - generic_rsrc.ResourceWithProps.handle_create().AndRaise(Exception) + generic_rsrc.ResourceWithProps.handle_delete().AndRaise(Exception) self.m.ReplayAll() self.stack.update(updated_stack) @@ -1289,6 +1289,40 @@ class StackTest(HeatTestCase): # Unset here so delete() is not stubbed for stack.delete cleanup self.m.UnsetStubs() + @utils.stack_delete_after + def test_update_rollback_replace(self): + tmpl = {'Resources': { + 'AResource': {'Type': 'ResourceWithPropsType', + 'Properties': {'Foo': 'foo'}}}} + + self.stack = parser.Stack(self.ctx, 'update_test_stack', + template.Template(tmpl), + disable_rollback=False) + self.stack.store() + self.stack.create() + self.assertEqual(self.stack.state, + (parser.Stack.CREATE, parser.Stack.COMPLETE)) + + tmpl2 = {'Resources': {'AResource': {'Type': 'ResourceWithPropsType', + 'Properties': {'Foo': 'bar'}}}} + + updated_stack = parser.Stack(self.ctx, 'updated_stack', + template.Template(tmpl2)) + + # patch in a dummy delete making the destroy fail + self.m.StubOutWithMock(generic_rsrc.ResourceWithProps, 'handle_delete') + generic_rsrc.ResourceWithProps.handle_delete().AndRaise(Exception) + generic_rsrc.ResourceWithProps.handle_delete().AndReturn(None) + generic_rsrc.ResourceWithProps.handle_delete().AndReturn(None) + self.m.ReplayAll() + + self.stack.update(updated_stack) + self.assertEqual(self.stack.state, + (parser.Stack.ROLLBACK, parser.Stack.COMPLETE)) + self.m.VerifyAll() + # Unset here so delete() is not stubbed for stack.delete cleanup + self.m.UnsetStubs() + @utils.stack_delete_after def test_update_replace_by_reference(self): ''' @@ -1386,16 +1420,12 @@ class StackTest(HeatTestCase): # resource.UpdateReplace because we've not specified the modified # key/property in update_allowed_keys/update_allowed_properties - generic_rsrc.ResourceWithProps.FnGetRefId().AndReturn( + generic_rsrc.ResourceWithProps.FnGetRefId().MultipleTimes().AndReturn( 'AResource') # mock to make the replace fail when creating the replacement resource generic_rsrc.ResourceWithProps.handle_create().AndRaise(Exception) - generic_rsrc.ResourceWithProps.handle_create().AndReturn(None) - generic_rsrc.ResourceWithProps.FnGetRefId().MultipleTimes().AndReturn( - 'AResource') - self.m.ReplayAll() updated_stack = parser.Stack(self.ctx, 'updated_stack', @@ -1464,12 +1494,6 @@ class StackTest(HeatTestCase): # replacement resource generic_rsrc.ResourceWithProps.handle_create().AndRaise(Exception) - # Calls to ResourceWithProps.handle_update will raise - # resource.UpdateReplace because we've not specified the modified - # key/property in update_allowed_keys/update_allowed_properties - - generic_rsrc.ResourceWithProps.handle_create().AndReturn(None) - self.m.ReplayAll() updated_stack = parser.Stack(self.ctx, 'updated_stack', @@ -1480,7 +1504,7 @@ class StackTest(HeatTestCase): (parser.Stack.ROLLBACK, parser.Stack.COMPLETE)) self.assertEqual(self.stack['AResource'].properties['Foo'], 'abc') self.assertEqual(self.stack['BResource'].properties['Foo'], - 'AResource3') + 'AResource1') self.m.VerifyAll() -- 2.45.2