Implement interruption-free update and rollback

During an update, where a resource must be replaced in its entirety, create
the replacement resource before deleting the old resource.

Also, allow rollback to the previous version of the resource without
replacing it, where that is possible.

Fixes bug #1176142

Change-Id: Id89654bad297815bdbcc86f666367772889b5df4

diff --git a/heat/engine/resource.py b/heat/engine/resource.py
index 302693ff18c06c4471e2c95c238c679742f63be0..cd82d88a5fa07aa50730505fa9f9b25b3af0a743 100644 (file)
--- a/heat/engine/resource.py
+++ b/heat/engine/resource.py
@@ -602,6 +602,7 @@ class Resource(object):
                 rs.update_and_save({'action': self.action,
                                     'status': self.status,
                                     'status_reason': reason,
+                                    'stack_id': self.stack.id,
                                     'nova_instance': self.resource_id})
 
                 self.stack.updated_time = datetime.utcnow()

diff --git a/heat/engine/update.py b/heat/engine/update.py
index 9ccbef5d9e3f6b1271681e02b52b22161b76911a..2b1ac7b214bf47dfd84f8b0f220a4b94373e0b2b 100644 (file)
--- a/heat/engine/update.py
+++ b/heat/engine/update.py
@@ -13,6 +13,8 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
+from heat.db import api as db_api
+
 from heat.engine import resource
 from heat.engine import scheduler
 
@@ -51,6 +53,10 @@ class StackUpdate(object):
         existing_deps = self.existing_stack.dependencies
         new_deps = self.new_stack.dependencies
 
+        cleanup_prev = scheduler.DependencyTaskGroup(
+            self.previous_stack.dependencies,
+            self._remove_backup_resource,
+            reverse=True)
         cleanup = scheduler.DependencyTaskGroup(existing_deps,
                                                 self._remove_old_resource,
                                                 reverse=True)
@@ -59,13 +65,32 @@ class StackUpdate(object):
         update = scheduler.DependencyTaskGroup(new_deps,
                                                self._update_resource)
 
-        yield cleanup()
+        if not self.rollback:
+            yield cleanup_prev()
+
         yield create_new()
-        yield update()
+        try:
+            yield update()
+        finally:
+            prev_deps = self.previous_stack._get_dependencies(
+                self.previous_stack.resources.itervalues())
+            self.previous_stack.dependencies = prev_deps
+        yield cleanup()
+
+    @scheduler.wrappertask
+    def _remove_backup_resource(self, prev_res):
+        if prev_res.state not in ((prev_res.INIT, prev_res.COMPLETE),
+                                  (prev_res.DELETE, prev_res.COMPLETE)):
+            logger.debug("Deleting backup resource %s" % prev_res.name)
+            yield prev_res.destroy()
 
     @scheduler.wrappertask
     def _remove_old_resource(self, existing_res):
         res_name = existing_res.name
+
+        if res_name in self.previous_stack:
+            yield self._remove_backup_resource(self.previous_stack[res_name])
+
         if res_name not in self.new_stack:
             logger.debug("resource %s not found in updated stack"
                          % res_name + " definition, deleting")
@@ -78,14 +103,45 @@ class StackUpdate(object):
         if res_name not in self.existing_stack:
             logger.debug("resource %s not found in current stack"
                          % res_name + " definition, adding")
-            new_res.stack = self.existing_stack
-            self.existing_stack[res_name] = new_res
-            yield new_res.create()
+            yield self._create_resource(new_res)
+
+    @staticmethod
+    def _exchange_stacks(existing_res, prev_res):
+        db_api.resource_exchange_stacks(existing_res.stack.context,
+                                        existing_res.id, prev_res.id)
+        existing_res.stack, prev_res.stack = prev_res.stack, existing_res.stack
+        existing_res.stack[existing_res.name] = existing_res
+        prev_res.stack[prev_res.name] = prev_res
 
     @scheduler.wrappertask
-    def _replace_resource(self, new_res):
+    def _create_resource(self, new_res):
         res_name = new_res.name
-        yield self.existing_stack[res_name].destroy()
+
+        # Clean up previous resource
+        if res_name in self.previous_stack:
+            prev_res = self.previous_stack[res_name]
+
+            if prev_res.state not in ((prev_res.INIT, prev_res.COMPLETE),
+                                      (prev_res.DELETE, prev_res.COMPLETE)):
+                # Swap in the backup resource if it is in a valid state,
+                # instead of creating a new resource
+                if prev_res.status == prev_res.COMPLETE:
+                    logger.debug("Swapping in backup Resource %s" % res_name)
+                    self._exchange_stacks(self.existing_stack[res_name],
+                                          prev_res)
+                    return
+
+                logger.debug("Deleting backup Resource %s" % res_name)
+                yield prev_res.destroy()
+
+        # Back up existing resource
+        if res_name in self.existing_stack:
+            logger.debug("Backing up existing Resource %s" % res_name)
+            existing_res = self.existing_stack[res_name]
+            existing_res.stack = self.previous_stack
+            self.previous_stack[res_name] = existing_res
+            existing_res.state_set(existing_res.UPDATE, existing_res.COMPLETE)
+
         new_res.stack = self.existing_stack
         self.existing_stack[res_name] = new_res
         yield new_res.create()
@@ -108,7 +164,7 @@ class StackUpdate(object):
                 yield self.existing_stack[res_name].update(new_snippet,
                                                            existing_snippet)
             except resource.UpdateReplace:
-                yield self._replace_resource(new_res)
+                yield self._create_resource(new_res)
             else:
                 logger.info("Resource %s for stack %s updated" %
                             (res_name, self.existing_stack.name))

diff --git a/heat/tests/test_parser.py b/heat/tests/test_parser.py
index d0419d1d9992fe08eddfd6b39793f1c6ca18b92a..ac2a71fa7746c2c8f13b7b8e8f6bd4489c274607 100644 (file)
--- a/heat/tests/test_parser.py
+++ b/heat/tests/test_parser.py
@@ -1179,10 +1179,9 @@ class StackTest(HeatTestCase):
         # key/property in update_allowed_keys/update_allowed_properties
 
         # patch in a dummy handle_create making the replace fail when creating
-        # the replacement rsrc, but succeed the second call (rollback)
+        # the replacement rsrc
         self.m.StubOutWithMock(generic_rsrc.ResourceWithProps, 'handle_create')
         generic_rsrc.ResourceWithProps.handle_create().AndRaise(Exception)
-        generic_rsrc.ResourceWithProps.handle_create().AndReturn(None)
         self.m.ReplayAll()
 
         self.stack.update(updated_stack)
@@ -1217,8 +1216,9 @@ class StackTest(HeatTestCase):
         # patch in a dummy handle_create making the replace fail when creating
         # the replacement rsrc, and again on the second call (rollback)
         self.m.StubOutWithMock(generic_rsrc.ResourceWithProps, 'handle_create')
+        self.m.StubOutWithMock(generic_rsrc.ResourceWithProps, 'handle_delete')
         generic_rsrc.ResourceWithProps.handle_create().AndRaise(Exception)
-        generic_rsrc.ResourceWithProps.handle_create().AndRaise(Exception)
+        generic_rsrc.ResourceWithProps.handle_delete().AndRaise(Exception)
         self.m.ReplayAll()
 
         self.stack.update(updated_stack)
@@ -1289,6 +1289,40 @@ class StackTest(HeatTestCase):
         # Unset here so delete() is not stubbed for stack.delete cleanup
         self.m.UnsetStubs()
 
+    @utils.stack_delete_after
+    def test_update_rollback_replace(self):
+        tmpl = {'Resources': {
+                'AResource': {'Type': 'ResourceWithPropsType',
+                              'Properties': {'Foo': 'foo'}}}}
+
+        self.stack = parser.Stack(self.ctx, 'update_test_stack',
+                                  template.Template(tmpl),
+                                  disable_rollback=False)
+        self.stack.store()
+        self.stack.create()
+        self.assertEqual(self.stack.state,
+                         (parser.Stack.CREATE, parser.Stack.COMPLETE))
+
+        tmpl2 = {'Resources': {'AResource': {'Type': 'ResourceWithPropsType',
+                                             'Properties': {'Foo': 'bar'}}}}
+
+        updated_stack = parser.Stack(self.ctx, 'updated_stack',
+                                     template.Template(tmpl2))
+
+        # patch in a dummy delete making the destroy fail
+        self.m.StubOutWithMock(generic_rsrc.ResourceWithProps, 'handle_delete')
+        generic_rsrc.ResourceWithProps.handle_delete().AndRaise(Exception)
+        generic_rsrc.ResourceWithProps.handle_delete().AndReturn(None)
+        generic_rsrc.ResourceWithProps.handle_delete().AndReturn(None)
+        self.m.ReplayAll()
+
+        self.stack.update(updated_stack)
+        self.assertEqual(self.stack.state,
+                         (parser.Stack.ROLLBACK, parser.Stack.COMPLETE))
+        self.m.VerifyAll()
+        # Unset here so delete() is not stubbed for stack.delete cleanup
+        self.m.UnsetStubs()
+
     @utils.stack_delete_after
     def test_update_replace_by_reference(self):
         '''
@@ -1386,16 +1420,12 @@ class StackTest(HeatTestCase):
         # resource.UpdateReplace because we've not specified the modified
         # key/property in update_allowed_keys/update_allowed_properties
 
-        generic_rsrc.ResourceWithProps.FnGetRefId().AndReturn(
+        generic_rsrc.ResourceWithProps.FnGetRefId().MultipleTimes().AndReturn(
             'AResource')
 
         # mock to make the replace fail when creating the replacement resource
         generic_rsrc.ResourceWithProps.handle_create().AndRaise(Exception)
 
-        generic_rsrc.ResourceWithProps.handle_create().AndReturn(None)
-        generic_rsrc.ResourceWithProps.FnGetRefId().MultipleTimes().AndReturn(
-            'AResource')
-
         self.m.ReplayAll()
 
         updated_stack = parser.Stack(self.ctx, 'updated_stack',
@@ -1464,12 +1494,6 @@ class StackTest(HeatTestCase):
         # replacement resource
         generic_rsrc.ResourceWithProps.handle_create().AndRaise(Exception)
 
-        # Calls to ResourceWithProps.handle_update will raise
-        # resource.UpdateReplace because we've not specified the modified
-        # key/property in update_allowed_keys/update_allowed_properties
-
-        generic_rsrc.ResourceWithProps.handle_create().AndReturn(None)
-
         self.m.ReplayAll()
 
         updated_stack = parser.Stack(self.ctx, 'updated_stack',
@@ -1480,7 +1504,7 @@ class StackTest(HeatTestCase):
                          (parser.Stack.ROLLBACK, parser.Stack.COMPLETE))
         self.assertEqual(self.stack['AResource'].properties['Foo'], 'abc')
         self.assertEqual(self.stack['BResource'].properties['Foo'],
-                         'AResource3')
+                         'AResource1')
 
         self.m.VerifyAll()