heat engine : Add support rollback support for stack updates

Adds support for rolling back failed stack updates

blueprint update-rollback

Change-Id: Ie90e37751365a8ef0ed24757c15f4020ee33ab8a

diff --git a/heat/engine/parser.py b/heat/engine/parser.py
index 274d9ab520071b9df244fd12dd42549f2e8678d4..e1df1a92ecf6c787c797e2d9cc9411ee38832fe0 100644 (file)
--- a/heat/engine/parser.py
+++ b/heat/engine/parser.py
@@ -264,7 +264,7 @@ class Stack(object):
         if stack_status == self.CREATE_FAILED and not self.disable_rollback:
             self.delete(action=self.ROLLBACK)
 
-    def update(self, newstack):
+    def update(self, newstack, action=UPDATE):
         '''
         Compare the current stack with newstack,
         and where necessary create/update/delete the resources until
@@ -276,11 +276,29 @@ class Stack(object):
         Update will fail if it exceeds the specified timeout. The default is
         60 minutes, set in the constructor
         '''
-        if self.state not in (self.CREATE_COMPLETE, self.UPDATE_COMPLETE):
-            self.state_set(self.UPDATE_FAILED, 'State invalid for update')
+        if action not in (self.UPDATE, self.ROLLBACK):
+            logger.error("Unexpected action %s passed to update!" % action)
+            self.state_set(self.UPDATE_FAILED, "Invalid action %s" % action)
             return
-        else:
+
+        if self.state not in (self.CREATE_COMPLETE, self.UPDATE_COMPLETE,
+                              self.ROLLBACK_COMPLETE):
+            if (action == self.ROLLBACK and
+                    self.state == self.UPDATE_IN_PROGRESS):
+                logger.debug("Starting update rollback for %s" % self.name)
+            else:
+                if action == self.UPDATE:
+                    self.state_set(self.UPDATE_FAILED,
+                                   'State invalid for update')
+                else:
+                    self.state_set(self.ROLLBACK_FAILED,
+                                   'State invalid for rollback')
+                return
+
+        if action == self.UPDATE:
             self.state_set(self.UPDATE_IN_PROGRESS, 'Stack update started')
+        else:
+            self.state_set(self.ROLLBACK_IN_PROGRESS, 'Stack rollback started')
 
         # Now make the resources match the new stack definition
         with eventlet.Timeout(self.timeout_mins * 60) as tmo:
@@ -369,7 +387,8 @@ class Stack(object):
                                     raise exception.ResourceUpdateFailed(
                                         resource_name=res.name)
                         else:
-                            logger.error("Failed to update %s" % res.name)
+                            logger.error("Failed to %s %s" %
+                                         (action, res.name))
                             raise exception.ResourceUpdateFailed(
                                 resource_name=res.name)
 
@@ -380,8 +399,12 @@ class Stack(object):
                 self.outputs = self.resolve_static_data(template_outputs)
                 self.store()
 
-                stack_status = self.UPDATE_COMPLETE
-                reason = 'Stack successfully updated'
+                if action == self.UPDATE:
+                    stack_status = self.UPDATE_COMPLETE
+                    reason = 'Stack successfully updated'
+                else:
+                    stack_status = self.ROLLBACK_COMPLETE
+                    reason = 'Stack rollback completed'
 
             except eventlet.Timeout as t:
                 if t is tmo:
@@ -391,10 +414,26 @@ class Stack(object):
                     # not my timeout
                     raise
             except exception.ResourceUpdateFailed as e:
-                stack_status = self.UPDATE_FAILED
                 reason = str(e) or "Error : %s" % type(e)
 
-        self.state_set(stack_status, reason)
+                if action == self.UPDATE:
+                    stack_status = self.UPDATE_FAILED
+                    # If rollback is enabled, we do another update, with the
+                    # existing template, so we roll back to the original state
+                    # Note - ensure nothing after the "flip the template..."
+                    # section above can raise ResourceUpdateFailed or this
+                    # will not work ;)
+                    if self.disable_rollback:
+                        stack_status = self.UPDATE_FAILED
+                    else:
+                        oldstack = Stack(self.context, self.name, self.t,
+                                         self.parameters)
+                        self.update(oldstack, action=self.ROLLBACK)
+                        return
+                else:
+                    stack_status = self.ROLLBACK_FAILED
+
+            self.state_set(stack_status, reason)
 
     def delete(self, action=DELETE):
         '''

diff --git a/heat/tests/test_parser.py b/heat/tests/test_parser.py
index 83b458cc3737a6af840f3f3733e928eb54a17496..6162d881fc78d313f0afc5ec8c4196835f4377b7 100644 (file)
--- a/heat/tests/test_parser.py
+++ b/heat/tests/test_parser.py
@@ -515,7 +515,8 @@ class StackTest(unittest.TestCase):
                                             'Properties': {'Foo': 'abc'}}}}
 
         self.stack = parser.Stack(self.ctx, 'update_test_stack',
-                                  template.Template(tmpl))
+                                  template.Template(tmpl),
+                                  disable_rollback=True)
         self.stack.store()
         self.stack.create()
         self.assertEqual(self.stack.state, parser.Stack.CREATE_COMPLETE)
@@ -547,7 +548,8 @@ class StackTest(unittest.TestCase):
                                             'Properties': {'Foo': 'abc'}}}}
 
         self.stack = parser.Stack(self.ctx, 'update_test_stack',
-                                  template.Template(tmpl))
+                                  template.Template(tmpl),
+                                  disable_rollback=True)
         self.stack.store()
         self.stack.create()
         self.assertEqual(self.stack.state, parser.Stack.CREATE_COMPLETE)
@@ -585,7 +587,8 @@ class StackTest(unittest.TestCase):
                                             'Properties': {'Foo': 'abc'}}}}
 
         self.stack = parser.Stack(self.ctx, 'update_test_stack',
-                                  template.Template(tmpl))
+                                  template.Template(tmpl),
+                                  disable_rollback=True)
         self.stack.store()
         self.stack.create()
         self.assertEqual(self.stack.state, parser.Stack.CREATE_COMPLETE)
@@ -610,3 +613,145 @@ class StackTest(unittest.TestCase):
         self.stack.update(updated_stack)
         self.assertEqual(self.stack.state, parser.Stack.UPDATE_FAILED)
         self.m.VerifyAll()
+
+    @stack_delete_after
+    def test_update_rollback(self):
+        # patch in a dummy property schema for GenericResource
+        dummy_schema = {'Foo': {'Type': 'String'}}
+        resource.GenericResource.properties_schema = dummy_schema
+
+        tmpl = {'Resources': {'AResource': {'Type': 'GenericResourceType',
+                                            'Properties': {'Foo': 'abc'}}}}
+
+        self.stack = parser.Stack(self.ctx, 'update_test_stack',
+                                  template.Template(tmpl))
+        self.stack.store()
+        self.stack.create()
+        self.assertEqual(self.stack.state, parser.Stack.CREATE_COMPLETE)
+
+        tmpl2 = {'Resources': {'AResource': {'Type': 'GenericResourceType',
+                                             'Properties': {'Foo': 'xyz'}}}}
+
+        updated_stack = parser.Stack(self.ctx, 'updated_stack',
+                                     template.Template(tmpl2))
+
+        # There will be two calls to handle_update, one for the new template
+        # then another (with the initial template) for rollback
+        self.m.StubOutWithMock(resource.GenericResource, 'handle_update')
+        resource.GenericResource.handle_update(
+            tmpl2['Resources']['AResource']).AndReturn(
+                resource.Resource.UPDATE_REPLACE)
+        resource.GenericResource.handle_update(
+            tmpl['Resources']['AResource']).AndReturn(
+                resource.Resource.UPDATE_REPLACE)
+
+        # patch in a dummy handle_create making the replace fail when creating
+        # the replacement resource, but succeed the second call (rollback)
+        self.m.StubOutWithMock(resource.GenericResource, 'handle_create')
+        resource.GenericResource.handle_create().AndRaise(Exception)
+        resource.GenericResource.handle_create().AndReturn(None)
+        self.m.ReplayAll()
+
+        self.stack.update(updated_stack)
+        self.assertEqual(self.stack.state, parser.Stack.ROLLBACK_COMPLETE)
+        self.assertEqual(self.stack['AResource'].properties['Foo'], 'abc')
+        self.m.VerifyAll()
+
+    @stack_delete_after
+    def test_update_rollback_fail(self):
+        # patch in a dummy property schema for GenericResource
+        dummy_schema = {'Foo': {'Type': 'String'}}
+        resource.GenericResource.properties_schema = dummy_schema
+
+        tmpl = {'Resources': {'AResource': {'Type': 'GenericResourceType',
+                                            'Properties': {'Foo': 'abc'}}}}
+
+        self.stack = parser.Stack(self.ctx, 'update_test_stack',
+                                  template.Template(tmpl))
+        self.stack.store()
+        self.stack.create()
+        self.assertEqual(self.stack.state, parser.Stack.CREATE_COMPLETE)
+
+        tmpl2 = {'Resources': {'AResource': {'Type': 'GenericResourceType',
+                                             'Properties': {'Foo': 'xyz'}}}}
+
+        updated_stack = parser.Stack(self.ctx, 'updated_stack',
+                                     template.Template(tmpl2))
+
+        # There will be two calls to handle_update, one for the new template
+        # then another (with the initial template) for rollback
+        self.m.StubOutWithMock(resource.GenericResource, 'handle_update')
+        resource.GenericResource.handle_update(
+            tmpl2['Resources']['AResource']).AndReturn(
+                resource.Resource.UPDATE_REPLACE)
+        resource.GenericResource.handle_update(
+            tmpl['Resources']['AResource']).AndReturn(
+                resource.Resource.UPDATE_REPLACE)
+
+        # patch in a dummy handle_create making the replace fail when creating
+        # the replacement resource, and again on the second call (rollback)
+        self.m.StubOutWithMock(resource.GenericResource, 'handle_create')
+        resource.GenericResource.handle_create().AndRaise(Exception)
+        resource.GenericResource.handle_create().AndRaise(Exception)
+        self.m.ReplayAll()
+
+        self.stack.update(updated_stack)
+        self.assertEqual(self.stack.state, parser.Stack.ROLLBACK_FAILED)
+        self.m.VerifyAll()
+
+    @stack_delete_after
+    def test_update_rollback_add(self):
+        tmpl = {'Resources': {'AResource': {'Type': 'GenericResourceType'}}}
+
+        self.stack = parser.Stack(self.ctx, 'update_test_stack',
+                                  template.Template(tmpl))
+        self.stack.store()
+        self.stack.create()
+        self.assertEqual(self.stack.state, parser.Stack.CREATE_COMPLETE)
+
+        tmpl2 = {'Resources': {
+                 'AResource': {'Type': 'GenericResourceType'},
+                 'BResource': {'Type': 'GenericResourceType'}}}
+
+        updated_stack = parser.Stack(self.ctx, 'updated_stack',
+                                     template.Template(tmpl2))
+
+        # patch in a dummy handle_create making the replace fail when creating
+        # the replacement resource, and succeed on the second call (rollback)
+        self.m.StubOutWithMock(resource.GenericResource, 'handle_create')
+        resource.GenericResource.handle_create().AndRaise(Exception)
+        self.m.ReplayAll()
+
+        self.stack.update(updated_stack)
+        self.assertEqual(self.stack.state, parser.Stack.ROLLBACK_COMPLETE)
+        self.assertFalse('BResource' in self.stack)
+        self.m.VerifyAll()
+
+    @stack_delete_after
+    def test_update_rollback_remove(self):
+        tmpl = {'Resources': {
+                'AResource': {'Type': 'GenericResourceType'},
+                'BResource': {'Type': 'GenericResourceType'}}}
+
+        self.stack = parser.Stack(self.ctx, 'update_test_stack',
+                                  template.Template(tmpl))
+        self.stack.store()
+        self.stack.create()
+        self.assertEqual(self.stack.state, parser.Stack.CREATE_COMPLETE)
+
+        tmpl2 = {'Resources': {'AResource': {'Type': 'GenericResourceType'}}}
+
+        updated_stack = parser.Stack(self.ctx, 'updated_stack',
+                                     template.Template(tmpl2))
+
+        # patch in a dummy destroy making the delete fail
+        self.m.StubOutWithMock(resource.Resource, 'destroy')
+        resource.Resource.destroy().AndReturn('Error')
+        self.m.ReplayAll()
+
+        self.stack.update(updated_stack)
+        self.assertEqual(self.stack.state, parser.Stack.ROLLBACK_COMPLETE)
+        self.assertTrue('BResource' in self.stack)
+        self.m.VerifyAll()
+        # Unset here so destroy() is not stubbed for stack.delete cleanup
+        self.m.UnsetStubs()