heat engine : Implement rollback for stack create

Implement stack rollback for create_stack, so if a stack
creation fails we automatically rollback (ie delete) the
stack

NOTE : this option defaults to on, so use the --disable-rollback
option to the CLI tools if you want to disable this feature

blueprint stack-rollback

Change-Id: I70a3822426706d0787e571517e059baff1406c0f

diff --git a/heat/db/sqlalchemy/migrate_repo/versions/015_add_stack_rollback_col.py b/heat/db/sqlalchemy/migrate_repo/versions/015_add_stack_rollback_col.py
new file mode 100644 (file)
index 0000000..2856fef
--- /dev/null
+++ b/heat/db/sqlalchemy/migrate_repo/versions/015_add_stack_rollback_col.py
@@ -0,0 +1,26 @@
+from sqlalchemy import *
+from migrate import *
+
+
+def upgrade(migrate_engine):
+    meta = MetaData(bind=migrate_engine)
+    stack = Table('stack', meta, autoload=True)
+
+    # Note hard-coded default 60 (minutes) here from the value in the
+    # engine, means we can upgrade and populate existing rows
+    try:
+        col = Column('disable_rollback', Boolean, nullable=False, default=True)
+        col.create(stack, populate_default=True)
+    except Exception as ex:
+        print "Caught exception adding disable_rollback column %s" % ex
+        # *Hack-alert* Sqlite in the unit tests can't handle the above
+        # approach to nullable=False, so retry with nullable=True
+        Column('disable_rollback', Boolean, nullable=True,
+               default=60).create(stack)
+
+
+def downgrade(migrate_engine):
+    meta = MetaData(bind=migrate_engine)
+    stack = Table('stack', meta, autoload=True)
+
+    stack.c.disable_rollback.drop()

diff --git a/heat/db/sqlalchemy/models.py b/heat/db/sqlalchemy/models.py
index 6de48bf6b2dc5de74cd187950fa8c2c3a682420c..65e53746d99eca337e11d4d6c80734a042a69d46 100644 (file)
--- a/heat/db/sqlalchemy/models.py
+++ b/heat/db/sqlalchemy/models.py
@@ -162,6 +162,7 @@ class Stack(BASE, HeatBase):
         nullable=False)
     owner_id = Column(Integer, nullable=True)
     timeout = Column(Integer)
+    disable_rollback = Column(Boolean)
 
 
 class UserCreds(BASE, HeatBase):

diff --git a/heat/engine/parser.py b/heat/engine/parser.py
index a469236ef8e5ed9e0b6c8c1dffa831eedacd41a0..963e15fc15d48a1854c23481a79669a8c6329b69 100644 (file)
--- a/heat/engine/parser.py
+++ b/heat/engine/parser.py
@@ -35,6 +35,10 @@ logger = logging.getLogger(__name__)
 
 
 class Stack(object):
+
+    ACTIONS = (CREATE, DELETE, UPDATE, ROLLBACK
+               ) = ('CREATE', 'DELETE', 'UPDATE', 'ROLLBACK')
+
     CREATE_IN_PROGRESS = 'CREATE_IN_PROGRESS'
     CREATE_FAILED = 'CREATE_FAILED'
     CREATE_COMPLETE = 'CREATE_COMPLETE'
@@ -47,12 +51,16 @@ class Stack(object):
     UPDATE_COMPLETE = 'UPDATE_COMPLETE'
     UPDATE_FAILED = 'UPDATE_FAILED'
 
+    ROLLBACK_IN_PROGRESS = 'ROLLBACK_IN_PROGRESS'
+    ROLLBACK_COMPLETE = 'ROLLBACK_COMPLETE'
+    ROLLBACK_FAILED = 'ROLLBACK_FAILED'
+
     created_time = timestamp.Timestamp(db_api.stack_get, 'created_at')
     updated_time = timestamp.Timestamp(db_api.stack_get, 'updated_at')
 
     def __init__(self, context, stack_name, tmpl, parameters=None,
                  stack_id=None, state=None, state_description='',
-                 timeout_mins=60, resolve_data=True):
+                 timeout_mins=60, resolve_data=True, disable_rollback=False):
         '''
         Initialise from a context, name, Template object and (optionally)
         Parameters object. The database ID may also be initialised, if the
@@ -70,6 +78,7 @@ class Stack(object):
         self.state = state
         self.state_description = state_description
         self.timeout_mins = timeout_mins
+        self.disable_rollback = disable_rollback
 
         if parameters is None:
             parameters = Parameters(self.name, self.t)
@@ -109,7 +118,7 @@ class Stack(object):
         params = Parameters(stack.name, template, stack.parameters)
         stack = cls(context, stack.name, template, params,
                     stack.id, stack.status, stack.status_reason, stack.timeout,
-                    resolve_data)
+                    resolve_data, stack.disable_rollback)
 
         return stack
 
@@ -131,6 +140,7 @@ class Stack(object):
             'status': self.state,
             'status_reason': self.state_description,
             'timeout': self.timeout_mins,
+            'disable_rollback': self.disable_rollback,
         }
         if self.id:
             db_api.stack_update(self.context, self.id, s)
@@ -251,6 +261,9 @@ class Stack(object):
 
         self.state_set(stack_status, reason)
 
+        if stack_status == self.CREATE_FAILED and not self.disable_rollback:
+            self.delete(action=self.ROLLBACK)
+
     def update(self, newstack):
         '''
         Compare the current stack with newstack,
@@ -375,11 +388,22 @@ class Stack(object):
 
         self.state_set(stack_status, reason)
 
-    def delete(self):
+    def delete(self, action=DELETE):
         '''
         Delete all of the resources, and then the stack itself.
+        The action parameter is used to differentiate between a user
+        initiated delete and an automatic stack rollback after a failed
+        create, which amount to the same thing, but the states are recorded
+        differently.
         '''
-        self.state_set(self.DELETE_IN_PROGRESS, 'Stack deletion started')
+        if action == self.DELETE:
+            self.state_set(self.DELETE_IN_PROGRESS, 'Stack deletion started')
+        elif action == self.ROLLBACK:
+            self.state_set(self.ROLLBACK_IN_PROGRESS, 'Stack rollback started')
+        else:
+            logger.error("Unexpected action %s passed to delete!" % action)
+            self.state_set(self.DELETE_FAILED, "Invalid action %s" % action)
+            return
 
         failures = []
         for res in reversed(self):
@@ -390,10 +414,17 @@ class Stack(object):
                 failures.append(str(res))
 
         if failures:
-            self.state_set(self.DELETE_FAILED,
-                           'Failed to delete ' + ', '.join(failures))
+            if action == self.DELETE:
+                self.state_set(self.DELETE_FAILED,
+                               'Failed to delete ' + ', '.join(failures))
+            elif action == self.ROLLBACK:
+                self.state_set(self.ROLLBACK_FAILED,
+                               'Failed to rollback ' + ', '.join(failures))
         else:
-            self.state_set(self.DELETE_COMPLETE, 'Deleted successfully')
+            if action == self.DELETE:
+                self.state_set(self.DELETE_COMPLETE, 'Deleted successfully')
+            elif action == self.ROLLBACK:
+                self.state_set(self.ROLLBACK_COMPLETE, 'Rollback completed')
             db_api.stack_delete(self.context, self.id)
 
     def output(self, key):

diff --git a/heat/engine/stack_resource.py b/heat/engine/stack_resource.py
index f392abb6e695a0b0bad0c388adda545aaa743f83..bec2f721f2554a79d4d68e482e40b926d658fada 100644 (file)
--- a/heat/engine/stack_resource.py
+++ b/heat/engine/stack_resource.py
@@ -53,10 +53,13 @@ class StackResource(resource.Resource):
         params = parser.Parameters(self.physical_resource_name(), template,
                                    user_params)
 
+        # Note we disable rollback for nested stacks, since they
+        # should be rolled back by the parent stack on failure
         self._nested = parser.Stack(self.context,
                                     self.physical_resource_name(),
                                     template,
-                                    params)
+                                    params,
+                                    disable_rollback=True)
 
         nested_id = self._nested.store(self.stack)
         self.resource_id_set(nested_id)

diff --git a/heat/tests/test_loadbalancer.py b/heat/tests/test_loadbalancer.py
index cfce3028cfbd728c50445269c623dab8725b855f..d15cd5078f9c1d13a8ed759d6e8c6889e07f9bb0 100644 (file)
--- a/heat/tests/test_loadbalancer.py
+++ b/heat/tests/test_loadbalancer.py
@@ -69,7 +69,7 @@ class LoadBalancerTest(unittest.TestCase):
         template = parser.Template(t)
         params = parser.Parameters('test_stack', template, {'KeyName': 'test'})
         stack = parser.Stack(create_context(self.m), 'test_stack', template,
-                             params, stack_id=None)
+                             params, stack_id=None, disable_rollback=True)
         stack.store()
 
         return stack

diff --git a/heat/tests/test_parser.py b/heat/tests/test_parser.py
index dbc8c4967bb631d15d7199aaf9fd72a9bb4a12c4..b0a899b076f563fb1ea8585adfc9ced45630d5e9 100644 (file)
--- a/heat/tests/test_parser.py
+++ b/heat/tests/test_parser.py
@@ -25,6 +25,7 @@ from heat.engine import parameters
 from heat.engine import template
 from heat.engine.resource import Resource
 from heat.tests.utils import stack_delete_after
+import heat.db as db_api
 
 
 def join(raw):
@@ -358,3 +359,48 @@ class StackTest(unittest.TestCase):
         self.stack.state_set(self.stack.CREATE_IN_PROGRESS, 'testing')
         self.assertNotEqual(self.stack.updated_time, None)
         self.assertNotEqual(self.stack.updated_time, stored_time)
+
+    @stack_delete_after
+    def test_delete(self):
+        self.stack = parser.Stack(self.ctx, 'delete_test',
+                                  parser.Template({}))
+        stack_id = self.stack.store()
+
+        db_s = db_api.stack_get(self.ctx, stack_id)
+        self.assertNotEqual(db_s, None)
+
+        self.stack.delete()
+
+        db_s = db_api.stack_get(self.ctx, stack_id)
+        self.assertEqual(db_s, None)
+        self.assertEqual(self.stack.state, self.stack.DELETE_COMPLETE)
+
+    @stack_delete_after
+    def test_delete_rollback(self):
+        self.stack = parser.Stack(self.ctx, 'delete_rollback_test',
+                                  parser.Template({}))
+        stack_id = self.stack.store()
+
+        db_s = db_api.stack_get(self.ctx, stack_id)
+        self.assertNotEqual(db_s, None)
+
+        self.stack.delete(action=self.stack.ROLLBACK)
+
+        db_s = db_api.stack_get(self.ctx, stack_id)
+        self.assertEqual(db_s, None)
+        self.assertEqual(self.stack.state, self.stack.ROLLBACK_COMPLETE)
+
+    @stack_delete_after
+    def test_delete_badaction(self):
+        self.stack = parser.Stack(self.ctx, 'delete_badaction_test',
+                                  parser.Template({}))
+        stack_id = self.stack.store()
+
+        db_s = db_api.stack_get(self.ctx, stack_id)
+        self.assertNotEqual(db_s, None)
+
+        self.stack.delete(action="wibble")
+
+        db_s = db_api.stack_get(self.ctx, stack_id)
+        self.assertNotEqual(db_s, None)
+        self.assertEqual(self.stack.state, self.stack.DELETE_FAILED)

diff --git a/heat/tests/test_waitcondition.py b/heat/tests/test_waitcondition.py
index 8baba8bbd3ce607d1969e0353a2b403390cd9a2c..26aa48528c295b978ef74d285d023926f6d0efc1 100644 (file)
--- a/heat/tests/test_waitcondition.py
+++ b/heat/tests/test_waitcondition.py
@@ -104,7 +104,8 @@ class WaitConditionTest(unittest.TestCase):
         parameters = parser.Parameters(stack_name, template, params)
         ctx = context.get_admin_context()
         ctx.tenant_id = 'test_tenant'
-        stack = parser.Stack(ctx, stack_name, template, parameters)
+        stack = parser.Stack(ctx, stack_name, template, parameters,
+                             disable_rollback=True)
 
         self.stack_id = stack.store()
 
@@ -396,7 +397,8 @@ class WaitConditionHandleTest(unittest.TestCase):
         parameters = parser.Parameters(stack_name, template, params)
         ctx = context.get_admin_context()
         ctx.tenant_id = 'test_tenant'
-        stack = parser.Stack(ctx, stack_name, template, parameters)
+        stack = parser.Stack(ctx, stack_name, template, parameters,
+                             disable_rollback=True)
         # Stub out the UUID for this test, so we can get an expected signature
         self.m.StubOutWithMock(uuid, 'uuid4')
         uuid.uuid4().AndReturn('STACKABCD1234')