From: Steven Hardy Date: Thu, 14 Feb 2013 16:17:25 +0000 (+0000) Subject: heat engine : Implement rollback for stack create X-Git-Tag: 2014.1~886 X-Git-Url: https://review.fuel-infra.org/gitweb?a=commitdiff_plain;h=54defea528b938a78f686d1f37daaa7025872a34;p=openstack-build%2Fheat-build.git heat engine : Implement rollback for stack create Implement stack rollback for create_stack, so if a stack creation fails we automatically rollback (ie delete) the stack NOTE : this option defaults to on, so use the --disable-rollback option to the CLI tools if you want to disable this feature blueprint stack-rollback Change-Id: I70a3822426706d0787e571517e059baff1406c0f --- diff --git a/heat/db/sqlalchemy/migrate_repo/versions/015_add_stack_rollback_col.py b/heat/db/sqlalchemy/migrate_repo/versions/015_add_stack_rollback_col.py new file mode 100644 index 00000000..2856fef6 --- /dev/null +++ b/heat/db/sqlalchemy/migrate_repo/versions/015_add_stack_rollback_col.py @@ -0,0 +1,26 @@ +from sqlalchemy import * +from migrate import * + + +def upgrade(migrate_engine): + meta = MetaData(bind=migrate_engine) + stack = Table('stack', meta, autoload=True) + + # Note hard-coded default 60 (minutes) here from the value in the + # engine, means we can upgrade and populate existing rows + try: + col = Column('disable_rollback', Boolean, nullable=False, default=True) + col.create(stack, populate_default=True) + except Exception as ex: + print "Caught exception adding disable_rollback column %s" % ex + # *Hack-alert* Sqlite in the unit tests can't handle the above + # approach to nullable=False, so retry with nullable=True + Column('disable_rollback', Boolean, nullable=True, + default=60).create(stack) + + +def downgrade(migrate_engine): + meta = MetaData(bind=migrate_engine) + stack = Table('stack', meta, autoload=True) + + stack.c.disable_rollback.drop() diff --git a/heat/db/sqlalchemy/models.py b/heat/db/sqlalchemy/models.py index 6de48bf6..65e53746 100644 --- a/heat/db/sqlalchemy/models.py +++ b/heat/db/sqlalchemy/models.py @@ -162,6 +162,7 @@ class Stack(BASE, HeatBase): nullable=False) owner_id = Column(Integer, nullable=True) timeout = Column(Integer) + disable_rollback = Column(Boolean) class UserCreds(BASE, HeatBase): diff --git a/heat/engine/parser.py b/heat/engine/parser.py index a469236e..963e15fc 100644 --- a/heat/engine/parser.py +++ b/heat/engine/parser.py @@ -35,6 +35,10 @@ logger = logging.getLogger(__name__) class Stack(object): + + ACTIONS = (CREATE, DELETE, UPDATE, ROLLBACK + ) = ('CREATE', 'DELETE', 'UPDATE', 'ROLLBACK') + CREATE_IN_PROGRESS = 'CREATE_IN_PROGRESS' CREATE_FAILED = 'CREATE_FAILED' CREATE_COMPLETE = 'CREATE_COMPLETE' @@ -47,12 +51,16 @@ class Stack(object): UPDATE_COMPLETE = 'UPDATE_COMPLETE' UPDATE_FAILED = 'UPDATE_FAILED' + ROLLBACK_IN_PROGRESS = 'ROLLBACK_IN_PROGRESS' + ROLLBACK_COMPLETE = 'ROLLBACK_COMPLETE' + ROLLBACK_FAILED = 'ROLLBACK_FAILED' + created_time = timestamp.Timestamp(db_api.stack_get, 'created_at') updated_time = timestamp.Timestamp(db_api.stack_get, 'updated_at') def __init__(self, context, stack_name, tmpl, parameters=None, stack_id=None, state=None, state_description='', - timeout_mins=60, resolve_data=True): + timeout_mins=60, resolve_data=True, disable_rollback=False): ''' Initialise from a context, name, Template object and (optionally) Parameters object. The database ID may also be initialised, if the @@ -70,6 +78,7 @@ class Stack(object): self.state = state self.state_description = state_description self.timeout_mins = timeout_mins + self.disable_rollback = disable_rollback if parameters is None: parameters = Parameters(self.name, self.t) @@ -109,7 +118,7 @@ class Stack(object): params = Parameters(stack.name, template, stack.parameters) stack = cls(context, stack.name, template, params, stack.id, stack.status, stack.status_reason, stack.timeout, - resolve_data) + resolve_data, stack.disable_rollback) return stack @@ -131,6 +140,7 @@ class Stack(object): 'status': self.state, 'status_reason': self.state_description, 'timeout': self.timeout_mins, + 'disable_rollback': self.disable_rollback, } if self.id: db_api.stack_update(self.context, self.id, s) @@ -251,6 +261,9 @@ class Stack(object): self.state_set(stack_status, reason) + if stack_status == self.CREATE_FAILED and not self.disable_rollback: + self.delete(action=self.ROLLBACK) + def update(self, newstack): ''' Compare the current stack with newstack, @@ -375,11 +388,22 @@ class Stack(object): self.state_set(stack_status, reason) - def delete(self): + def delete(self, action=DELETE): ''' Delete all of the resources, and then the stack itself. + The action parameter is used to differentiate between a user + initiated delete and an automatic stack rollback after a failed + create, which amount to the same thing, but the states are recorded + differently. ''' - self.state_set(self.DELETE_IN_PROGRESS, 'Stack deletion started') + if action == self.DELETE: + self.state_set(self.DELETE_IN_PROGRESS, 'Stack deletion started') + elif action == self.ROLLBACK: + self.state_set(self.ROLLBACK_IN_PROGRESS, 'Stack rollback started') + else: + logger.error("Unexpected action %s passed to delete!" % action) + self.state_set(self.DELETE_FAILED, "Invalid action %s" % action) + return failures = [] for res in reversed(self): @@ -390,10 +414,17 @@ class Stack(object): failures.append(str(res)) if failures: - self.state_set(self.DELETE_FAILED, - 'Failed to delete ' + ', '.join(failures)) + if action == self.DELETE: + self.state_set(self.DELETE_FAILED, + 'Failed to delete ' + ', '.join(failures)) + elif action == self.ROLLBACK: + self.state_set(self.ROLLBACK_FAILED, + 'Failed to rollback ' + ', '.join(failures)) else: - self.state_set(self.DELETE_COMPLETE, 'Deleted successfully') + if action == self.DELETE: + self.state_set(self.DELETE_COMPLETE, 'Deleted successfully') + elif action == self.ROLLBACK: + self.state_set(self.ROLLBACK_COMPLETE, 'Rollback completed') db_api.stack_delete(self.context, self.id) def output(self, key): diff --git a/heat/engine/stack_resource.py b/heat/engine/stack_resource.py index f392abb6..bec2f721 100644 --- a/heat/engine/stack_resource.py +++ b/heat/engine/stack_resource.py @@ -53,10 +53,13 @@ class StackResource(resource.Resource): params = parser.Parameters(self.physical_resource_name(), template, user_params) + # Note we disable rollback for nested stacks, since they + # should be rolled back by the parent stack on failure self._nested = parser.Stack(self.context, self.physical_resource_name(), template, - params) + params, + disable_rollback=True) nested_id = self._nested.store(self.stack) self.resource_id_set(nested_id) diff --git a/heat/tests/test_loadbalancer.py b/heat/tests/test_loadbalancer.py index cfce3028..d15cd507 100644 --- a/heat/tests/test_loadbalancer.py +++ b/heat/tests/test_loadbalancer.py @@ -69,7 +69,7 @@ class LoadBalancerTest(unittest.TestCase): template = parser.Template(t) params = parser.Parameters('test_stack', template, {'KeyName': 'test'}) stack = parser.Stack(create_context(self.m), 'test_stack', template, - params, stack_id=None) + params, stack_id=None, disable_rollback=True) stack.store() return stack diff --git a/heat/tests/test_parser.py b/heat/tests/test_parser.py index dbc8c496..b0a899b0 100644 --- a/heat/tests/test_parser.py +++ b/heat/tests/test_parser.py @@ -25,6 +25,7 @@ from heat.engine import parameters from heat.engine import template from heat.engine.resource import Resource from heat.tests.utils import stack_delete_after +import heat.db as db_api def join(raw): @@ -358,3 +359,48 @@ class StackTest(unittest.TestCase): self.stack.state_set(self.stack.CREATE_IN_PROGRESS, 'testing') self.assertNotEqual(self.stack.updated_time, None) self.assertNotEqual(self.stack.updated_time, stored_time) + + @stack_delete_after + def test_delete(self): + self.stack = parser.Stack(self.ctx, 'delete_test', + parser.Template({})) + stack_id = self.stack.store() + + db_s = db_api.stack_get(self.ctx, stack_id) + self.assertNotEqual(db_s, None) + + self.stack.delete() + + db_s = db_api.stack_get(self.ctx, stack_id) + self.assertEqual(db_s, None) + self.assertEqual(self.stack.state, self.stack.DELETE_COMPLETE) + + @stack_delete_after + def test_delete_rollback(self): + self.stack = parser.Stack(self.ctx, 'delete_rollback_test', + parser.Template({})) + stack_id = self.stack.store() + + db_s = db_api.stack_get(self.ctx, stack_id) + self.assertNotEqual(db_s, None) + + self.stack.delete(action=self.stack.ROLLBACK) + + db_s = db_api.stack_get(self.ctx, stack_id) + self.assertEqual(db_s, None) + self.assertEqual(self.stack.state, self.stack.ROLLBACK_COMPLETE) + + @stack_delete_after + def test_delete_badaction(self): + self.stack = parser.Stack(self.ctx, 'delete_badaction_test', + parser.Template({})) + stack_id = self.stack.store() + + db_s = db_api.stack_get(self.ctx, stack_id) + self.assertNotEqual(db_s, None) + + self.stack.delete(action="wibble") + + db_s = db_api.stack_get(self.ctx, stack_id) + self.assertNotEqual(db_s, None) + self.assertEqual(self.stack.state, self.stack.DELETE_FAILED) diff --git a/heat/tests/test_waitcondition.py b/heat/tests/test_waitcondition.py index 8baba8bb..26aa4852 100644 --- a/heat/tests/test_waitcondition.py +++ b/heat/tests/test_waitcondition.py @@ -104,7 +104,8 @@ class WaitConditionTest(unittest.TestCase): parameters = parser.Parameters(stack_name, template, params) ctx = context.get_admin_context() ctx.tenant_id = 'test_tenant' - stack = parser.Stack(ctx, stack_name, template, parameters) + stack = parser.Stack(ctx, stack_name, template, parameters, + disable_rollback=True) self.stack_id = stack.store() @@ -396,7 +397,8 @@ class WaitConditionHandleTest(unittest.TestCase): parameters = parser.Parameters(stack_name, template, params) ctx = context.get_admin_context() ctx.tenant_id = 'test_tenant' - stack = parser.Stack(ctx, stack_name, template, parameters) + stack = parser.Stack(ctx, stack_name, template, parameters, + disable_rollback=True) # Stub out the UUID for this test, so we can get an expected signature self.m.StubOutWithMock(uuid, 'uuid4') uuid.uuid4().AndReturn('STACKABCD1234')