From 4a5ddb64f4095d0af8a2c7e87862486a8e34d0a2 Mon Sep 17 00:00:00 2001 From: Liang Chen Date: Sat, 29 Jun 2013 21:45:43 +0800 Subject: [PATCH] destroy failed instances created by auto scaling The ResourceFailure exception was silently eaten when raise_on_error set to False. This patch handles the ResourceFailure exception locally when raise_on_error set to False - called from ScalingPolicy alarm (i.e. runtime adjustment). So users will not be charged for orphaned instances. But they can still see what happened through event list. Fixes bug #1192554 Change-Id: I01106d2e443dbc37af066b04ef7064bfa5656675 --- heat/engine/resources/autoscaling.py | 8 +++++++- heat/tests/test_autoscaling.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 1 deletion(-) diff --git a/heat/engine/resources/autoscaling.py b/heat/engine/resources/autoscaling.py index a4f3518e..f9078927 100644 --- a/heat/engine/resources/autoscaling.py +++ b/heat/engine/resources/autoscaling.py @@ -192,9 +192,15 @@ class InstanceGroup(resource.Resource): try: yield inst.create() - except exception.ResourceFailure: + except exception.ResourceFailure as ex: if raise_on_error: raise + # Handle instance creation failure locally by destroying the + # failed instance to avoid orphaned instances costing user + # extra memory + logger.warn('Creating %s instance %d failed %s, destroying' + % (str(self), index, str(ex))) + inst.destroy() else: inst_list.append(name) self.resource_id_set(','.join(inst_list)) diff --git a/heat/tests/test_autoscaling.py b/heat/tests/test_autoscaling.py index 06ab59d6..7e340e42 100644 --- a/heat/tests/test_autoscaling.py +++ b/heat/tests/test_autoscaling.py @@ -561,6 +561,34 @@ class AutoScalingTest(HeatTestCase): rsrc.resource_id) self.m.VerifyAll() + def test_scaling_group_scale_up_failure(self): + t = template_format.parse(as_template) + stack = parse_stack(t) + + # Create initial group + self._stub_lb_reload(['WebServerGroup-0']) + now = timeutils.utcnow() + self._stub_meta_expected(now, 'ExactCapacity : 1') + self._stub_create(1) + self.m.ReplayAll() + rsrc = self.create_scaling_group(t, stack, 'WebServerGroup') + self.assertEqual('WebServerGroup-0', rsrc.resource_id) + self.m.VerifyAll() + self.m.UnsetStubs() + + # Scale up one 1 instance with resource failure + self.m.StubOutWithMock(instance.Instance, 'handle_create') + exc = exception.ResourceFailure(Exception()) + instance.Instance.handle_create().AndRaise(exc) + self.m.StubOutWithMock(instance.Instance, 'destroy') + instance.Instance.destroy() + self.m.ReplayAll() + + rsrc.adjust(1) + self.assertEqual('WebServerGroup-0', rsrc.resource_id) + + self.m.VerifyAll() + def test_scaling_group_nochange(self): t = template_format.parse(as_template) stack = parse_stack(t) -- 2.45.2