destroy failed instances created by auto scaling

The ResourceFailure exception was silently eaten when raise_on_error
set to False. This patch handles the ResourceFailure exception locally
when raise_on_error set to False - called from ScalingPolicy alarm
(i.e. runtime adjustment). So users will not be charged for orphaned
instances. But they can still see what happened through event list.

Fixes bug #1192554

Change-Id: I01106d2e443dbc37af066b04ef7064bfa5656675

diff --git a/heat/engine/resources/autoscaling.py b/heat/engine/resources/autoscaling.py
index a4f3518e2e03baee4ea9b257b02830d8847a6ea0..f9078927b41fd8e588206794f9812377a7afbf35 100644 (file)
--- a/heat/engine/resources/autoscaling.py
+++ b/heat/engine/resources/autoscaling.py
@@ -192,9 +192,15 @@ class InstanceGroup(resource.Resource):
 
             try:
                 yield inst.create()
-            except exception.ResourceFailure:
+            except exception.ResourceFailure as ex:
                 if raise_on_error:
                     raise
+                # Handle instance creation failure locally by destroying the
+                # failed instance to avoid orphaned instances costing user
+                # extra memory
+                logger.warn('Creating %s instance %d failed %s, destroying'
+                            % (str(self), index, str(ex)))
+                inst.destroy()
             else:
                 inst_list.append(name)
                 self.resource_id_set(','.join(inst_list))

diff --git a/heat/tests/test_autoscaling.py b/heat/tests/test_autoscaling.py
index 06ab59d667be3548b7ef889343dc603801f23917..7e340e42fb0e4ba4fe840edc8f8982cd3faac7bc 100644 (file)
--- a/heat/tests/test_autoscaling.py
+++ b/heat/tests/test_autoscaling.py
@@ -561,6 +561,34 @@ class AutoScalingTest(HeatTestCase):
                          rsrc.resource_id)
         self.m.VerifyAll()
 
+    def test_scaling_group_scale_up_failure(self):
+        t = template_format.parse(as_template)
+        stack = parse_stack(t)
+
+        # Create initial group
+        self._stub_lb_reload(['WebServerGroup-0'])
+        now = timeutils.utcnow()
+        self._stub_meta_expected(now, 'ExactCapacity : 1')
+        self._stub_create(1)
+        self.m.ReplayAll()
+        rsrc = self.create_scaling_group(t, stack, 'WebServerGroup')
+        self.assertEqual('WebServerGroup-0', rsrc.resource_id)
+        self.m.VerifyAll()
+        self.m.UnsetStubs()
+
+        # Scale up one 1 instance with resource failure
+        self.m.StubOutWithMock(instance.Instance, 'handle_create')
+        exc = exception.ResourceFailure(Exception())
+        instance.Instance.handle_create().AndRaise(exc)
+        self.m.StubOutWithMock(instance.Instance, 'destroy')
+        instance.Instance.destroy()
+        self.m.ReplayAll()
+
+        rsrc.adjust(1)
+        self.assertEqual('WebServerGroup-0', rsrc.resource_id)
+
+        self.m.VerifyAll()
+
     def test_scaling_group_nochange(self):
         t = template_format.parse(as_template)
         stack = parse_stack(t)