Make sure failures in groups (autoscaling & static) are raised.

Fixes bug 1104617

Change-Id: Iebd2b0c6a4adf8c252db31348e4dc241c0479765

diff --git a/heat/common/exception.py b/heat/common/exception.py
index 2d9aceeb3d3c22d7df45137b40e55ff02153b392..16b012d0d1c1079df0c815e87cd04aadacdbb348 100644 (file)
--- a/heat/common/exception.py
+++ b/heat/common/exception.py
@@ -232,3 +232,7 @@ class PhysicalResourceNotFound(OpenstackException):
 
 class WatchRuleNotFound(OpenstackException):
     message = _("The Watch Rule (%(watch_name)s) could not be found.")
+
+
+class NestedResourceFailure(OpenstackException):
+    message = _("%(message)s")

diff --git a/heat/engine/resources/autoscaling.py b/heat/engine/resources/autoscaling.py
index 2a5cfe90e8404516bf588e35f4a07c2830370d5a..9935e4ca4db07ed34b0b550577b0e448139c22b7 100644 (file)
--- a/heat/engine/resources/autoscaling.py
+++ b/heat/engine/resources/autoscaling.py
@@ -13,8 +13,9 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 
-from heat.engine import resource
+from heat.common import exception
 from heat.engine.resources import instance
+from heat.engine import resource
 
 from heat.openstack.common import log as logging
 from heat.openstack.common import timeutils
@@ -75,7 +76,7 @@ class InstanceGroup(resource.Resource):
         # resource_id is a list of resources
 
     def handle_create(self):
-        self.resize(int(self.properties['Size']))
+        self.resize(int(self.properties['Size']), raise_on_error=True)
 
     def handle_update(self):
         # TODO(asalkeld) if the only thing that has changed is the size then
@@ -108,9 +109,13 @@ class InstanceGroup(resource.Resource):
             for victim in inst_list:
                 logger.debug('handle_delete %s' % victim)
                 inst = self._make_instance(victim)
-                inst.destroy()
+                error_str = inst.destroy()
+                if error_str is not None:
+                    # try suck out the grouped resouces failure reason
+                    # and re-raise
+                    raise exception.NestedResourceFailure(message=error_str)
 
-    def resize(self, new_capacity):
+    def resize(self, new_capacity, raise_on_error=False):
         inst_list = []
         if self.resource_id is not None:
             inst_list = sorted(self.resource_id.split(','))
@@ -129,7 +134,12 @@ class InstanceGroup(resource.Resource):
                 inst = self._make_instance(name)
                 inst_list.append(name)
                 self.resource_id_set(','.join(inst_list))
-                inst.create()
+                logger.info('creating inst')
+                error_str = inst.create()
+                if raise_on_error and error_str is not None:
+                    # try suck out the grouped resouces failure reason
+                    # and re-raise
+                    raise exception.NestedResourceFailure(message=error_str)
         else:
             # shrink (kill largest numbered first)
             del_list = inst_list[new_capacity:]
@@ -192,12 +202,14 @@ class AutoScalingGroup(InstanceGroup, CooldownMixin):
         else:
             num_to_create = int(self.properties['MinSize'])
 
-        self.adjust(num_to_create, adjustment_type='ExactCapacity')
+        self.adjust(num_to_create, adjustment_type='ExactCapacity',
+                    raise_on_error=True)
 
     def handle_update(self):
         return self.UPDATE_REPLACE
 
-    def adjust(self, adjustment, adjustment_type='ChangeInCapacity'):
+    def adjust(self, adjustment, adjustment_type='ChangeInCapacity',
+               raise_on_error=False):
         if self._cooldown_inprogress():
             logger.info("%s NOT performing scaling adjustment, cooldown %s" %
                         (self.name, self.properties['Cooldown']))
@@ -227,7 +239,7 @@ class AutoScalingGroup(InstanceGroup, CooldownMixin):
             logger.debug('no change in capacity %d' % capacity)
             return
 
-        self.resize(new_capacity)
+        self.resize(new_capacity, raise_on_error=raise_on_error)
 
         self._cooldown_timestamp("%s : %s" % (adjustment_type, adjustment))
 

diff --git a/heat/tests/test_autoscaling.py b/heat/tests/test_autoscaling.py
index 2ea569f07462601f583184d1d49457242ba6c584..34a11764e70bc573ad0af7481afd7069825bead9 100644 (file)
--- a/heat/tests/test_autoscaling.py
+++ b/heat/tests/test_autoscaling.py
@@ -25,6 +25,7 @@ from heat.common import context
 from heat.common import template_format
 from heat.engine.resources import autoscaling as asc
 from heat.engine.resources import loadbalancer
+from heat.engine.resources import instance
 from heat.engine import parser
 from heat.engine.resource import Metadata
 from heat.openstack.common import timeutils
@@ -73,12 +74,18 @@ class AutoScalingTest(unittest.TestCase):
         resource = asc.ScalingPolicy(resource_name,
                                      t['Resources'][resource_name],
                                      stack)
+
         self.assertEqual(None, resource.validate())
         self.assertEqual(None, resource.create())
         self.assertEqual(asc.ScalingPolicy.CREATE_COMPLETE,
                          resource.state)
         return resource
 
+    def _stub_create(self, num):
+        self.m.StubOutWithMock(instance.Instance, 'create')
+        for x in range(num):
+            instance.Instance.create().AndReturn(None)
+
     def _stub_lb_reload(self, expected_list, unset=True):
         if unset:
             self.m.VerifyAll()
@@ -107,6 +114,7 @@ class AutoScalingTest(unittest.TestCase):
         self._stub_lb_reload(['WebServerGroup-0'])
         now = timeutils.utcnow()
         self._stub_meta_expected(now, 'ExactCapacity : 1')
+        self._stub_create(1)
         self.m.ReplayAll()
         resource = self.create_scaling_group(t, stack, 'WebServerGroup')
 
@@ -129,6 +137,7 @@ class AutoScalingTest(unittest.TestCase):
                               'WebServerGroup-2'])
         now = timeutils.utcnow()
         self._stub_meta_expected(now, 'ExactCapacity : 3')
+        self._stub_create(3)
         self.m.ReplayAll()
         resource = self.create_scaling_group(t, stack, 'WebServerGroup')
         self.assertEqual('WebServerGroup-0,WebServerGroup-1,WebServerGroup-2',
@@ -145,6 +154,7 @@ class AutoScalingTest(unittest.TestCase):
         self._stub_lb_reload(['WebServerGroup-0', 'WebServerGroup-1',
                               'WebServerGroup-2'])
         self._stub_meta_expected(now, 'ChangeInCapacity : 2')
+        self._stub_create(2)
         self.m.ReplayAll()
         resource.adjust(2)
         self.assertEqual('WebServerGroup-0,WebServerGroup-1,WebServerGroup-2',
@@ -169,6 +179,7 @@ class AutoScalingTest(unittest.TestCase):
         self._stub_lb_reload(['WebServerGroup-0', 'WebServerGroup-1'])
         now = timeutils.utcnow()
         self._stub_meta_expected(now, 'ExactCapacity : 2')
+        self._stub_create(2)
         self.m.ReplayAll()
         resource = self.create_scaling_group(t, stack, 'WebServerGroup')
         stack.resources['WebServerGroup'] = resource
@@ -200,6 +211,7 @@ class AutoScalingTest(unittest.TestCase):
         properties = t['Resources']['WebServerGroup']['Properties']
         properties['DesiredCapacity'] = '2'
         self._stub_lb_reload(['WebServerGroup-0', 'WebServerGroup-1'])
+        self._stub_create(2)
         now = timeutils.utcnow()
         self._stub_meta_expected(now, 'ExactCapacity : 2')
         self.m.ReplayAll()
@@ -220,6 +232,7 @@ class AutoScalingTest(unittest.TestCase):
         self._stub_lb_reload(['WebServerGroup-0', 'WebServerGroup-1',
                               'WebServerGroup-2'])
         self._stub_meta_expected(now, 'PercentChangeInCapacity : 200')
+        self._stub_create(2)
         self.m.ReplayAll()
         resource.adjust(200, 'PercentChangeInCapacity')
         self.assertEqual('WebServerGroup-0,WebServerGroup-1,WebServerGroup-2',
@@ -238,6 +251,7 @@ class AutoScalingTest(unittest.TestCase):
         self._stub_lb_reload(['WebServerGroup-0', 'WebServerGroup-1'])
         now = timeutils.utcnow()
         self._stub_meta_expected(now, 'ExactCapacity : 2')
+        self._stub_create(2)
         self.m.ReplayAll()
         resource = self.create_scaling_group(t, stack, 'WebServerGroup')
         stack.resources['WebServerGroup'] = resource
@@ -288,6 +302,7 @@ class AutoScalingTest(unittest.TestCase):
         properties['DesiredCapacity'] = '2'
         properties['Cooldown'] = '60'
         self._stub_lb_reload(['WebServerGroup-0', 'WebServerGroup-1'])
+        self._stub_create(2)
         now = timeutils.utcnow()
         self._stub_meta_expected(now, 'ExactCapacity : 2')
         self.m.ReplayAll()
@@ -322,6 +337,7 @@ class AutoScalingTest(unittest.TestCase):
         # raise by 200%, should work
         self._stub_lb_reload(['WebServerGroup-0', 'WebServerGroup-1',
                               'WebServerGroup-2'], unset=False)
+        self._stub_create(2)
         self._stub_meta_expected(now, 'PercentChangeInCapacity : 200')
         self.m.ReplayAll()
         resource.adjust(200, 'PercentChangeInCapacity')
@@ -341,6 +357,7 @@ class AutoScalingTest(unittest.TestCase):
         self._stub_lb_reload(['WebServerGroup-0', 'WebServerGroup-1'])
         now = timeutils.utcnow()
         self._stub_meta_expected(now, 'ExactCapacity : 2')
+        self._stub_create(2)
         self.m.ReplayAll()
         resource = self.create_scaling_group(t, stack, 'WebServerGroup')
         stack.resources['WebServerGroup'] = resource
@@ -370,6 +387,7 @@ class AutoScalingTest(unittest.TestCase):
         self._stub_lb_reload(['WebServerGroup-0', 'WebServerGroup-1',
                               'WebServerGroup-2'], unset=False)
         self._stub_meta_expected(now, 'PercentChangeInCapacity : 200')
+        self._stub_create(2)
         self.m.ReplayAll()
         resource.adjust(200, 'PercentChangeInCapacity')
         self.assertEqual('WebServerGroup-0,WebServerGroup-1,WebServerGroup-2',
@@ -386,6 +404,7 @@ class AutoScalingTest(unittest.TestCase):
         self._stub_lb_reload(['WebServerGroup-0'])
         now = timeutils.utcnow()
         self._stub_meta_expected(now, 'ExactCapacity : 1')
+        self._stub_create(1)
         self.m.ReplayAll()
         resource = self.create_scaling_group(t, stack, 'WebServerGroup')
         stack.resources['WebServerGroup'] = resource
@@ -394,6 +413,7 @@ class AutoScalingTest(unittest.TestCase):
         # Scale up one
         self._stub_lb_reload(['WebServerGroup-0', 'WebServerGroup-1'])
         self._stub_meta_expected(now, 'ChangeInCapacity : 1', 2)
+        self._stub_create(1)
         self.m.ReplayAll()
         up_policy = self.create_scaling_policy(t, stack,
                                                'WebServerScaleUpPolicy')
@@ -414,6 +434,7 @@ class AutoScalingTest(unittest.TestCase):
         self._stub_lb_reload(['WebServerGroup-0', 'WebServerGroup-1'])
         now = timeutils.utcnow()
         self._stub_meta_expected(now, 'ExactCapacity : 2')
+        self._stub_create(2)
         self.m.ReplayAll()
         resource = self.create_scaling_group(t, stack, 'WebServerGroup')
         stack.resources['WebServerGroup'] = resource
@@ -440,6 +461,7 @@ class AutoScalingTest(unittest.TestCase):
         self._stub_lb_reload(['WebServerGroup-0'])
         now = timeutils.utcnow()
         self._stub_meta_expected(now, 'ExactCapacity : 1')
+        self._stub_create(1)
         self.m.ReplayAll()
         resource = self.create_scaling_group(t, stack, 'WebServerGroup')
         stack.resources['WebServerGroup'] = resource
@@ -448,6 +470,7 @@ class AutoScalingTest(unittest.TestCase):
         # Scale up one
         self._stub_lb_reload(['WebServerGroup-0', 'WebServerGroup-1'])
         self._stub_meta_expected(now, 'ChangeInCapacity : 1', 2)
+        self._stub_create(1)
         self.m.ReplayAll()
         up_policy = self.create_scaling_policy(t, stack,
                                                'WebServerScaleUpPolicy')
@@ -489,6 +512,7 @@ class AutoScalingTest(unittest.TestCase):
         self._stub_lb_reload(['WebServerGroup-0'])
         now = timeutils.utcnow()
         self._stub_meta_expected(now, 'ExactCapacity : 1')
+        self._stub_create(1)
         self.m.ReplayAll()
         resource = self.create_scaling_group(t, stack, 'WebServerGroup')
         stack.resources['WebServerGroup'] = resource
@@ -497,6 +521,7 @@ class AutoScalingTest(unittest.TestCase):
         # Scale up one
         self._stub_lb_reload(['WebServerGroup-0', 'WebServerGroup-1'])
         self._stub_meta_expected(now, 'ChangeInCapacity : 1', 2)
+        self._stub_create(1)
         self.m.ReplayAll()
         up_policy = self.create_scaling_policy(t, stack,
                                                'WebServerScaleUpPolicy')
@@ -520,6 +545,7 @@ class AutoScalingTest(unittest.TestCase):
         self._stub_lb_reload(['WebServerGroup-0', 'WebServerGroup-1',
                               'WebServerGroup-2'], unset=False)
         self._stub_meta_expected(now, 'ChangeInCapacity : 1', 2)
+        self._stub_create(1)
 
         self.m.ReplayAll()
         up_policy.alarm()
@@ -537,6 +563,7 @@ class AutoScalingTest(unittest.TestCase):
         self._stub_lb_reload(['WebServerGroup-0'])
         now = timeutils.utcnow()
         self._stub_meta_expected(now, 'ExactCapacity : 1')
+        self._stub_create(1)
         self.m.ReplayAll()
         resource = self.create_scaling_group(t, stack, 'WebServerGroup')
         stack.resources['WebServerGroup'] = resource
@@ -547,6 +574,7 @@ class AutoScalingTest(unittest.TestCase):
         properties['Cooldown'] = '0'
         self._stub_lb_reload(['WebServerGroup-0', 'WebServerGroup-1'])
         self._stub_meta_expected(now, 'ChangeInCapacity : 1', 2)
+        self._stub_create(1)
         self.m.ReplayAll()
         up_policy = self.create_scaling_policy(t, stack,
                                                'WebServerScaleUpPolicy')
@@ -568,6 +596,7 @@ class AutoScalingTest(unittest.TestCase):
         self._stub_lb_reload(['WebServerGroup-0', 'WebServerGroup-1',
                               'WebServerGroup-2'], unset=False)
         self._stub_meta_expected(now, 'ChangeInCapacity : 1', 2)
+        self._stub_create(1)
 
         self.m.ReplayAll()
         up_policy.alarm()
@@ -585,6 +614,7 @@ class AutoScalingTest(unittest.TestCase):
         self._stub_lb_reload(['WebServerGroup-0'])
         now = timeutils.utcnow()
         self._stub_meta_expected(now, 'ExactCapacity : 1')
+        self._stub_create(1)
         self.m.ReplayAll()
         resource = self.create_scaling_group(t, stack, 'WebServerGroup')
         stack.resources['WebServerGroup'] = resource
@@ -597,6 +627,7 @@ class AutoScalingTest(unittest.TestCase):
         self._stub_lb_reload(['WebServerGroup-0', 'WebServerGroup-1'])
         now = timeutils.utcnow()
         self._stub_meta_expected(now, 'ChangeInCapacity : 1', 2)
+        self._stub_create(1)
         self.m.ReplayAll()
         up_policy = self.create_scaling_policy(t, stack,
                                                'WebServerScaleUpPolicy')
@@ -618,6 +649,7 @@ class AutoScalingTest(unittest.TestCase):
         self._stub_lb_reload(['WebServerGroup-0', 'WebServerGroup-1',
                               'WebServerGroup-2'], unset=False)
         self._stub_meta_expected(now, 'ChangeInCapacity : 1', 2)
+        self._stub_create(1)
 
         self.m.ReplayAll()
         up_policy.alarm()

diff --git a/heat/tests/test_instance_group.py b/heat/tests/test_instance_group.py
index 5086afd0ea90ff8f0e1084aa920abd142054f2d9..ee2d2aae260816a85e9b4bcab468764d940881f6 100644 (file)
--- a/heat/tests/test_instance_group.py
+++ b/heat/tests/test_instance_group.py
@@ -20,9 +20,11 @@ import mox
 
 from nose.plugins.attrib import attr
 
+from heat.tests.v1_1 import fakes
 from heat.common import context
 from heat.common import template_format
 from heat.engine.resources import autoscaling as asc
+from heat.engine.resources import instance
 from heat.engine.resources import loadbalancer
 from heat.engine import parser
 
@@ -31,6 +33,7 @@ from heat.engine import parser
 @attr(speed='fast')
 class InstanceGroupTest(unittest.TestCase):
     def setUp(self):
+        self.fc = fakes.FakeClient()
         self.m = mox.Mox()
         self.m.StubOutWithMock(loadbalancer.LoadBalancer, 'reload')
 
@@ -58,6 +61,11 @@ class InstanceGroupTest(unittest.TestCase):
 
         return stack
 
+    def _stub_create(self, num):
+        self.m.StubOutWithMock(instance.Instance, 'create')
+        for x in range(num):
+            instance.Instance.create().AndReturn(None)
+
     def create_instance_group(self, t, stack, resource_name):
         resource = asc.InstanceGroup(resource_name,
                                      t['Resources'][resource_name],
@@ -73,6 +81,8 @@ class InstanceGroupTest(unittest.TestCase):
         stack = self.parse_stack(t)
 
         # start with min then delete
+        self._stub_create(1)
+        self.m.ReplayAll()
         resource = self.create_instance_group(t, stack, 'JobServerGroup')
 
         self.assertEqual('JobServerGroup', resource.FnGetRefId())
@@ -81,3 +91,22 @@ class InstanceGroupTest(unittest.TestCase):
                          resource.handle_update())
 
         resource.delete()
+
+    def test_missing_image(self):
+
+        t = self.load_template()
+        stack = self.parse_stack(t)
+
+        resource = asc.InstanceGroup('JobServerGroup',
+                                     t['Resources']['JobServerGroup'],
+                                     stack)
+
+        self.m.StubOutWithMock(instance.Instance, 'create')
+        instance.Instance.create().AndReturn('ImageNotFound: bla')
+
+        self.m.ReplayAll()
+
+        self.assertEqual(resource.create(), 'ImageNotFound: bla')
+        self.assertEqual(asc.InstanceGroup.CREATE_FAILED, resource.state)
+
+        self.m.VerifyAll()