From 6d32597f067f20797a204ff89e528336288fb8d4 Mon Sep 17 00:00:00 2001 From: Angus Salkeld Date: Mon, 28 Jan 2013 21:57:27 +1100 Subject: [PATCH] Make sure failures in groups (autoscaling & static) are raised. Fixes bug 1104617 Change-Id: Iebd2b0c6a4adf8c252db31348e4dc241c0479765 --- heat/common/exception.py | 4 ++++ heat/engine/resources/autoscaling.py | 28 +++++++++++++++++------- heat/tests/test_autoscaling.py | 32 ++++++++++++++++++++++++++++ heat/tests/test_instance_group.py | 29 +++++++++++++++++++++++++ 4 files changed, 85 insertions(+), 8 deletions(-) diff --git a/heat/common/exception.py b/heat/common/exception.py index 2d9aceeb..16b012d0 100644 --- a/heat/common/exception.py +++ b/heat/common/exception.py @@ -232,3 +232,7 @@ class PhysicalResourceNotFound(OpenstackException): class WatchRuleNotFound(OpenstackException): message = _("The Watch Rule (%(watch_name)s) could not be found.") + + +class NestedResourceFailure(OpenstackException): + message = _("%(message)s") diff --git a/heat/engine/resources/autoscaling.py b/heat/engine/resources/autoscaling.py index 2a5cfe90..9935e4ca 100644 --- a/heat/engine/resources/autoscaling.py +++ b/heat/engine/resources/autoscaling.py @@ -13,8 +13,9 @@ # License for the specific language governing permissions and limitations # under the License. -from heat.engine import resource +from heat.common import exception from heat.engine.resources import instance +from heat.engine import resource from heat.openstack.common import log as logging from heat.openstack.common import timeutils @@ -75,7 +76,7 @@ class InstanceGroup(resource.Resource): # resource_id is a list of resources def handle_create(self): - self.resize(int(self.properties['Size'])) + self.resize(int(self.properties['Size']), raise_on_error=True) def handle_update(self): # TODO(asalkeld) if the only thing that has changed is the size then @@ -108,9 +109,13 @@ class InstanceGroup(resource.Resource): for victim in inst_list: logger.debug('handle_delete %s' % victim) inst = self._make_instance(victim) - inst.destroy() + error_str = inst.destroy() + if error_str is not None: + # try suck out the grouped resouces failure reason + # and re-raise + raise exception.NestedResourceFailure(message=error_str) - def resize(self, new_capacity): + def resize(self, new_capacity, raise_on_error=False): inst_list = [] if self.resource_id is not None: inst_list = sorted(self.resource_id.split(',')) @@ -129,7 +134,12 @@ class InstanceGroup(resource.Resource): inst = self._make_instance(name) inst_list.append(name) self.resource_id_set(','.join(inst_list)) - inst.create() + logger.info('creating inst') + error_str = inst.create() + if raise_on_error and error_str is not None: + # try suck out the grouped resouces failure reason + # and re-raise + raise exception.NestedResourceFailure(message=error_str) else: # shrink (kill largest numbered first) del_list = inst_list[new_capacity:] @@ -192,12 +202,14 @@ class AutoScalingGroup(InstanceGroup, CooldownMixin): else: num_to_create = int(self.properties['MinSize']) - self.adjust(num_to_create, adjustment_type='ExactCapacity') + self.adjust(num_to_create, adjustment_type='ExactCapacity', + raise_on_error=True) def handle_update(self): return self.UPDATE_REPLACE - def adjust(self, adjustment, adjustment_type='ChangeInCapacity'): + def adjust(self, adjustment, adjustment_type='ChangeInCapacity', + raise_on_error=False): if self._cooldown_inprogress(): logger.info("%s NOT performing scaling adjustment, cooldown %s" % (self.name, self.properties['Cooldown'])) @@ -227,7 +239,7 @@ class AutoScalingGroup(InstanceGroup, CooldownMixin): logger.debug('no change in capacity %d' % capacity) return - self.resize(new_capacity) + self.resize(new_capacity, raise_on_error=raise_on_error) self._cooldown_timestamp("%s : %s" % (adjustment_type, adjustment)) diff --git a/heat/tests/test_autoscaling.py b/heat/tests/test_autoscaling.py index 2ea569f0..34a11764 100644 --- a/heat/tests/test_autoscaling.py +++ b/heat/tests/test_autoscaling.py @@ -25,6 +25,7 @@ from heat.common import context from heat.common import template_format from heat.engine.resources import autoscaling as asc from heat.engine.resources import loadbalancer +from heat.engine.resources import instance from heat.engine import parser from heat.engine.resource import Metadata from heat.openstack.common import timeutils @@ -73,12 +74,18 @@ class AutoScalingTest(unittest.TestCase): resource = asc.ScalingPolicy(resource_name, t['Resources'][resource_name], stack) + self.assertEqual(None, resource.validate()) self.assertEqual(None, resource.create()) self.assertEqual(asc.ScalingPolicy.CREATE_COMPLETE, resource.state) return resource + def _stub_create(self, num): + self.m.StubOutWithMock(instance.Instance, 'create') + for x in range(num): + instance.Instance.create().AndReturn(None) + def _stub_lb_reload(self, expected_list, unset=True): if unset: self.m.VerifyAll() @@ -107,6 +114,7 @@ class AutoScalingTest(unittest.TestCase): self._stub_lb_reload(['WebServerGroup-0']) now = timeutils.utcnow() self._stub_meta_expected(now, 'ExactCapacity : 1') + self._stub_create(1) self.m.ReplayAll() resource = self.create_scaling_group(t, stack, 'WebServerGroup') @@ -129,6 +137,7 @@ class AutoScalingTest(unittest.TestCase): 'WebServerGroup-2']) now = timeutils.utcnow() self._stub_meta_expected(now, 'ExactCapacity : 3') + self._stub_create(3) self.m.ReplayAll() resource = self.create_scaling_group(t, stack, 'WebServerGroup') self.assertEqual('WebServerGroup-0,WebServerGroup-1,WebServerGroup-2', @@ -145,6 +154,7 @@ class AutoScalingTest(unittest.TestCase): self._stub_lb_reload(['WebServerGroup-0', 'WebServerGroup-1', 'WebServerGroup-2']) self._stub_meta_expected(now, 'ChangeInCapacity : 2') + self._stub_create(2) self.m.ReplayAll() resource.adjust(2) self.assertEqual('WebServerGroup-0,WebServerGroup-1,WebServerGroup-2', @@ -169,6 +179,7 @@ class AutoScalingTest(unittest.TestCase): self._stub_lb_reload(['WebServerGroup-0', 'WebServerGroup-1']) now = timeutils.utcnow() self._stub_meta_expected(now, 'ExactCapacity : 2') + self._stub_create(2) self.m.ReplayAll() resource = self.create_scaling_group(t, stack, 'WebServerGroup') stack.resources['WebServerGroup'] = resource @@ -200,6 +211,7 @@ class AutoScalingTest(unittest.TestCase): properties = t['Resources']['WebServerGroup']['Properties'] properties['DesiredCapacity'] = '2' self._stub_lb_reload(['WebServerGroup-0', 'WebServerGroup-1']) + self._stub_create(2) now = timeutils.utcnow() self._stub_meta_expected(now, 'ExactCapacity : 2') self.m.ReplayAll() @@ -220,6 +232,7 @@ class AutoScalingTest(unittest.TestCase): self._stub_lb_reload(['WebServerGroup-0', 'WebServerGroup-1', 'WebServerGroup-2']) self._stub_meta_expected(now, 'PercentChangeInCapacity : 200') + self._stub_create(2) self.m.ReplayAll() resource.adjust(200, 'PercentChangeInCapacity') self.assertEqual('WebServerGroup-0,WebServerGroup-1,WebServerGroup-2', @@ -238,6 +251,7 @@ class AutoScalingTest(unittest.TestCase): self._stub_lb_reload(['WebServerGroup-0', 'WebServerGroup-1']) now = timeutils.utcnow() self._stub_meta_expected(now, 'ExactCapacity : 2') + self._stub_create(2) self.m.ReplayAll() resource = self.create_scaling_group(t, stack, 'WebServerGroup') stack.resources['WebServerGroup'] = resource @@ -288,6 +302,7 @@ class AutoScalingTest(unittest.TestCase): properties['DesiredCapacity'] = '2' properties['Cooldown'] = '60' self._stub_lb_reload(['WebServerGroup-0', 'WebServerGroup-1']) + self._stub_create(2) now = timeutils.utcnow() self._stub_meta_expected(now, 'ExactCapacity : 2') self.m.ReplayAll() @@ -322,6 +337,7 @@ class AutoScalingTest(unittest.TestCase): # raise by 200%, should work self._stub_lb_reload(['WebServerGroup-0', 'WebServerGroup-1', 'WebServerGroup-2'], unset=False) + self._stub_create(2) self._stub_meta_expected(now, 'PercentChangeInCapacity : 200') self.m.ReplayAll() resource.adjust(200, 'PercentChangeInCapacity') @@ -341,6 +357,7 @@ class AutoScalingTest(unittest.TestCase): self._stub_lb_reload(['WebServerGroup-0', 'WebServerGroup-1']) now = timeutils.utcnow() self._stub_meta_expected(now, 'ExactCapacity : 2') + self._stub_create(2) self.m.ReplayAll() resource = self.create_scaling_group(t, stack, 'WebServerGroup') stack.resources['WebServerGroup'] = resource @@ -370,6 +387,7 @@ class AutoScalingTest(unittest.TestCase): self._stub_lb_reload(['WebServerGroup-0', 'WebServerGroup-1', 'WebServerGroup-2'], unset=False) self._stub_meta_expected(now, 'PercentChangeInCapacity : 200') + self._stub_create(2) self.m.ReplayAll() resource.adjust(200, 'PercentChangeInCapacity') self.assertEqual('WebServerGroup-0,WebServerGroup-1,WebServerGroup-2', @@ -386,6 +404,7 @@ class AutoScalingTest(unittest.TestCase): self._stub_lb_reload(['WebServerGroup-0']) now = timeutils.utcnow() self._stub_meta_expected(now, 'ExactCapacity : 1') + self._stub_create(1) self.m.ReplayAll() resource = self.create_scaling_group(t, stack, 'WebServerGroup') stack.resources['WebServerGroup'] = resource @@ -394,6 +413,7 @@ class AutoScalingTest(unittest.TestCase): # Scale up one self._stub_lb_reload(['WebServerGroup-0', 'WebServerGroup-1']) self._stub_meta_expected(now, 'ChangeInCapacity : 1', 2) + self._stub_create(1) self.m.ReplayAll() up_policy = self.create_scaling_policy(t, stack, 'WebServerScaleUpPolicy') @@ -414,6 +434,7 @@ class AutoScalingTest(unittest.TestCase): self._stub_lb_reload(['WebServerGroup-0', 'WebServerGroup-1']) now = timeutils.utcnow() self._stub_meta_expected(now, 'ExactCapacity : 2') + self._stub_create(2) self.m.ReplayAll() resource = self.create_scaling_group(t, stack, 'WebServerGroup') stack.resources['WebServerGroup'] = resource @@ -440,6 +461,7 @@ class AutoScalingTest(unittest.TestCase): self._stub_lb_reload(['WebServerGroup-0']) now = timeutils.utcnow() self._stub_meta_expected(now, 'ExactCapacity : 1') + self._stub_create(1) self.m.ReplayAll() resource = self.create_scaling_group(t, stack, 'WebServerGroup') stack.resources['WebServerGroup'] = resource @@ -448,6 +470,7 @@ class AutoScalingTest(unittest.TestCase): # Scale up one self._stub_lb_reload(['WebServerGroup-0', 'WebServerGroup-1']) self._stub_meta_expected(now, 'ChangeInCapacity : 1', 2) + self._stub_create(1) self.m.ReplayAll() up_policy = self.create_scaling_policy(t, stack, 'WebServerScaleUpPolicy') @@ -489,6 +512,7 @@ class AutoScalingTest(unittest.TestCase): self._stub_lb_reload(['WebServerGroup-0']) now = timeutils.utcnow() self._stub_meta_expected(now, 'ExactCapacity : 1') + self._stub_create(1) self.m.ReplayAll() resource = self.create_scaling_group(t, stack, 'WebServerGroup') stack.resources['WebServerGroup'] = resource @@ -497,6 +521,7 @@ class AutoScalingTest(unittest.TestCase): # Scale up one self._stub_lb_reload(['WebServerGroup-0', 'WebServerGroup-1']) self._stub_meta_expected(now, 'ChangeInCapacity : 1', 2) + self._stub_create(1) self.m.ReplayAll() up_policy = self.create_scaling_policy(t, stack, 'WebServerScaleUpPolicy') @@ -520,6 +545,7 @@ class AutoScalingTest(unittest.TestCase): self._stub_lb_reload(['WebServerGroup-0', 'WebServerGroup-1', 'WebServerGroup-2'], unset=False) self._stub_meta_expected(now, 'ChangeInCapacity : 1', 2) + self._stub_create(1) self.m.ReplayAll() up_policy.alarm() @@ -537,6 +563,7 @@ class AutoScalingTest(unittest.TestCase): self._stub_lb_reload(['WebServerGroup-0']) now = timeutils.utcnow() self._stub_meta_expected(now, 'ExactCapacity : 1') + self._stub_create(1) self.m.ReplayAll() resource = self.create_scaling_group(t, stack, 'WebServerGroup') stack.resources['WebServerGroup'] = resource @@ -547,6 +574,7 @@ class AutoScalingTest(unittest.TestCase): properties['Cooldown'] = '0' self._stub_lb_reload(['WebServerGroup-0', 'WebServerGroup-1']) self._stub_meta_expected(now, 'ChangeInCapacity : 1', 2) + self._stub_create(1) self.m.ReplayAll() up_policy = self.create_scaling_policy(t, stack, 'WebServerScaleUpPolicy') @@ -568,6 +596,7 @@ class AutoScalingTest(unittest.TestCase): self._stub_lb_reload(['WebServerGroup-0', 'WebServerGroup-1', 'WebServerGroup-2'], unset=False) self._stub_meta_expected(now, 'ChangeInCapacity : 1', 2) + self._stub_create(1) self.m.ReplayAll() up_policy.alarm() @@ -585,6 +614,7 @@ class AutoScalingTest(unittest.TestCase): self._stub_lb_reload(['WebServerGroup-0']) now = timeutils.utcnow() self._stub_meta_expected(now, 'ExactCapacity : 1') + self._stub_create(1) self.m.ReplayAll() resource = self.create_scaling_group(t, stack, 'WebServerGroup') stack.resources['WebServerGroup'] = resource @@ -597,6 +627,7 @@ class AutoScalingTest(unittest.TestCase): self._stub_lb_reload(['WebServerGroup-0', 'WebServerGroup-1']) now = timeutils.utcnow() self._stub_meta_expected(now, 'ChangeInCapacity : 1', 2) + self._stub_create(1) self.m.ReplayAll() up_policy = self.create_scaling_policy(t, stack, 'WebServerScaleUpPolicy') @@ -618,6 +649,7 @@ class AutoScalingTest(unittest.TestCase): self._stub_lb_reload(['WebServerGroup-0', 'WebServerGroup-1', 'WebServerGroup-2'], unset=False) self._stub_meta_expected(now, 'ChangeInCapacity : 1', 2) + self._stub_create(1) self.m.ReplayAll() up_policy.alarm() diff --git a/heat/tests/test_instance_group.py b/heat/tests/test_instance_group.py index 5086afd0..ee2d2aae 100644 --- a/heat/tests/test_instance_group.py +++ b/heat/tests/test_instance_group.py @@ -20,9 +20,11 @@ import mox from nose.plugins.attrib import attr +from heat.tests.v1_1 import fakes from heat.common import context from heat.common import template_format from heat.engine.resources import autoscaling as asc +from heat.engine.resources import instance from heat.engine.resources import loadbalancer from heat.engine import parser @@ -31,6 +33,7 @@ from heat.engine import parser @attr(speed='fast') class InstanceGroupTest(unittest.TestCase): def setUp(self): + self.fc = fakes.FakeClient() self.m = mox.Mox() self.m.StubOutWithMock(loadbalancer.LoadBalancer, 'reload') @@ -58,6 +61,11 @@ class InstanceGroupTest(unittest.TestCase): return stack + def _stub_create(self, num): + self.m.StubOutWithMock(instance.Instance, 'create') + for x in range(num): + instance.Instance.create().AndReturn(None) + def create_instance_group(self, t, stack, resource_name): resource = asc.InstanceGroup(resource_name, t['Resources'][resource_name], @@ -73,6 +81,8 @@ class InstanceGroupTest(unittest.TestCase): stack = self.parse_stack(t) # start with min then delete + self._stub_create(1) + self.m.ReplayAll() resource = self.create_instance_group(t, stack, 'JobServerGroup') self.assertEqual('JobServerGroup', resource.FnGetRefId()) @@ -81,3 +91,22 @@ class InstanceGroupTest(unittest.TestCase): resource.handle_update()) resource.delete() + + def test_missing_image(self): + + t = self.load_template() + stack = self.parse_stack(t) + + resource = asc.InstanceGroup('JobServerGroup', + t['Resources']['JobServerGroup'], + stack) + + self.m.StubOutWithMock(instance.Instance, 'create') + instance.Instance.create().AndReturn('ImageNotFound: bla') + + self.m.ReplayAll() + + self.assertEqual(resource.create(), 'ImageNotFound: bla') + self.assertEqual(asc.InstanceGroup.CREATE_FAILED, resource.state) + + self.m.VerifyAll() -- 2.45.2