From: Steven Hardy Date: Wed, 3 Jul 2013 17:03:57 +0000 (+0100) Subject: engine : add suspend/resume support to watch resource X-Git-Tag: 2014.1~388^2 X-Git-Url: https://review.fuel-infra.org/gitweb?a=commitdiff_plain;h=dfe60926c96139680804bfe0e84dd6b4e75d282a;p=openstack-build%2Fheat-build.git engine : add suspend/resume support to watch resource Add suspend/resume support to CloudWatchAlarm resource and underlying watchrule - this means that when in suspended state watches will not allow new metric data to be added or for alarm evaluation/actions to happen blueprint stack-suspend-resume Change-Id: Ic9a1c1bd86ac8156f260791089a38889eabc6528 --- diff --git a/heat/engine/resources/cloud_watch.py b/heat/engine/resources/cloud_watch.py index 0d2d9ecb..d4d787ef 100644 --- a/heat/engine/resources/cloud_watch.py +++ b/heat/engine/resources/cloud_watch.py @@ -112,6 +112,17 @@ class CloudWatchAlarm(resource.Resource): except exception.NotFound: pass + def handle_suspend(self): + wr = watchrule.WatchRule.load(self.context, + watch_name=self.physical_resource_name()) + wr.state_set(wr.SUSPENDED) + + def handle_resume(self): + wr = watchrule.WatchRule.load(self.context, + watch_name=self.physical_resource_name()) + # Just set to NODATA, which will be re-evaluated next periodic task + wr.state_set(wr.NODATA) + def FnGetRefId(self): return unicode(self.physical_resource_name()) diff --git a/heat/engine/watchrule.py b/heat/engine/watchrule.py index d646b29b..3dc8e983 100644 --- a/heat/engine/watchrule.py +++ b/heat/engine/watchrule.py @@ -30,11 +30,13 @@ class WatchRule(object): WATCH_STATES = ( ALARM, NORMAL, - NODATA + NODATA, + SUSPENDED ) = ( rpc_api.WATCH_STATE_ALARM, rpc_api.WATCH_STATE_OK, - rpc_api.WATCH_STATE_NODATA + rpc_api.WATCH_STATE_NODATA, + rpc_api.WATCH_STATE_SUSPENDED ) ACTION_MAP = {ALARM: 'AlarmActions', NORMAL: 'OKActions', @@ -215,6 +217,8 @@ class WatchRule(object): return fn() def evaluate(self): + if self.state == self.SUSPENDED: + return [] # has enough time progressed to run the rule self.now = timeutils.utcnow() if self.now < (self.last_evaluated + self.timeperiod): @@ -250,6 +254,11 @@ class WatchRule(object): return actions def create_watch_data(self, data): + if self.state == self.SUSPENDED: + logger.debug('Ignoring metric data for %s, SUSPENDED state' + % self.name) + return [] + if self.rule['MetricName'] not in data: # Our simplified cloudwatch implementation only expects a single # Metric associated with each alarm, but some cfn-push-stats @@ -267,6 +276,16 @@ class WatchRule(object): wd = db_api.watch_data_create(None, watch_data) logger.debug('new watch:%s data:%s' % (self.name, str(wd.data))) + def state_set(self, state): + ''' + Persistently store the watch state + ''' + if state not in self.WATCH_STATES: + raise ValueError("Invalid watch state %s" % state) + + self.state = state + self.store() + def set_watch_state(self, state): ''' Temporarily set the watch state, returns list of functions to be diff --git a/heat/rpc/api.py b/heat/rpc/api.py index 23adca6b..529e2e90 100644 --- a/heat/rpc/api.py +++ b/heat/rpc/api.py @@ -117,9 +117,10 @@ WATCH_RULE_KEYS = ( 'Statistic', 'Threshold', 'Unit', 'StackName') WATCH_STATES = ( - WATCH_STATE_OK, WATCH_STATE_ALARM, WATCH_STATE_NODATA + WATCH_STATE_OK, WATCH_STATE_ALARM, WATCH_STATE_NODATA, + WATCH_STATE_SUSPENDED ) = ( - 'NORMAL', 'ALARM', 'NODATA', + 'NORMAL', 'ALARM', 'NODATA', 'SUSPENDED' ) WATCH_DATA_KEYS = ( diff --git a/heat/tests/test_cw_alarm.py b/heat/tests/test_cw_alarm.py index caa341e5..188bff07 100644 --- a/heat/tests/test_cw_alarm.py +++ b/heat/tests/test_cw_alarm.py @@ -18,10 +18,10 @@ import copy from heat.common import template_format from heat.engine.resources import cloud_watch from heat.engine import resource +from heat.engine import watchrule from heat.engine import scheduler from heat.tests.common import HeatTestCase -from heat.tests.utils import setup_dummy_db -from heat.tests.utils import parse_stack +from heat.tests import utils alarm_template = ''' @@ -53,7 +53,7 @@ alarm_template = ''' class CloudWatchAlarmTest(HeatTestCase): def setUp(self): super(CloudWatchAlarmTest, self).setUp() - setup_dummy_db() + utils.setup_dummy_db() def create_alarm(self, t, stack, resource_name): rsrc = cloud_watch.CloudWatchAlarm(resource_name, @@ -76,7 +76,7 @@ class CloudWatchAlarmTest(HeatTestCase): properties['AlarmActions'] = ['a'] properties['Dimensions'] = [{'a': 'v'}] - stack = parse_stack(t) + stack = utils.parse_stack(t) # the watch rule needs a valid stack_id stack.store() @@ -107,7 +107,7 @@ class CloudWatchAlarmTest(HeatTestCase): properties['AlarmActions'] = ['a'] properties['Dimensions'] = [{'a': 'v'}] - stack = parse_stack(t) + stack = utils.parse_stack(t) # the watch rule needs a valid stack_id stack.store() @@ -121,3 +121,30 @@ class CloudWatchAlarmTest(HeatTestCase): rsrc.delete() self.m.VerifyAll() + + def test_suspend_resume(self): + t = template_format.parse(alarm_template) + stack = utils.parse_stack(t) + # the watch rule needs a valid stack_id + stack.store() + + self.m.ReplayAll() + rsrc = self.create_alarm(t, stack, 'MEMAlarmHigh') + scheduler.TaskRunner(rsrc.suspend)() + self.assertEqual(rsrc.state, (rsrc.SUSPEND, rsrc.COMPLETE)) + + wr = watchrule.WatchRule.load( + None, watch_name="test_stack-MEMAlarmHigh") + + self.assertEqual(wr.state, watchrule.WatchRule.SUSPENDED) + + scheduler.TaskRunner(rsrc.resume)() + self.assertEqual(rsrc.state, (rsrc.RESUME, rsrc.COMPLETE)) + + wr = watchrule.WatchRule.load( + None, watch_name="test_stack-MEMAlarmHigh") + + self.assertEqual(wr.state, watchrule.WatchRule.NODATA) + + rsrc.delete() + self.m.VerifyAll() diff --git a/heat/tests/test_watch.py b/heat/tests/test_watch.py index 2ac4cc32..d50c2bc1 100644 --- a/heat/tests/test_watch.py +++ b/heat/tests/test_watch.py @@ -391,6 +391,36 @@ class WatchRuleTest(HeatTestCase): self.assertEqual(self.wr.last_evaluated, now) self.assertEqual(actions, []) + @utils.wr_delete_after + def test_evaluate_suspend(self): + rule = {'EvaluationPeriods': '1', + 'MetricName': 'test_metric', + 'Period': '300', + 'Statistic': 'Maximum', + 'ComparisonOperator': 'GreaterThanOrEqualToThreshold', + 'Threshold': '30'} + + now = timeutils.utcnow() + self.m.StubOutWithMock(timeutils, 'utcnow') + timeutils.utcnow().MultipleTimes().AndReturn(now) + self.m.ReplayAll() + + # Now data breaches Threshold, but we're suspended + last = now - datetime.timedelta(seconds=300) + data = WatchData(35, now - datetime.timedelta(seconds=150)) + self.wr = watchrule.WatchRule(context=self.ctx, + watch_name="testwatch", + rule=rule, + watch_data=[data], + stack_id=self.stack_id, + last_evaluated=last) + + self.wr.state_set(self.wr.SUSPENDED) + + actions = self.wr.evaluate() + self.assertEqual(self.wr.state, self.wr.SUSPENDED) + self.assertEqual(actions, []) + @utils.wr_delete_after def test_rule_actions_alarm_normal(self): rule = {'EvaluationPeriods': '1', @@ -592,6 +622,30 @@ class WatchRuleTest(HeatTestCase): # correctly get a list of all datapoints where watch_rule_id == # watch_rule.id, so leave it as a single-datapoint test for now. + @utils.wr_delete_after + def test_create_watch_data_suspended(self): + rule = {u'EvaluationPeriods': u'1', + u'AlarmDescription': u'test alarm', + u'Period': u'300', + u'ComparisonOperator': u'GreaterThanThreshold', + u'Statistic': u'SampleCount', + u'Threshold': u'2', + u'MetricName': u'CreateDataMetric'} + self.wr = watchrule.WatchRule(context=self.ctx, + watch_name='create_data_test', + stack_id=self.stack_id, rule=rule, + state=watchrule.WatchRule.SUSPENDED) + + self.wr.store() + + data = {u'CreateDataMetric': {"Unit": "Counter", + "Value": "1", + "Dimensions": []}} + self.wr.create_watch_data(data) + + dbwr = db_api.watch_rule_get_by_name(self.ctx, 'create_data_test') + self.assertEqual(dbwr.watch_data, []) + def test_destroy(self): rule = {'EvaluationPeriods': '1', 'MetricName': 'test_metric', @@ -620,6 +674,30 @@ class WatchRuleTest(HeatTestCase): watchrule.WatchRule.load, context=self.ctx, watch_name="testwatch_destroy") + def test_state_set(self): + rule = {'EvaluationPeriods': '1', + 'MetricName': 'test_metric', + 'AlarmActions': ['DummyAction'], + 'Period': '300', + 'Statistic': 'Maximum', + 'ComparisonOperator': 'GreaterThanOrEqualToThreshold', + 'Threshold': '30'} + + last = timeutils.utcnow() + watcher = watchrule.WatchRule(context=self.ctx, + watch_name="testwatch_set_state", + rule=rule, + watch_data=[], + stack_id=self.stack_id, + last_evaluated=last) + + watcher.state_set(watcher.SUSPENDED) + self.assertEqual(watcher.state, watcher.SUSPENDED) + + check = watchrule.WatchRule.load(context=self.ctx, + watch_name="testwatch_set_state") + self.assertEqual(check.state, watchrule.WatchRule.SUSPENDED) + def test_set_watch_state(self): rule = {'EvaluationPeriods': '1', 'MetricName': 'test_metric',