From 3c6e40f2bb9039dfae60307ea730199973ddc247 Mon Sep 17 00:00:00 2001 From: Angus Salkeld Date: Wed, 25 Jul 2012 21:45:09 +1000 Subject: [PATCH] Make a template that demonstrates instance monitoring and restarting. We use cfn-push-stats to send a heartbeat and if we don't get it within the set interval we restart the instance. Other changes to make this work: - clear the waitcondition when it is deleted. - set the resource id to None when deleted, else it won't re-create properly. - don't run watch alarms if the stack is not completed. Change-Id: I5dfc8b372f557cf43379c6c5b7436d1010f83e3c Signed-off-by: Angus Salkeld --- heat/engine/manager.py | 16 +- heat/engine/parser.py | 4 +- heat/engine/resources.py | 2 + heat/engine/wait_condition.py | 8 + ...ordPress_Single_Instance_With_IHA.template | 232 ++++++++++++++++++ 5 files changed, 255 insertions(+), 7 deletions(-) create mode 100644 templates/WordPress_Single_Instance_With_IHA.template diff --git a/heat/engine/manager.py b/heat/engine/manager.py index 5db4d3e6..239bf679 100644 --- a/heat/engine/manager.py +++ b/heat/engine/manager.py @@ -408,7 +408,12 @@ class EngineManager(manager.Manager): def _periodic_watcher_task(self, context): now = timeutils.utcnow() - wrs = db_api.watch_rule_get_all(context) + try: + wrs = db_api.watch_rule_get_all(context) + except Exception as ex: + logger.warn('periodic_task db error (%s) %s' % + ('watch rule removed?', str(ex))) + return for wr in wrs: # has enough time progressed to run the rule dt_period = datetime.timedelta(seconds=int(wr.rule['Period'])) @@ -427,22 +432,25 @@ class EngineManager(manager.Manager): new_state = watcher.get_alarm_state() if new_state != wr.state: - wr.state = new_state - wr.save() logger.warn('WATCH: stack:%s, watch_name:%s %s', wr.stack_name, wr.name, new_state) if not action_map[new_state] in wr.rule: logger.info('no action for new state %s', new_state) + wr.state = new_state + wr.save() else: s = db_api.stack_get_by_name(None, wr.stack_name) - if s: + if s and s.status in ('CREATE_COMPLETE', + 'UPDATE_COMPLETE'): user_creds = db_api.user_creds_get(s.user_creds_id) ctxt = ctxtlib.RequestContext.from_dict(dict(user_creds)) stack = parser.Stack.load(ctxt, s.id) for a in wr.rule[action_map[new_state]]: greenpool.spawn_n(stack[a].alarm) + wr.state = new_state + wr.save() wr.last_evaluated = now diff --git a/heat/engine/parser.py b/heat/engine/parser.py index fd1b00c5..b54d9693 100644 --- a/heat/engine/parser.py +++ b/heat/engine/parser.py @@ -589,9 +589,7 @@ class Stack(object): for res in reversed(deps): try: - res.delete() - re = db_api.resource_get(self.context, res.id) - re.delete() + res.destroy() except Exception as ex: failed = True logger.error('delete: %s' % str(ex)) diff --git a/heat/engine/resources.py b/heat/engine/resources.py index 8be4182a..df942283 100644 --- a/heat/engine/resources.py +++ b/heat/engine/resources.py @@ -342,6 +342,8 @@ class Resource(object): logger.exception('Delete %s from DB' % str(self)) return str(ex) + self.id = None + def instance_id_set(self, inst): self.instance_id = inst if self.id is not None: diff --git a/heat/engine/wait_condition.py b/heat/engine/wait_condition.py index 85f50c08..73512130 100644 --- a/heat/engine/wait_condition.py +++ b/heat/engine/wait_condition.py @@ -116,6 +116,14 @@ class WaitCondition(resources.Resource): def handle_update(self): return self.UPDATE_REPLACE + def handle_delete(self): + self._get_handle_resource_id() + if self.resource_id is None: + return + + handle = self.stack[self.resource_id] + handle.metadata = {} + def FnGetAtt(self, key): res = None if key == 'Data': diff --git a/templates/WordPress_Single_Instance_With_IHA.template b/templates/WordPress_Single_Instance_With_IHA.template new file mode 100644 index 00000000..de61090c --- /dev/null +++ b/templates/WordPress_Single_Instance_With_IHA.template @@ -0,0 +1,232 @@ +{ + "AWSTemplateFormatVersion" : "2010-09-09", + + "Description" : "AWS CloudFormation Sample Template WordPress_Multi_Instance: WordPress is web software you can use to create a beautiful website or blog. This template installs two instances: one running a WordPress deployment and the other using a local MySQL database to store the data.", + + "Parameters" : { + + "KeyName" : { + "Description" : "Name of an existing EC2 KeyPair to enable SSH access to the instances", + "Type" : "String" + }, + + "InstanceType" : { + "Description" : "WebServer EC2 instance type", + "Type" : "String", + "Default" : "m1.large", + "AllowedValues" : [ "t1.micro", "m1.small", "m1.large", "m1.xlarge", "m2.xlarge", "m2.2xlarge", "m2.4xlarge", "c1.medium", "c1.xlarge", "cc1.4xlarge" ], + "ConstraintDescription" : "must be a valid EC2 instance type." + }, + + "DBName": { + "Default": "wordpress", + "Description" : "The WordPress database name", + "Type": "String", + "MinLength": "1", + "MaxLength": "64", + "AllowedPattern" : "[a-zA-Z][a-zA-Z0-9]*", + "ConstraintDescription" : "must begin with a letter and contain only alphanumeric characters." + }, + + "DBUsername": { + "Default": "admin", + "NoEcho": "true", + "Description" : "The WordPress database admin account username", + "Type": "String", + "MinLength": "1", + "MaxLength": "16", + "AllowedPattern" : "[a-zA-Z][a-zA-Z0-9]*", + "ConstraintDescription" : "must begin with a letter and contain only alphanumeric characters." + }, + + "DBPassword": { + "Default": "admin", + "NoEcho": "true", + "Description" : "The WordPress database admin account password", + "Type": "String", + "MinLength": "1", + "MaxLength": "41", + "AllowedPattern" : "[a-zA-Z0-9]*", + "ConstraintDescription" : "must contain only alphanumeric characters." + }, + + "DBRootPassword": { + "Default": "admin", + "NoEcho": "true", + "Description" : "Root password for MySQL", + "Type": "String", + "MinLength": "1", + "MaxLength": "41", + "AllowedPattern" : "[a-zA-Z0-9]*", + "ConstraintDescription" : "must contain only alphanumeric characters." + }, + "LinuxDistribution": { + "Default": "F16", + "Description" : "Distribution of choice", + "Type": "String", + "AllowedValues" : [ "F16", "F17", "U10", "RHEL-6.1", "RHEL-6.2", "RHEL-6.3" ] + }, + "HupPollInterval": { + "Default": "1", + "Description" : "Interval for cfn-hup", + "Type": "String" + } + }, + + "Mappings" : { + "AWSInstanceType2Arch" : { + "t1.micro" : { "Arch" : "32" }, + "m1.small" : { "Arch" : "32" }, + "m1.large" : { "Arch" : "64" }, + "m1.xlarge" : { "Arch" : "64" }, + "m2.xlarge" : { "Arch" : "64" }, + "m2.2xlarge" : { "Arch" : "64" }, + "m2.4xlarge" : { "Arch" : "64" }, + "c1.medium" : { "Arch" : "32" }, + "c1.xlarge" : { "Arch" : "64" }, + "cc1.4xlarge" : { "Arch" : "64" } + }, + "DistroArch2AMI": { + "F16" : { "32" : "F16-i386-cfntools", "64" : "F16-x86_64-cfntools" }, + "F17" : { "32" : "F17-i386-cfntools", "64" : "F17-x86_64-cfntools" }, + "U10" : { "32" : "U10-i386-cfntools", "64" : "U10-x86_64-cfntools" }, + "RHEL-6.1" : { "32" : "rhel61-i386-cfntools", "64" : "rhel61-x86_64-cfntools" }, + "RHEL-6.2" : { "32" : "rhel62-i386-cfntools", "64" : "rhel62-x86_64-cfntools" }, + "RHEL-6.3" : { "32" : "rhel63-i386-cfntools", "64" : "rhel63-x86_64-cfntools" } + } + }, + + "Resources" : { + "WebServerRestartPolicy" : { + "Type" : "HEAT::HA::Restarter", + "Properties" : { + "InstanceId" : { "Ref" : "WikiDatabase" } + } + }, + "HeartbeatFailureAlarm": { + "Type": "AWS::CloudWatch::Alarm", + "Properties": { + "AlarmDescription": "Restart the WikiDatabase if we miss a heartbeat", + "MetricName": "Heartbeat", + "Namespace": "system/linux", + "Statistic": "SampleCount", + "Period": "60", + "EvaluationPeriods": "1", + "Threshold": "1", + "AlarmActions": [ { "Ref": "WebServerRestartPolicy" } ], + "ComparisonOperator": "LessThanThreshold" + } + }, + "WikiDatabase": { + "Type": "AWS::EC2::Instance", + "Metadata" : { + "AWS::CloudFormation::Init" : { + "config" : { + "files" : { + "/tmp/cfn-hup-crontab.txt" : { + "content" : { "Fn::Join" : ["", [ + "MAIL=\"\"\n", + "\n", + "* * * * * /opt/aws/bin/cfn-push-stats ", + " --watch HeartbeatFailureAlarm --heartbeat\n" + ]]}, + "mode" : "000600", + "owner" : "root", + "group" : "root" + }, + + "/tmp/setup.mysql" : { + "content" : { "Fn::Join" : ["", [ + "CREATE DATABASE ", { "Ref" : "DBName" }, ";\n", + "GRANT ALL PRIVILEGES ON ", { "Ref" : "DBName" }, + ".* TO '", { "Ref" : "DBUsername" }, "'@'localhost'\n", + "IDENTIFIED BY '", { "Ref" : "DBPassword" }, "';\n", + "FLUSH PRIVILEGES;\n", + "EXIT\n" + ]]}, + "mode" : "000644", + "owner" : "root", + "group" : "root" + } + }, + "packages" : { + "yum" : { + "cronie" : [], + "mysql" : [], + "mysql-server" : [], + "httpd" : [], + "wordpress" : [] + } + }, + "services" : { + "systemd" : { + "mysqld" : { "enabled" : "true", "ensureRunning" : "true" }, + "httpd" : { "enabled" : "true", "ensureRunning" : "true" }, + "crond" : { "enabled" : "true", "ensureRunning" : "true" } + } + } + } + } + }, + "Properties": { + "ImageId" : { "Fn::FindInMap" : [ "DistroArch2AMI", { "Ref" : "LinuxDistribution" }, + { "Fn::FindInMap" : [ "AWSInstanceType2Arch", { "Ref" : "InstanceType" }, "Arch" ] } ] }, + "InstanceType" : { "Ref" : "InstanceType" }, + "KeyName" : { "Ref" : "KeyName" }, + "UserData" : { "Fn::Base64" : { "Fn::Join" : ["", [ + "#!/bin/bash -v\n", + "# Helper function\n", + "function error_exit\n", + "{\n", + " /opt/aws/bin/cfn-signal -e 1 -r \"$1\" '", { "Ref" : "WaitHandle" }, "'\n", + " exit 1\n", + "}\n", + + "/opt/aws/bin/cfn-init -s ", { "Ref" : "AWS::StackName" }, + " -r WikiDatabase ", + " --region ", { "Ref" : "AWS::Region" }, + " || error_exit 'Failed to run cfn-init'\n", + + "# Setup MySQL root password and create a user\n", + "mysqladmin -u root password '", { "Ref" : "DBRootPassword" }, + "' || error_exit 'Failed to initialize root password'\n", + + "mysql -u root --password='", { "Ref" : "DBRootPassword" }, + "' < /tmp/setup.mysql || error_exit 'Failed to create database.'\n", + + "sed --in-place --e s/database_name_here/", { "Ref" : "DBName" }, + "/ --e s/username_here/", { "Ref" : "DBUsername" }, + "/ --e s/password_here/", { "Ref" : "DBPassword" }, + "/ /usr/share/wordpress/wp-config.php\n", + + "# install cfn-hup crontab\n", + "crontab /tmp/cfn-hup-crontab.txt\n", + + "# All is well so signal success\n", + "/opt/aws/bin/cfn-signal -e 0 -r \"Wiki server setup complete\" '", + { "Ref" : "WaitHandle" }, "'\n" + ]]}} + } + }, + + "WaitHandle" : { + "Type" : "AWS::CloudFormation::WaitConditionHandle" + }, + + "WaitCondition" : { + "Type" : "AWS::CloudFormation::WaitCondition", + "DependsOn" : "WikiDatabase", + "Properties" : { + "Handle" : {"Ref" : "WaitHandle"}, + "Timeout" : "600" + } + } + }, + + "Outputs" : { + "WebsiteURL" : { + "Value" : { "Fn::Join" : ["", ["http://", { "Fn::GetAtt" : [ "WikiDatabase", "PublicIp" ]}, "/wordpress"]] }, + "Description" : "URL for Wordpress wiki" + } + } +} -- 2.45.2