Make a template that demonstrates instance monitoring and restarting.

We use cfn-push-stats to send a heartbeat and if we don't get
it within the set interval we restart the instance.

Other changes to make this work:
- clear the waitcondition when it is deleted.
- set the resource id to None when deleted, else it won't re-create
  properly.
- don't run watch alarms if the stack is not completed.

Change-Id: I5dfc8b372f557cf43379c6c5b7436d1010f83e3c
Signed-off-by: Angus Salkeld <asalkeld@redhat.com>

diff --git a/heat/engine/manager.py b/heat/engine/manager.py
index 5db4d3e65c0c78866688eab8e9a687351be0f116..239bf6793cab4a444d962ae8dac085eaa5279071 100644 (file)
--- a/heat/engine/manager.py
+++ b/heat/engine/manager.py
@@ -408,7 +408,12 @@ class EngineManager(manager.Manager):
     def _periodic_watcher_task(self, context):
 
         now = timeutils.utcnow()
-        wrs = db_api.watch_rule_get_all(context)
+        try:
+            wrs = db_api.watch_rule_get_all(context)
+        except Exception as ex:
+            logger.warn('periodic_task db error (%s) %s' %
+                        ('watch rule removed?', str(ex)))
+            return
         for wr in wrs:
             # has enough time progressed to run the rule
             dt_period = datetime.timedelta(seconds=int(wr.rule['Period']))
@@ -427,22 +432,25 @@ class EngineManager(manager.Manager):
         new_state = watcher.get_alarm_state()
 
         if new_state != wr.state:
-            wr.state = new_state
-            wr.save()
             logger.warn('WATCH: stack:%s, watch_name:%s %s',
                         wr.stack_name, wr.name, new_state)
 
             if not action_map[new_state] in wr.rule:
                 logger.info('no action for new state %s',
                             new_state)
+                wr.state = new_state
+                wr.save()
             else:
                 s = db_api.stack_get_by_name(None, wr.stack_name)
-                if s:
+                if s and s.status in ('CREATE_COMPLETE',
+                                      'UPDATE_COMPLETE'):
                     user_creds = db_api.user_creds_get(s.user_creds_id)
                     ctxt = ctxtlib.RequestContext.from_dict(dict(user_creds))
                     stack = parser.Stack.load(ctxt, s.id)
                     for a in wr.rule[action_map[new_state]]:
                         greenpool.spawn_n(stack[a].alarm)
+                    wr.state = new_state
+                    wr.save()
 
         wr.last_evaluated = now
 

diff --git a/heat/engine/parser.py b/heat/engine/parser.py
index fd1b00c5c8317e750f4f560090f6652dbfee468a..b54d9693a937e12499f77e180c388b911a5d374a 100644 (file)
--- a/heat/engine/parser.py
+++ b/heat/engine/parser.py
@@ -589,9 +589,7 @@ class Stack(object):
 
         for res in reversed(deps):
             try:
-                res.delete()
-                re = db_api.resource_get(self.context, res.id)
-                re.delete()
+                res.destroy()
             except Exception as ex:
                 failed = True
                 logger.error('delete: %s' % str(ex))

diff --git a/heat/engine/resources.py b/heat/engine/resources.py
index 8be4182aa6d1d55ec8413cbde57944c398f91e6e..df9422839665cb765bf7a273a0e6b0a9e60321e6 100644 (file)
--- a/heat/engine/resources.py
+++ b/heat/engine/resources.py
@@ -342,6 +342,8 @@ class Resource(object):
             logger.exception('Delete %s from DB' % str(self))
             return str(ex)
 
+        self.id = None
+
     def instance_id_set(self, inst):
         self.instance_id = inst
         if self.id is not None:

diff --git a/heat/engine/wait_condition.py b/heat/engine/wait_condition.py
index 85f50c083e62d3113521cc2478e4e0492e468ea3..73512130d63eeae27e83e792b520e53b15d970ae 100644 (file)
--- a/heat/engine/wait_condition.py
+++ b/heat/engine/wait_condition.py
@@ -116,6 +116,14 @@ class WaitCondition(resources.Resource):
     def handle_update(self):
         return self.UPDATE_REPLACE
 
+    def handle_delete(self):
+        self._get_handle_resource_id()
+        if self.resource_id is None:
+            return
+
+        handle = self.stack[self.resource_id]
+        handle.metadata = {}
+
     def FnGetAtt(self, key):
         res = None
         if key == 'Data':

diff --git a/templates/WordPress_Single_Instance_With_IHA.template b/templates/WordPress_Single_Instance_With_IHA.template
new file mode 100644 (file)
index 0000000..de61090
--- /dev/null
+++ b/templates/WordPress_Single_Instance_With_IHA.template
@@ -0,0 +1,232 @@
+{
+  "AWSTemplateFormatVersion" : "2010-09-09",
+
+  "Description" : "AWS CloudFormation Sample Template WordPress_Multi_Instance: WordPress is web software you can use to create a beautiful website or blog. This template installs two instances: one running a WordPress deployment and the other using a local MySQL database to store the data.",
+
+  "Parameters" : {
+
+    "KeyName" : {
+      "Description" : "Name of an existing EC2 KeyPair to enable SSH access to the instances",
+      "Type" : "String"
+    },
+
+    "InstanceType" : {
+      "Description" : "WebServer EC2 instance type",
+      "Type" : "String",
+      "Default" : "m1.large",
+      "AllowedValues" : [ "t1.micro", "m1.small", "m1.large", "m1.xlarge", "m2.xlarge", "m2.2xlarge", "m2.4xlarge", "c1.medium", "c1.xlarge", "cc1.4xlarge" ],
+      "ConstraintDescription" : "must be a valid EC2 instance type."
+    },
+
+    "DBName": {
+      "Default": "wordpress",
+      "Description" : "The WordPress database name",
+      "Type": "String",
+      "MinLength": "1",
+      "MaxLength": "64",
+      "AllowedPattern" : "[a-zA-Z][a-zA-Z0-9]*",
+      "ConstraintDescription" : "must begin with a letter and contain only alphanumeric characters."
+    },
+
+    "DBUsername": {
+      "Default": "admin",
+      "NoEcho": "true",
+      "Description" : "The WordPress database admin account username",
+      "Type": "String",
+      "MinLength": "1",
+      "MaxLength": "16",
+      "AllowedPattern" : "[a-zA-Z][a-zA-Z0-9]*",
+      "ConstraintDescription" : "must begin with a letter and contain only alphanumeric characters."
+    },
+
+    "DBPassword": {
+      "Default": "admin",
+      "NoEcho": "true",
+      "Description" : "The WordPress database admin account password",
+      "Type": "String",
+      "MinLength": "1",
+      "MaxLength": "41",
+      "AllowedPattern" : "[a-zA-Z0-9]*",
+      "ConstraintDescription" : "must contain only alphanumeric characters."
+    },
+
+    "DBRootPassword": {
+      "Default": "admin",
+      "NoEcho": "true",
+      "Description" : "Root password for MySQL",
+      "Type": "String",
+      "MinLength": "1",
+      "MaxLength": "41",
+      "AllowedPattern" : "[a-zA-Z0-9]*",
+      "ConstraintDescription" : "must contain only alphanumeric characters."
+    },
+    "LinuxDistribution": {
+      "Default": "F16",
+      "Description" : "Distribution of choice",
+      "Type": "String",
+      "AllowedValues" : [ "F16", "F17", "U10", "RHEL-6.1", "RHEL-6.2", "RHEL-6.3" ]
+    },
+    "HupPollInterval": {
+      "Default": "1",
+      "Description" : "Interval for cfn-hup",
+      "Type": "String"
+    }
+  },
+
+  "Mappings" : {
+    "AWSInstanceType2Arch" : {
+      "t1.micro"    : { "Arch" : "32" },
+      "m1.small"    : { "Arch" : "32" },
+      "m1.large"    : { "Arch" : "64" },
+      "m1.xlarge"   : { "Arch" : "64" },
+      "m2.xlarge"   : { "Arch" : "64" },
+      "m2.2xlarge"  : { "Arch" : "64" },
+      "m2.4xlarge"  : { "Arch" : "64" },
+      "c1.medium"   : { "Arch" : "32" },
+      "c1.xlarge"   : { "Arch" : "64" },
+      "cc1.4xlarge" : { "Arch" : "64" }
+    },
+    "DistroArch2AMI": {
+      "F16"      : { "32" : "F16-i386-cfntools", "64" : "F16-x86_64-cfntools" },
+      "F17"      : { "32" : "F17-i386-cfntools", "64" : "F17-x86_64-cfntools" },
+      "U10"      : { "32" : "U10-i386-cfntools", "64" : "U10-x86_64-cfntools" },
+      "RHEL-6.1" : { "32" : "rhel61-i386-cfntools", "64" : "rhel61-x86_64-cfntools" },
+      "RHEL-6.2" : { "32" : "rhel62-i386-cfntools", "64" : "rhel62-x86_64-cfntools" },
+      "RHEL-6.3" : { "32" : "rhel63-i386-cfntools", "64" : "rhel63-x86_64-cfntools" }
+    }
+  },
+
+  "Resources" : {
+    "WebServerRestartPolicy" : {
+      "Type" : "HEAT::HA::Restarter",
+      "Properties" : {
+        "InstanceId" : { "Ref" : "WikiDatabase" }
+      }
+    },
+    "HeartbeatFailureAlarm": {
+     "Type": "AWS::CloudWatch::Alarm",
+     "Properties": {
+        "AlarmDescription": "Restart the WikiDatabase if we miss a heartbeat",
+        "MetricName": "Heartbeat",
+        "Namespace": "system/linux",
+        "Statistic": "SampleCount",
+        "Period": "60",
+        "EvaluationPeriods": "1",
+        "Threshold": "1",
+        "AlarmActions": [ { "Ref": "WebServerRestartPolicy" } ],
+        "ComparisonOperator": "LessThanThreshold"
+      }
+    },
+    "WikiDatabase": {
+      "Type": "AWS::EC2::Instance",
+      "Metadata" : {
+        "AWS::CloudFormation::Init" : {
+          "config" : {
+            "files" : {
+              "/tmp/cfn-hup-crontab.txt" : {
+                "content" : { "Fn::Join" : ["", [
+                "MAIL=\"\"\n",
+                "\n",
+                "* * * * * /opt/aws/bin/cfn-push-stats ",
+                " --watch HeartbeatFailureAlarm --heartbeat\n"
+                ]]},
+                "mode"    : "000600",
+                "owner"   : "root",
+                "group"   : "root"
+              },
+
+              "/tmp/setup.mysql" : {
+                "content" : { "Fn::Join" : ["", [
+                "CREATE DATABASE ", { "Ref" : "DBName" }, ";\n",
+                "GRANT ALL PRIVILEGES ON ", { "Ref" : "DBName" },
+                ".* TO '", { "Ref" : "DBUsername" }, "'@'localhost'\n",
+                "IDENTIFIED BY '", { "Ref" : "DBPassword" }, "';\n",
+                "FLUSH PRIVILEGES;\n",
+                "EXIT\n"
+                  ]]},
+                "mode"  : "000644",
+                "owner" : "root",
+                "group" : "root"
+              }
+            },
+            "packages" : {
+              "yum" : {
+                "cronie"       : [],
+                "mysql"        : [],
+                "mysql-server" : [],
+                "httpd"        : [],
+                "wordpress"    : []
+              }
+            },
+            "services" : {
+              "systemd" : {
+                "mysqld"   : { "enabled" : "true", "ensureRunning" : "true" },
+                "httpd"    : { "enabled" : "true", "ensureRunning" : "true" },
+                "crond"    : { "enabled" : "true", "ensureRunning" : "true" }
+              }
+            }
+          }
+        }
+      },
+      "Properties": {
+        "ImageId" : { "Fn::FindInMap" : [ "DistroArch2AMI", { "Ref" : "LinuxDistribution" },
+                          { "Fn::FindInMap" : [ "AWSInstanceType2Arch", { "Ref" : "InstanceType" }, "Arch" ] } ] },
+        "InstanceType"   : { "Ref" : "InstanceType" },
+        "KeyName"        : { "Ref" : "KeyName" },
+        "UserData"       : { "Fn::Base64" : { "Fn::Join" : ["", [
+          "#!/bin/bash -v\n",
+          "# Helper function\n",
+          "function error_exit\n",
+          "{\n",
+          "  /opt/aws/bin/cfn-signal -e 1 -r \"$1\" '", { "Ref" : "WaitHandle" }, "'\n",
+          "  exit 1\n",
+          "}\n",
+
+          "/opt/aws/bin/cfn-init -s ", { "Ref" : "AWS::StackName" },
+          " -r WikiDatabase ",
+          " --region ", { "Ref" : "AWS::Region" },
+          " || error_exit 'Failed to run cfn-init'\n",
+
+          "# Setup MySQL root password and create a user\n",
+          "mysqladmin -u root password '", { "Ref" : "DBRootPassword" },
+          "' || error_exit 'Failed to initialize root password'\n",
+
+          "mysql -u root --password='", { "Ref" : "DBRootPassword" },
+          "' < /tmp/setup.mysql || error_exit 'Failed to create database.'\n",
+
+          "sed --in-place --e s/database_name_here/", { "Ref" : "DBName" },
+          "/ --e s/username_here/", { "Ref" : "DBUsername" },
+          "/ --e s/password_here/", { "Ref" : "DBPassword" },
+          "/ /usr/share/wordpress/wp-config.php\n",
+
+          "# install cfn-hup crontab\n",
+          "crontab /tmp/cfn-hup-crontab.txt\n",
+
+          "# All is well so signal success\n",
+          "/opt/aws/bin/cfn-signal -e 0 -r \"Wiki server setup complete\" '",
+          { "Ref" : "WaitHandle" }, "'\n"
+        ]]}}
+      }
+    },
+
+    "WaitHandle" : {
+      "Type" : "AWS::CloudFormation::WaitConditionHandle"
+    },
+
+    "WaitCondition" : {
+      "Type" : "AWS::CloudFormation::WaitCondition",
+      "DependsOn" : "WikiDatabase",
+      "Properties" : {
+        "Handle" : {"Ref" : "WaitHandle"},
+        "Timeout" : "600"
+      }
+    }
+  },
+
+  "Outputs" : {
+    "WebsiteURL" : {
+      "Value" : { "Fn::Join" : ["", ["http://", { "Fn::GetAtt" : [ "WikiDatabase", "PublicIp" ]}, "/wordpress"]] },
+      "Description" : "URL for Wordpress wiki"
+    }
+  }
+}