]> review.fuel-infra Code Review - openstack-build/heat-build.git/commitdiff
Combined autoscaling and loadbalancer
authorAngus Salkeld <asalkeld@redhat.com>
Fri, 20 Jul 2012 06:08:32 +0000 (16:08 +1000)
committerAngus Salkeld <asalkeld@redhat.com>
Fri, 20 Jul 2012 06:08:44 +0000 (16:08 +1000)
Change-Id: Ib2cff867eb5d9fbee8cedb828e9c35a93436cbe5

heat/engine/autoscaling.py
heat/engine/cloud_watch.py
heat/engine/loadbalancer.py
heat/engine/resource_types.py
heat/engine/resources.py
heat/metadata/api/v1/metadata.py
templates/AutoScalingMultiAZSample.template [new file with mode: 0644]

index ccecf21e0e61a185e248ecf5d4e374b280fbe04e..8146e2e919046f8c997aa2f3f41ffd0dfc02211c 100644 (file)
@@ -104,10 +104,11 @@ class AutoScalingGroup(Resource):
         if new_capacity > capacity:
             # grow
             for x in range(capacity, new_capacity):
-                inst = instance.Instance('%s-%d' % (self.name, x),
+                name = '%s-%d' % (self.name, x)
+                inst = instance.Instance(name,
                                          self.stack.t['Resources'][conf],
                                          self.stack)
-                inst_list.append('%s-%d' % (self.name, x))
+                inst_list.append(name)
                 self.instance_id_set(','.join(inst_list))
                 inst.create()
         else:
@@ -121,6 +122,20 @@ class AutoScalingGroup(Resource):
                 inst_list.remove(victim)
                 self.instance_id_set(','.join(inst_list))
 
+        # notify the LoadBalancer to reload it's config to include
+        # the changes in instances we have just made.
+        if self.properties['LoadBalancerNames']:
+            # convert the list of instance names into a list of instance id's
+            id_list = []
+            for inst_name in inst_list:
+                inst = instance.Instance(inst_name,
+                                         self.stack.t['Resources'][conf],
+                                         self.stack)
+                id_list.append(inst.FnGetRefId())
+
+            for lb in self.properties['LoadBalancerNames']:
+                self.stack[lb].reload(id_list)
+
 
 class LaunchConfiguration(Resource):
     tags_schema = {'Key': {'Type': 'String',
@@ -147,3 +162,31 @@ class LaunchConfiguration(Resource):
 
     def __init__(self, name, json_snippet, stack):
         super(LaunchConfiguration, self).__init__(name, json_snippet, stack)
+
+
+class ScalingPolicy(Resource):
+    properties_schema = {
+        'AutoScalingGroupName': {'Type': 'String',
+                                 'Required': True},
+        'ScalingAdjustment': {'Type': 'Integer',
+                              'Required': True},
+        'AdjustmentType': {'Type': 'String',
+                           'AllowedValues': ['ChangeInCapacity',
+                                             'ExactCapacity',
+                                             'PercentChangeInCapacity'],
+                           'Required': True},
+        'Cooldown': {'Type': 'Integer'},
+    }
+
+    def __init__(self, name, json_snippet, stack):
+        super(ScalingPolicy, self).__init__(name, json_snippet, stack)
+
+    def alarm(self):
+        self.calculate_properties()
+        group = self.stack.resources[self.properties['AutoScalingGroupName']]
+
+        logger.info('%s Alarm, adjusting Group %s by %s' %
+                    (self.name, group.name,
+                     self.properties['ScalingAdjustment']))
+        group.adjust(int(self.properties['ScalingAdjustment']),
+                     self.properties['AdjustmentType'])
index 68966286c6984fc6f4df03c4ee018123dae79570..cdb5c1e0d5ed6df634337ec66ab70828f47f99fd 100644 (file)
@@ -41,6 +41,7 @@ class CloudWatchAlarm(Resource):
                                         'Minimum', 'Maximum']},
         'AlarmActions': {'Type': 'List'},
         'OKActions': {'Type': 'List'},
+        'Dimensions': {'Type': 'List'},
         'InsufficientDataActions': {'Type': 'List'},
         'Threshold': {'Type': 'String'},
         'Units': {'Type': 'String',
index dfe473695b66dcde1c5b12881e0ce377a50f53c5..0bfade85bebc1213442df8d74a6e7a3327c8913a 100644 (file)
@@ -49,7 +49,7 @@ lb_template = '''
         "ComparisonOperator": "GreaterThanThreshold"
       }
     },
-    "LoadBalancerInstance": {
+    "LB_instance": {
       "Type": "AWS::EC2::Instance",
       "Metadata": {
         "AWS::CloudFormation::Init": {
@@ -69,6 +69,39 @@ lb_template = '''
               }
             },
             "files": {
+              "/etc/cfn/cfn-hup.conf" : {
+                "content" : { "Fn::Join" : ["", [
+                  "[main]\\n",
+                  "stack=", { "Ref" : "AWS::StackName" }, "\\n",
+                  "credential-file=/etc/cfn/cfn-credentials\\n",
+                  "region=", { "Ref" : "AWS::Region" }, "\\n",
+                  "interval=60\\n"
+                ]]},
+                "mode"    : "000400",
+                "owner"   : "root",
+                "group"   : "root"
+              },
+              "/etc/cfn/hooks.conf" : {
+                "content": { "Fn::Join" : ["", [
+                  "[cfn-init]\\n",
+                  "triggers=post.update\\n",
+                  "path=Resources.LB_instance.Metadata\\n",
+                  "action=/opt/aws/bin/cfn-init -s ",
+                  { "Ref": "AWS::StackName" },
+                  "    -r LB_instance ",
+                  "    --region ", { "Ref": "AWS::Region" }, "\\n",
+                  "runas=root\\n",
+                  "\\n",
+                  "[reload]\\n",
+                  "triggers=post.update\\n",
+                  "path=Resources.LB_instance.Metadata\\n",
+                  "action=systemctl reload haproxy.service\\n",
+                  "runas=root\\n"
+                ]]},
+                "mode"    : "000400",
+                "owner"   : "root",
+                "group"   : "root"
+              },
               "/etc/haproxy/haproxy.cfg": {
                 "content": "",
                 "mode": "000644",
@@ -79,6 +112,7 @@ lb_template = '''
                 "content" : { "Fn::Join" : ["", [
                 "MAIL=\\"\\"\\n",
                 "\\n",
+                "* * * * * /opt/aws/bin/cfn-hup -f\\n",
                 "* * * * * /opt/aws/bin/cfn-push-stats ",
                 " --watch latency_watcher --haproxy\\n"
                 ]]},
@@ -98,7 +132,9 @@ lb_template = '''
           "#!/bin/bash -v\\n",
           "/opt/aws/bin/cfn-init -s ",
           { "Ref": "AWS::StackName" },
+          "    -r LB_instance ",
           "    --region ", { "Ref": "AWS::Region" }, "\\n",
+          "touch /etc/cfn/cfn-credentials\\n",
           "# install cfn-hup crontab\\n",
           "crontab /tmp/cfn-hup-crontab.txt\\n"
         ]]}}
@@ -108,7 +144,7 @@ lb_template = '''
 
   "Outputs": {
     "PublicIp": {
-      "Value": { "Fn::GetAtt": [ "LoadBalancerInstance", "PublicIp" ] },
+      "Value": { "Fn::GetAtt": [ "LB_instance", "PublicIp" ] },
       "Description": "instance IP"
     }
   }
@@ -145,7 +181,7 @@ class LoadBalancer(stack.Stack):
                    'Required': True},
         'Timeout': {'Type': 'Integer',
                     'Required': True},
-        'UnHealthyTheshold': {'Type': 'Integer',
+        'UnhealthyThreshold': {'Type': 'Integer',
                               'Required': True},
     }
 
@@ -180,7 +216,7 @@ class LoadBalancer(stack.Stack):
         try:
             server = self.nova().servers.get(inst)
         except NotFound as ex:
-            logger.warn('Instance IP address not found (%s)' % str(ex))
+            logger.warn('Instance (%s) not found: %s' % (inst, str(ex)))
         else:
             for n in server.networks:
                 return server.networks[n][0]
@@ -190,7 +226,6 @@ class LoadBalancer(stack.Stack):
     def _haproxy_config(self, templ):
         # initial simplifications:
         # - only one Listener
-        # - static (only use Instances)
         # - only http (no tcp or ssl)
         #
         # option httpchk HEAD /check.txt HTTP/1.0
@@ -234,10 +269,12 @@ class LoadBalancer(stack.Stack):
             option forwardfor
             option httpchk
 '''
+
         servers = []
         n = 1
         for i in self.properties['Instances']:
             ip = self._instance_to_ipaddress(i)
+            logger.debug('haproxy server:%s' % ip)
             servers.append('%sserver server%d %s:%s %s' % (spaces, n,
                                                            ip, inst_port,
                                                            check))
@@ -248,12 +285,32 @@ class LoadBalancer(stack.Stack):
     def handle_create(self):
         templ = json.loads(lb_template)
 
-        md = templ['Resources']['LoadBalancerInstance']['Metadata']
-        files = md['AWS::CloudFormation::Init']['config']['files']
+        if self.properties['Instances']:
+            md = templ['Resources']['LB_instance']['Metadata']
+            files = md['AWS::CloudFormation::Init']['config']['files']
+            cfg = self._haproxy_config(templ)
+            files['/etc/haproxy/haproxy.cfg']['content'] = cfg
+
+        self.create_with_template(templ)
+
+    def reload(self, inst_list):
+        '''
+        re-generate the Metadata
+        save it to the db.
+        rely on the cfn-hup to reconfigure HAProxy
+        '''
+        self.properties['Instances'] = inst_list
+        templ = json.loads(lb_template)
         cfg = self._haproxy_config(templ)
+
+        md = self.nested()['LB_instance'].metadata
+        files = md['AWS::CloudFormation::Init']['config']['files']
         files['/etc/haproxy/haproxy.cfg']['content'] = cfg
 
-        self.create_with_template(templ)
+        self.nested()['LB_instance'].metadata = md
+
+    def FnGetRefId(self):
+        return unicode(self.name)
 
     def FnGetAtt(self, key):
         '''
index 7c2371102b786d3c4859b0f077e35da536cb7298..e12e3f3088762fd813d2705582a83dc33becc589 100644 (file)
@@ -50,6 +50,7 @@ _resource_classes = {
     'HEAT::HA::Restarter': instance.Restarter,
     'AWS::AutoScaling::LaunchConfiguration': autoscaling.LaunchConfiguration,
     'AWS::AutoScaling::AutoScalingGroup': autoscaling.AutoScalingGroup,
+    'AWS::AutoScaling::ScalingPolicy': autoscaling.ScalingPolicy,
 }
 
 
index c6f29a1760ce9f512ffd026a525515a48291532f..b868d65964a488a6a1db821c71a21dec0a6319a1 100644 (file)
@@ -273,6 +273,9 @@ class Resource(object):
         if result:
             return result
 
+        if self.id is None:
+            return
+
         try:
             db_api.resource_get(self.context, self.id).delete()
         except exception.NotFound:
index 5f913079d4404165674037da55f5da8294fed974..5677eb6e7f963f3ece8c6c2deed6fb836abb2f02 100644 (file)
@@ -76,7 +76,7 @@ class MetadataController:
                             'The stack "%s" does not exist.' % stack_name)
             else:
                 return json_error(404,
-                            'The resource "%s" does not exist.' % resource_id)
+                       'The resource "%s" does not exist.' % resource_name)
         return metadata
 
     def update_metadata(self, req, body, stack_id, resource_name):
diff --git a/templates/AutoScalingMultiAZSample.template b/templates/AutoScalingMultiAZSample.template
new file mode 100644 (file)
index 0000000..af95995
--- /dev/null
@@ -0,0 +1,243 @@
+{
+  "AWSTemplateFormatVersion" : "2010-09-09",
+
+  "Description" : "AWS CloudFormation Sample Template",
+
+  "Parameters" : {
+
+    "KeyName" : {
+      "Description" : "Name of an existing EC2 KeyPair to enable SSH access to the instances",
+      "Type" : "String"
+    },
+
+    "InstanceType" : {
+      "Description" : "WebServer EC2 instance type",
+      "Type" : "String",
+      "Default" : "m1.small",
+      "AllowedValues" : [ "t1.micro","m1.small","m1.medium","m1.large"],
+      "ConstraintDescription" : "must be a valid EC2 instance type."
+    },
+
+    "DBName": {
+      "Default": "wordpress",
+      "Description" : "The WordPress database name",
+      "Type": "String",
+      "MinLength": "1",
+      "MaxLength": "64",
+      "AllowedPattern" : "[a-zA-Z][a-zA-Z0-9]*",
+      "ConstraintDescription" : "must begin with a letter and contain only alphanumeric characters."
+    },
+
+    "DBUsername": {
+      "Default": "admin",
+      "NoEcho": "true",
+      "Description" : "The WordPress database admin account username",
+      "Type": "String",
+      "MinLength": "1",
+      "MaxLength": "16",
+      "AllowedPattern" : "[a-zA-Z][a-zA-Z0-9]*",
+      "ConstraintDescription" : "must begin with a letter and contain only alphanumeric characters."
+    },
+
+    "DBPassword": {
+      "Default": "admin",
+      "NoEcho": "true",
+      "Description" : "The WordPress database admin account password",
+      "Type": "String",
+      "MinLength": "1",
+      "MaxLength": "41",
+      "AllowedPattern" : "[a-zA-Z0-9]*",
+      "ConstraintDescription" : "must contain only alphanumeric characters."
+    },
+
+    "DBRootPassword": {
+      "Default": "admin",
+      "NoEcho": "true",
+      "Description" : "Root password for MySQL",
+      "Type": "String",
+      "MinLength": "1",
+      "MaxLength": "41",
+      "AllowedPattern" : "[a-zA-Z0-9]*",
+      "ConstraintDescription" : "must contain only alphanumeric characters."
+    }
+  },
+
+  "Resources" : {
+    "WebServerGroup" : {
+      "Type" : "AWS::AutoScaling::AutoScalingGroup",
+      "Properties" : {
+        "AvailabilityZones" : { "Fn::GetAZs" : ""},
+        "LaunchConfigurationName" : { "Ref" : "LaunchConfig" },
+        "MinSize" : "1",
+        "MaxSize" : "3",
+        "LoadBalancerNames" : [ { "Ref" : "ElasticLoadBalancer" } ]
+      }
+    },
+
+    "WebServerScaleUpPolicy" : {
+      "Type" : "AWS::AutoScaling::ScalingPolicy",
+      "Properties" : {
+        "AdjustmentType" : "ChangeInCapacity",
+        "AutoScalingGroupName" : { "Ref" : "WebServerGroup" },
+        "Cooldown" : "60",
+        "ScalingAdjustment" : "1"
+      }
+    },
+
+    "WebServerScaleDownPolicy" : {
+      "Type" : "AWS::AutoScaling::ScalingPolicy",
+      "Properties" : {
+        "AdjustmentType" : "ChangeInCapacity",
+        "AutoScalingGroupName" : { "Ref" : "WebServerGroup" },
+        "Cooldown" : "60",
+        "ScalingAdjustment" : "-1"
+      }
+    },
+
+    "MEMAlarmHigh": {
+     "Type": "AWS::CloudWatch::Alarm",
+     "Properties": {
+        "AlarmDescription": "Scale-up if MEM > 90% for 10 minutes",
+        "MetricName": "MemoryUtilization",
+        "Namespace": "system/linux",
+        "Statistic": "Average",
+        "Period": "300",
+        "EvaluationPeriods": "2",
+        "Threshold": "90",
+        "AlarmActions": [ { "Ref": "WebServerScaleUpPolicy" } ],
+        "Dimensions": [
+          {
+            "Name": "AutoScalingGroupName",
+            "Value": { "Ref": "WebServerGroup" }
+          }
+        ],
+        "ComparisonOperator": "GreaterThanThreshold"
+      }
+    },
+    "MEMAlarmLow": {
+     "Type": "AWS::CloudWatch::Alarm",
+     "Properties": {
+        "AlarmDescription": "Scale-down if MEM < 70% for 10 minutes",
+        "MetricName": "MemoryUtilization",
+        "Namespace": "system/linux",
+        "Statistic": "Average",
+        "Period": "300",
+        "EvaluationPeriods": "2",
+        "Threshold": "70",
+        "AlarmActions": [ { "Ref": "WebServerScaleDownPolicy" } ],
+        "Dimensions": [
+          {
+            "Name": "AutoScalingGroupName",
+            "Value": { "Ref": "WebServerGroup" }
+          }
+        ],
+        "ComparisonOperator": "LessThanThreshold"
+      }
+    },
+
+    "ElasticLoadBalancer" : {
+      "Type" : "AWS::ElasticLoadBalancing::LoadBalancer",
+      "Properties" : {
+        "AvailabilityZones" : { "Fn::GetAZs" : "" },
+        "Listeners" : [ {
+          "LoadBalancerPort" : "80",
+          "InstancePort" : "80",
+          "Protocol" : "HTTP"
+        } ],
+        "HealthCheck" : {
+          "Target" : "HTTP:80/",
+          "HealthyThreshold" : "3",
+          "UnhealthyThreshold" : "5",
+          "Interval" : "30",
+          "Timeout" : "5"
+        }
+      }
+    },
+
+    "LaunchConfig" : {
+      "Type" : "AWS::AutoScaling::LaunchConfiguration",
+      "Metadata" : {
+        "AWS::CloudFormation::Init" : {
+          "config" : {
+            "files" : {
+              "/tmp/setup.mysql" : {
+                "content" : { "Fn::Join" : ["", [
+                "CREATE DATABASE ", { "Ref" : "DBName" }, ";\n",
+                "GRANT ALL PRIVILEGES ON ", { "Ref" : "DBName" },
+                ".* TO '", { "Ref" : "DBUsername" }, "'@'localhost'\n",
+                "IDENTIFIED BY '", { "Ref" : "DBPassword" }, "';\n",
+                "FLUSH PRIVILEGES;\n",
+                "EXIT\n"
+                  ]]},
+                "mode"  : "000644",
+                "owner" : "root",
+                "group" : "root"
+              },
+              "/tmp/stats-crontab.txt" : {
+                "content" : { "Fn::Join" : ["", [
+                "MAIL=\"\"\n",
+                "\n",
+                "* * * * * /opt/aws/bin/cfn-push-stats --watch ",
+                { "Ref" : "MEMAlarmHigh" }, " ----mem-util\n",
+                "* * * * * /opt/aws/bin/cfn-push-stats --watch ",
+                { "Ref" : "MEMAlarmLow" }, " ----mem-util\n"
+                ]]},
+                "mode"    : "000600",
+                "owner"   : "root",
+                "group"   : "root"
+              }
+            },
+            "packages" : {
+              "yum" : {
+                "cronie"       : [],
+                "mysql"        : [],
+                "mysql-server" : [],
+                "httpd"        : [],
+                "wordpress"    : []
+              }
+            },
+            "services" : {
+              "systemd" : {
+                "mysqld"   : { "enabled" : "true", "ensureRunning" : "true" },
+                "httpd"    : { "enabled" : "true", "ensureRunning" : "true" },
+                "crond"    : { "enabled" : "true", "ensureRunning" : "true" }
+              }
+            }
+          }
+        }
+      },
+      "Properties": {
+        "ImageId" : "F16-x86_64-cfntools",
+        "InstanceType"   : { "Ref" : "InstanceType" },
+        "KeyName"        : { "Ref" : "KeyName" },
+        "UserData"       : { "Fn::Base64" : { "Fn::Join" : ["", [
+          "#!/bin/bash -v\n",
+          "/opt/aws/bin/cfn-init -s ", { "Ref" : "AWS::StackName" },
+          " -r LaunchConfig ",
+          " --region ", { "Ref" : "AWS::Region" }, "\n",
+
+          "# Setup MySQL root password and create a user\n",
+          "mysqladmin -u root password '", { "Ref" : "DBRootPassword" }, "'\n",
+
+          "mysql -u root --password='", { "Ref" : "DBRootPassword" },
+          "' < /tmp/setup.mysql\n",
+
+          "sed --in-place --e s/database_name_here/", { "Ref" : "DBName" },
+          "/ --e s/username_here/", { "Ref" : "DBUsername" },
+          "/ --e s/password_here/", { "Ref" : "DBPassword" },
+          "/ /usr/share/wordpress/wp-config.php\n",
+
+          "# install crontab\n",
+          "crontab /tmp/stats-crontab.txt\n"
+        ]]}}
+      }
+    }
+  },
+
+  "Outputs" : {
+    "URL" : {
+      "Description" : "The URL of the website",
+      "Value" :  { "Fn::Join" : [ "", [ "http://", { "Fn::GetAtt" : [ "ElasticLoadBalancer", "DNSName" ]}]]}
+    }
+  }
+}