From 0f4c86bd3a13b5f8ff2e57b727709c7ff46aeea4 Mon Sep 17 00:00:00 2001 From: Angus Salkeld Date: Fri, 20 Jul 2012 16:08:32 +1000 Subject: [PATCH] Combined autoscaling and loadbalancer Change-Id: Ib2cff867eb5d9fbee8cedb828e9c35a93436cbe5 --- heat/engine/autoscaling.py | 47 +++- heat/engine/cloud_watch.py | 1 + heat/engine/loadbalancer.py | 73 +++++- heat/engine/resource_types.py | 1 + heat/engine/resources.py | 3 + heat/metadata/api/v1/metadata.py | 2 +- templates/AutoScalingMultiAZSample.template | 243 ++++++++++++++++++++ 7 files changed, 359 insertions(+), 11 deletions(-) create mode 100644 templates/AutoScalingMultiAZSample.template diff --git a/heat/engine/autoscaling.py b/heat/engine/autoscaling.py index ccecf21e..8146e2e9 100644 --- a/heat/engine/autoscaling.py +++ b/heat/engine/autoscaling.py @@ -104,10 +104,11 @@ class AutoScalingGroup(Resource): if new_capacity > capacity: # grow for x in range(capacity, new_capacity): - inst = instance.Instance('%s-%d' % (self.name, x), + name = '%s-%d' % (self.name, x) + inst = instance.Instance(name, self.stack.t['Resources'][conf], self.stack) - inst_list.append('%s-%d' % (self.name, x)) + inst_list.append(name) self.instance_id_set(','.join(inst_list)) inst.create() else: @@ -121,6 +122,20 @@ class AutoScalingGroup(Resource): inst_list.remove(victim) self.instance_id_set(','.join(inst_list)) + # notify the LoadBalancer to reload it's config to include + # the changes in instances we have just made. + if self.properties['LoadBalancerNames']: + # convert the list of instance names into a list of instance id's + id_list = [] + for inst_name in inst_list: + inst = instance.Instance(inst_name, + self.stack.t['Resources'][conf], + self.stack) + id_list.append(inst.FnGetRefId()) + + for lb in self.properties['LoadBalancerNames']: + self.stack[lb].reload(id_list) + class LaunchConfiguration(Resource): tags_schema = {'Key': {'Type': 'String', @@ -147,3 +162,31 @@ class LaunchConfiguration(Resource): def __init__(self, name, json_snippet, stack): super(LaunchConfiguration, self).__init__(name, json_snippet, stack) + + +class ScalingPolicy(Resource): + properties_schema = { + 'AutoScalingGroupName': {'Type': 'String', + 'Required': True}, + 'ScalingAdjustment': {'Type': 'Integer', + 'Required': True}, + 'AdjustmentType': {'Type': 'String', + 'AllowedValues': ['ChangeInCapacity', + 'ExactCapacity', + 'PercentChangeInCapacity'], + 'Required': True}, + 'Cooldown': {'Type': 'Integer'}, + } + + def __init__(self, name, json_snippet, stack): + super(ScalingPolicy, self).__init__(name, json_snippet, stack) + + def alarm(self): + self.calculate_properties() + group = self.stack.resources[self.properties['AutoScalingGroupName']] + + logger.info('%s Alarm, adjusting Group %s by %s' % + (self.name, group.name, + self.properties['ScalingAdjustment'])) + group.adjust(int(self.properties['ScalingAdjustment']), + self.properties['AdjustmentType']) diff --git a/heat/engine/cloud_watch.py b/heat/engine/cloud_watch.py index 68966286..cdb5c1e0 100644 --- a/heat/engine/cloud_watch.py +++ b/heat/engine/cloud_watch.py @@ -41,6 +41,7 @@ class CloudWatchAlarm(Resource): 'Minimum', 'Maximum']}, 'AlarmActions': {'Type': 'List'}, 'OKActions': {'Type': 'List'}, + 'Dimensions': {'Type': 'List'}, 'InsufficientDataActions': {'Type': 'List'}, 'Threshold': {'Type': 'String'}, 'Units': {'Type': 'String', diff --git a/heat/engine/loadbalancer.py b/heat/engine/loadbalancer.py index dfe47369..0bfade85 100644 --- a/heat/engine/loadbalancer.py +++ b/heat/engine/loadbalancer.py @@ -49,7 +49,7 @@ lb_template = ''' "ComparisonOperator": "GreaterThanThreshold" } }, - "LoadBalancerInstance": { + "LB_instance": { "Type": "AWS::EC2::Instance", "Metadata": { "AWS::CloudFormation::Init": { @@ -69,6 +69,39 @@ lb_template = ''' } }, "files": { + "/etc/cfn/cfn-hup.conf" : { + "content" : { "Fn::Join" : ["", [ + "[main]\\n", + "stack=", { "Ref" : "AWS::StackName" }, "\\n", + "credential-file=/etc/cfn/cfn-credentials\\n", + "region=", { "Ref" : "AWS::Region" }, "\\n", + "interval=60\\n" + ]]}, + "mode" : "000400", + "owner" : "root", + "group" : "root" + }, + "/etc/cfn/hooks.conf" : { + "content": { "Fn::Join" : ["", [ + "[cfn-init]\\n", + "triggers=post.update\\n", + "path=Resources.LB_instance.Metadata\\n", + "action=/opt/aws/bin/cfn-init -s ", + { "Ref": "AWS::StackName" }, + " -r LB_instance ", + " --region ", { "Ref": "AWS::Region" }, "\\n", + "runas=root\\n", + "\\n", + "[reload]\\n", + "triggers=post.update\\n", + "path=Resources.LB_instance.Metadata\\n", + "action=systemctl reload haproxy.service\\n", + "runas=root\\n" + ]]}, + "mode" : "000400", + "owner" : "root", + "group" : "root" + }, "/etc/haproxy/haproxy.cfg": { "content": "", "mode": "000644", @@ -79,6 +112,7 @@ lb_template = ''' "content" : { "Fn::Join" : ["", [ "MAIL=\\"\\"\\n", "\\n", + "* * * * * /opt/aws/bin/cfn-hup -f\\n", "* * * * * /opt/aws/bin/cfn-push-stats ", " --watch latency_watcher --haproxy\\n" ]]}, @@ -98,7 +132,9 @@ lb_template = ''' "#!/bin/bash -v\\n", "/opt/aws/bin/cfn-init -s ", { "Ref": "AWS::StackName" }, + " -r LB_instance ", " --region ", { "Ref": "AWS::Region" }, "\\n", + "touch /etc/cfn/cfn-credentials\\n", "# install cfn-hup crontab\\n", "crontab /tmp/cfn-hup-crontab.txt\\n" ]]}} @@ -108,7 +144,7 @@ lb_template = ''' "Outputs": { "PublicIp": { - "Value": { "Fn::GetAtt": [ "LoadBalancerInstance", "PublicIp" ] }, + "Value": { "Fn::GetAtt": [ "LB_instance", "PublicIp" ] }, "Description": "instance IP" } } @@ -145,7 +181,7 @@ class LoadBalancer(stack.Stack): 'Required': True}, 'Timeout': {'Type': 'Integer', 'Required': True}, - 'UnHealthyTheshold': {'Type': 'Integer', + 'UnhealthyThreshold': {'Type': 'Integer', 'Required': True}, } @@ -180,7 +216,7 @@ class LoadBalancer(stack.Stack): try: server = self.nova().servers.get(inst) except NotFound as ex: - logger.warn('Instance IP address not found (%s)' % str(ex)) + logger.warn('Instance (%s) not found: %s' % (inst, str(ex))) else: for n in server.networks: return server.networks[n][0] @@ -190,7 +226,6 @@ class LoadBalancer(stack.Stack): def _haproxy_config(self, templ): # initial simplifications: # - only one Listener - # - static (only use Instances) # - only http (no tcp or ssl) # # option httpchk HEAD /check.txt HTTP/1.0 @@ -234,10 +269,12 @@ class LoadBalancer(stack.Stack): option forwardfor option httpchk ''' + servers = [] n = 1 for i in self.properties['Instances']: ip = self._instance_to_ipaddress(i) + logger.debug('haproxy server:%s' % ip) servers.append('%sserver server%d %s:%s %s' % (spaces, n, ip, inst_port, check)) @@ -248,12 +285,32 @@ class LoadBalancer(stack.Stack): def handle_create(self): templ = json.loads(lb_template) - md = templ['Resources']['LoadBalancerInstance']['Metadata'] - files = md['AWS::CloudFormation::Init']['config']['files'] + if self.properties['Instances']: + md = templ['Resources']['LB_instance']['Metadata'] + files = md['AWS::CloudFormation::Init']['config']['files'] + cfg = self._haproxy_config(templ) + files['/etc/haproxy/haproxy.cfg']['content'] = cfg + + self.create_with_template(templ) + + def reload(self, inst_list): + ''' + re-generate the Metadata + save it to the db. + rely on the cfn-hup to reconfigure HAProxy + ''' + self.properties['Instances'] = inst_list + templ = json.loads(lb_template) cfg = self._haproxy_config(templ) + + md = self.nested()['LB_instance'].metadata + files = md['AWS::CloudFormation::Init']['config']['files'] files['/etc/haproxy/haproxy.cfg']['content'] = cfg - self.create_with_template(templ) + self.nested()['LB_instance'].metadata = md + + def FnGetRefId(self): + return unicode(self.name) def FnGetAtt(self, key): ''' diff --git a/heat/engine/resource_types.py b/heat/engine/resource_types.py index 7c237110..e12e3f30 100644 --- a/heat/engine/resource_types.py +++ b/heat/engine/resource_types.py @@ -50,6 +50,7 @@ _resource_classes = { 'HEAT::HA::Restarter': instance.Restarter, 'AWS::AutoScaling::LaunchConfiguration': autoscaling.LaunchConfiguration, 'AWS::AutoScaling::AutoScalingGroup': autoscaling.AutoScalingGroup, + 'AWS::AutoScaling::ScalingPolicy': autoscaling.ScalingPolicy, } diff --git a/heat/engine/resources.py b/heat/engine/resources.py index c6f29a17..b868d659 100644 --- a/heat/engine/resources.py +++ b/heat/engine/resources.py @@ -273,6 +273,9 @@ class Resource(object): if result: return result + if self.id is None: + return + try: db_api.resource_get(self.context, self.id).delete() except exception.NotFound: diff --git a/heat/metadata/api/v1/metadata.py b/heat/metadata/api/v1/metadata.py index 5f913079..5677eb6e 100644 --- a/heat/metadata/api/v1/metadata.py +++ b/heat/metadata/api/v1/metadata.py @@ -76,7 +76,7 @@ class MetadataController: 'The stack "%s" does not exist.' % stack_name) else: return json_error(404, - 'The resource "%s" does not exist.' % resource_id) + 'The resource "%s" does not exist.' % resource_name) return metadata def update_metadata(self, req, body, stack_id, resource_name): diff --git a/templates/AutoScalingMultiAZSample.template b/templates/AutoScalingMultiAZSample.template new file mode 100644 index 00000000..af959956 --- /dev/null +++ b/templates/AutoScalingMultiAZSample.template @@ -0,0 +1,243 @@ +{ + "AWSTemplateFormatVersion" : "2010-09-09", + + "Description" : "AWS CloudFormation Sample Template", + + "Parameters" : { + + "KeyName" : { + "Description" : "Name of an existing EC2 KeyPair to enable SSH access to the instances", + "Type" : "String" + }, + + "InstanceType" : { + "Description" : "WebServer EC2 instance type", + "Type" : "String", + "Default" : "m1.small", + "AllowedValues" : [ "t1.micro","m1.small","m1.medium","m1.large"], + "ConstraintDescription" : "must be a valid EC2 instance type." + }, + + "DBName": { + "Default": "wordpress", + "Description" : "The WordPress database name", + "Type": "String", + "MinLength": "1", + "MaxLength": "64", + "AllowedPattern" : "[a-zA-Z][a-zA-Z0-9]*", + "ConstraintDescription" : "must begin with a letter and contain only alphanumeric characters." + }, + + "DBUsername": { + "Default": "admin", + "NoEcho": "true", + "Description" : "The WordPress database admin account username", + "Type": "String", + "MinLength": "1", + "MaxLength": "16", + "AllowedPattern" : "[a-zA-Z][a-zA-Z0-9]*", + "ConstraintDescription" : "must begin with a letter and contain only alphanumeric characters." + }, + + "DBPassword": { + "Default": "admin", + "NoEcho": "true", + "Description" : "The WordPress database admin account password", + "Type": "String", + "MinLength": "1", + "MaxLength": "41", + "AllowedPattern" : "[a-zA-Z0-9]*", + "ConstraintDescription" : "must contain only alphanumeric characters." + }, + + "DBRootPassword": { + "Default": "admin", + "NoEcho": "true", + "Description" : "Root password for MySQL", + "Type": "String", + "MinLength": "1", + "MaxLength": "41", + "AllowedPattern" : "[a-zA-Z0-9]*", + "ConstraintDescription" : "must contain only alphanumeric characters." + } + }, + + "Resources" : { + "WebServerGroup" : { + "Type" : "AWS::AutoScaling::AutoScalingGroup", + "Properties" : { + "AvailabilityZones" : { "Fn::GetAZs" : ""}, + "LaunchConfigurationName" : { "Ref" : "LaunchConfig" }, + "MinSize" : "1", + "MaxSize" : "3", + "LoadBalancerNames" : [ { "Ref" : "ElasticLoadBalancer" } ] + } + }, + + "WebServerScaleUpPolicy" : { + "Type" : "AWS::AutoScaling::ScalingPolicy", + "Properties" : { + "AdjustmentType" : "ChangeInCapacity", + "AutoScalingGroupName" : { "Ref" : "WebServerGroup" }, + "Cooldown" : "60", + "ScalingAdjustment" : "1" + } + }, + + "WebServerScaleDownPolicy" : { + "Type" : "AWS::AutoScaling::ScalingPolicy", + "Properties" : { + "AdjustmentType" : "ChangeInCapacity", + "AutoScalingGroupName" : { "Ref" : "WebServerGroup" }, + "Cooldown" : "60", + "ScalingAdjustment" : "-1" + } + }, + + "MEMAlarmHigh": { + "Type": "AWS::CloudWatch::Alarm", + "Properties": { + "AlarmDescription": "Scale-up if MEM > 90% for 10 minutes", + "MetricName": "MemoryUtilization", + "Namespace": "system/linux", + "Statistic": "Average", + "Period": "300", + "EvaluationPeriods": "2", + "Threshold": "90", + "AlarmActions": [ { "Ref": "WebServerScaleUpPolicy" } ], + "Dimensions": [ + { + "Name": "AutoScalingGroupName", + "Value": { "Ref": "WebServerGroup" } + } + ], + "ComparisonOperator": "GreaterThanThreshold" + } + }, + "MEMAlarmLow": { + "Type": "AWS::CloudWatch::Alarm", + "Properties": { + "AlarmDescription": "Scale-down if MEM < 70% for 10 minutes", + "MetricName": "MemoryUtilization", + "Namespace": "system/linux", + "Statistic": "Average", + "Period": "300", + "EvaluationPeriods": "2", + "Threshold": "70", + "AlarmActions": [ { "Ref": "WebServerScaleDownPolicy" } ], + "Dimensions": [ + { + "Name": "AutoScalingGroupName", + "Value": { "Ref": "WebServerGroup" } + } + ], + "ComparisonOperator": "LessThanThreshold" + } + }, + + "ElasticLoadBalancer" : { + "Type" : "AWS::ElasticLoadBalancing::LoadBalancer", + "Properties" : { + "AvailabilityZones" : { "Fn::GetAZs" : "" }, + "Listeners" : [ { + "LoadBalancerPort" : "80", + "InstancePort" : "80", + "Protocol" : "HTTP" + } ], + "HealthCheck" : { + "Target" : "HTTP:80/", + "HealthyThreshold" : "3", + "UnhealthyThreshold" : "5", + "Interval" : "30", + "Timeout" : "5" + } + } + }, + + "LaunchConfig" : { + "Type" : "AWS::AutoScaling::LaunchConfiguration", + "Metadata" : { + "AWS::CloudFormation::Init" : { + "config" : { + "files" : { + "/tmp/setup.mysql" : { + "content" : { "Fn::Join" : ["", [ + "CREATE DATABASE ", { "Ref" : "DBName" }, ";\n", + "GRANT ALL PRIVILEGES ON ", { "Ref" : "DBName" }, + ".* TO '", { "Ref" : "DBUsername" }, "'@'localhost'\n", + "IDENTIFIED BY '", { "Ref" : "DBPassword" }, "';\n", + "FLUSH PRIVILEGES;\n", + "EXIT\n" + ]]}, + "mode" : "000644", + "owner" : "root", + "group" : "root" + }, + "/tmp/stats-crontab.txt" : { + "content" : { "Fn::Join" : ["", [ + "MAIL=\"\"\n", + "\n", + "* * * * * /opt/aws/bin/cfn-push-stats --watch ", + { "Ref" : "MEMAlarmHigh" }, " ----mem-util\n", + "* * * * * /opt/aws/bin/cfn-push-stats --watch ", + { "Ref" : "MEMAlarmLow" }, " ----mem-util\n" + ]]}, + "mode" : "000600", + "owner" : "root", + "group" : "root" + } + }, + "packages" : { + "yum" : { + "cronie" : [], + "mysql" : [], + "mysql-server" : [], + "httpd" : [], + "wordpress" : [] + } + }, + "services" : { + "systemd" : { + "mysqld" : { "enabled" : "true", "ensureRunning" : "true" }, + "httpd" : { "enabled" : "true", "ensureRunning" : "true" }, + "crond" : { "enabled" : "true", "ensureRunning" : "true" } + } + } + } + } + }, + "Properties": { + "ImageId" : "F16-x86_64-cfntools", + "InstanceType" : { "Ref" : "InstanceType" }, + "KeyName" : { "Ref" : "KeyName" }, + "UserData" : { "Fn::Base64" : { "Fn::Join" : ["", [ + "#!/bin/bash -v\n", + "/opt/aws/bin/cfn-init -s ", { "Ref" : "AWS::StackName" }, + " -r LaunchConfig ", + " --region ", { "Ref" : "AWS::Region" }, "\n", + + "# Setup MySQL root password and create a user\n", + "mysqladmin -u root password '", { "Ref" : "DBRootPassword" }, "'\n", + + "mysql -u root --password='", { "Ref" : "DBRootPassword" }, + "' < /tmp/setup.mysql\n", + + "sed --in-place --e s/database_name_here/", { "Ref" : "DBName" }, + "/ --e s/username_here/", { "Ref" : "DBUsername" }, + "/ --e s/password_here/", { "Ref" : "DBPassword" }, + "/ /usr/share/wordpress/wp-config.php\n", + + "# install crontab\n", + "crontab /tmp/stats-crontab.txt\n" + ]]}} + } + } + }, + + "Outputs" : { + "URL" : { + "Description" : "The URL of the website", + "Value" : { "Fn::Join" : [ "", [ "http://", { "Fn::GetAtt" : [ "ElasticLoadBalancer", "DNSName" ]}]]} + } + } +} -- 2.45.2