From 4ee2730522ff7bcebf2c5f6dd917b500f608bc52 Mon Sep 17 00:00:00 2001 From: Zane Bitter Date: Tue, 30 Apr 2013 19:28:32 +0200 Subject: [PATCH] Make Resource.create() a co-routine At the moment, we will just run each co-routine to completion, but in future we can use this to return control to the caller so that resources may be created in parallel. Change-Id: I5c2daf3839e53d855acb73fc4288b5be76794c7f --- heat/engine/parser.py | 9 +++++---- heat/engine/resource.py | 4 ++-- heat/engine/resources/autoscaling.py | 22 +++++++--------------- heat/tests/test_autoscaling.py | 5 +++-- heat/tests/test_dbinstance.py | 3 ++- heat/tests/test_eip.py | 5 +++-- heat/tests/test_instance.py | 7 ++++--- heat/tests/test_instance_group.py | 6 ++++-- heat/tests/test_loadbalancer.py | 3 ++- heat/tests/test_nokey.py | 3 ++- heat/tests/test_quantum.py | 13 +++++++------ heat/tests/test_resource.py | 17 ++++++++++------- heat/tests/test_s3.py | 3 ++- heat/tests/test_swift.py | 3 ++- heat/tests/test_user.py | 12 +++++++----- heat/tests/test_volume.py | 15 +++++++++------ 16 files changed, 71 insertions(+), 59 deletions(-) diff --git a/heat/engine/parser.py b/heat/engine/parser.py index b77c0cd1..f5d45ce7 100644 --- a/heat/engine/parser.py +++ b/heat/engine/parser.py @@ -22,6 +22,7 @@ from heat.engine import dependencies from heat.common import identifier from heat.engine import resource from heat.engine import resources +from heat.engine import scheduler from heat.engine import template from heat.engine import timestamp from heat.engine.parameters import Parameters @@ -279,7 +280,7 @@ class Stack(object): for res in self: if stack_status != self.CREATE_FAILED: try: - res.create() + scheduler.TaskRunner(res.create)() except exception.ResourceFailure as ex: stack_status = self.CREATE_FAILED reason = 'Resource %s failed with: %s' % (str(res), @@ -371,7 +372,7 @@ class Stack(object): self.dependencies = self._get_dependencies( self.resources.itervalues()) try: - self[res.name].create() + scheduler.TaskRunner(res.create)() except exception.ResourceFailure as ex: logger.error("Failed to add %s : %s" % (res.name, str(ex))) @@ -425,7 +426,7 @@ class Stack(object): self.dependencies = self._get_dependencies( self.resources.itervalues()) try: - self[res.name].create() + scheduler.TaskRunner(res.create)() except exception.ResourceFailure as ex: logger.error("Failed to create %s : %s" % (res.name, str(ex))) @@ -548,7 +549,7 @@ class Stack(object): for res in deps: if not failed: try: - res.create() + scheduler.TaskRunner(res.create)() except exception.ResourceFailure as ex: logger.exception('create') failed = True diff --git a/heat/engine/resource.py b/heat/engine/resource.py index f5ff8531..460c26a8 100644 --- a/heat/engine/resource.py +++ b/heat/engine/resource.py @@ -15,7 +15,6 @@ import base64 from datetime import datetime -import eventlet from eventlet.support import greenlets as greenlet from heat.engine import event @@ -319,8 +318,9 @@ class Resource(object): create_data = None if callable(getattr(self, 'handle_create', None)): create_data = self.handle_create() + yield while not self.check_active(create_data): - eventlet.sleep(1) + yield except greenlet.GreenletExit: # Older versions of greenlet erroneously had GreenletExit inherit # from Exception instead of BaseException diff --git a/heat/engine/resources/autoscaling.py b/heat/engine/resources/autoscaling.py index a2233fd3..30d08281 100644 --- a/heat/engine/resources/autoscaling.py +++ b/heat/engine/resources/autoscaling.py @@ -18,6 +18,7 @@ import itertools from heat.common import exception from heat.engine import resource +from heat.engine import scheduler from heat.openstack.common import log as logging from heat.openstack.common import timeutils @@ -84,9 +85,8 @@ class InstanceGroup(resource.Resource): def check_active(self, instances): if instances: - check_active = lambda i: i.check_active(i._create_data, - override=False) - remaining = itertools.dropwhile(check_active, instances) + remaining = itertools.dropwhile(lambda i: i.step(), + instances) instances[:] = list(remaining) if not instances: # When all instances are active, reload the LB config @@ -146,16 +146,6 @@ class InstanceGroup(resource.Resource): def state_set(self, new_state, reason="state changed"): self._store_or_update(new_state, reason) - def check_active(self, create_data=None, override=True): - ''' - By default, report that the instance is active so that we - won't wait for it in create(). - ''' - if override: - self._create_data = create_data - return True - return super(GroupedInstance, self).check_active(create_data) - conf = self.properties['LaunchConfigurationName'] instance_definition = self.stack.t['Resources'][conf] return GroupedInstance(name, instance_definition, self.stack) @@ -188,13 +178,15 @@ class InstanceGroup(resource.Resource): self.resource_id_set(','.join(inst_list)) logger.info('Creating Autoscaling instance %s' % name) + runner = scheduler.TaskRunner(inst.create) + try: - inst.create() + runner.start() except exception.ResourceFailure as ex: if raise_on_error: raise - return inst + return runner if new_capacity > capacity: # grow diff --git a/heat/tests/test_autoscaling.py b/heat/tests/test_autoscaling.py index 069f98f0..ab8576d4 100644 --- a/heat/tests/test_autoscaling.py +++ b/heat/tests/test_autoscaling.py @@ -29,6 +29,7 @@ from heat.engine.resources import autoscaling as asc from heat.engine.resources import loadbalancer from heat.engine.resources import instance from heat.engine import parser +from heat.engine import scheduler from heat.engine.resource import Metadata from heat.openstack.common import timeutils @@ -68,7 +69,7 @@ class AutoScalingTest(unittest.TestCase): t['Resources'][resource_name], stack) self.assertEqual(None, resource.validate()) - self.assertEqual(None, resource.create()) + scheduler.TaskRunner(resource.create)() self.assertEqual(asc.AutoScalingGroup.CREATE_COMPLETE, resource.state) return resource @@ -78,7 +79,7 @@ class AutoScalingTest(unittest.TestCase): stack) self.assertEqual(None, resource.validate()) - self.assertEqual(None, resource.create()) + scheduler.TaskRunner(resource.create)() self.assertEqual(asc.ScalingPolicy.CREATE_COMPLETE, resource.state) return resource diff --git a/heat/tests/test_dbinstance.py b/heat/tests/test_dbinstance.py index f99b8115..e4d018f9 100644 --- a/heat/tests/test_dbinstance.py +++ b/heat/tests/test_dbinstance.py @@ -24,6 +24,7 @@ from heat.common import context from heat.common import exception from heat.common import template_format from heat.engine import parser +from heat.engine import scheduler from heat.engine.resources import dbinstance as dbi @@ -65,7 +66,7 @@ class DBInstanceTest(unittest.TestCase): t['Resources'][resource_name], stack) self.assertEqual(None, resource.validate()) - self.assertEqual(None, resource.create()) + scheduler.TaskRunner(resource.create)() self.assertEqual(dbi.DBInstance.CREATE_COMPLETE, resource.state) return resource diff --git a/heat/tests/test_eip.py b/heat/tests/test_eip.py index 645684da..068a4dd4 100644 --- a/heat/tests/test_eip.py +++ b/heat/tests/test_eip.py @@ -24,6 +24,7 @@ from heat.common import context from heat.common import template_format from heat.engine.resources import eip from heat.engine import parser +from heat.engine import scheduler from heat.tests.v1_1 import fakes @@ -66,7 +67,7 @@ class EIPTest(unittest.TestCase): t['Resources'][resource_name], stack) self.assertEqual(None, resource.validate()) - self.assertEqual(None, resource.create()) + scheduler.TaskRunner(resource.create)() self.assertEqual(eip.ElasticIp.CREATE_COMPLETE, resource.state) return resource @@ -75,7 +76,7 @@ class EIPTest(unittest.TestCase): t['Resources'][resource_name], stack) self.assertEqual(None, resource.validate()) - self.assertEqual(None, resource.create()) + scheduler.TaskRunner(resource.create)() self.assertEqual(eip.ElasticIpAssociation.CREATE_COMPLETE, resource.state) return resource diff --git a/heat/tests/test_instance.py b/heat/tests/test_instance.py index fcc22c86..319e0294 100644 --- a/heat/tests/test_instance.py +++ b/heat/tests/test_instance.py @@ -25,6 +25,7 @@ from heat.tests.v1_1 import fakes from heat.engine.resources import instance as instances from heat.common import template_format from heat.engine import parser +from heat.engine import scheduler from heat.openstack.common import uuidutils @@ -76,7 +77,7 @@ class instancesTest(unittest.TestCase): self.fc.servers.list()[1]) self.m.ReplayAll() - self.assertEqual(instance.create(), None) + scheduler.TaskRunner(instance.create)() # this makes sure the auto increment worked on instance creation self.assertTrue(instance.id > 0) @@ -123,7 +124,7 @@ class instancesTest(unittest.TestCase): self.fc.servers.list()[1]) self.m.ReplayAll() - self.assertEqual(instance.create(), None) + scheduler.TaskRunner(instance.create)() instance.resource_id = 1234 # this makes sure the auto increment worked on instance creation @@ -174,7 +175,7 @@ class instancesTest(unittest.TestCase): self.fc.servers.list()[1]) self.m.ReplayAll() - self.assertEqual(instance.create(), None) + scheduler.TaskRunner(instance.create)() update_template = copy.deepcopy(instance.t) update_template['Metadata'] = {'test': 123} diff --git a/heat/tests/test_instance_group.py b/heat/tests/test_instance_group.py index ecd3c87b..4334e030 100644 --- a/heat/tests/test_instance_group.py +++ b/heat/tests/test_instance_group.py @@ -29,6 +29,7 @@ from heat.engine.resources import autoscaling as asc from heat.engine.resources import instance from heat.engine.resources import loadbalancer from heat.engine import parser +from heat.engine import scheduler @attr(tag=['unit', 'resource']) @@ -80,7 +81,7 @@ class InstanceGroupTest(unittest.TestCase): t['Resources'][resource_name], stack) self.assertEqual(None, resource.validate()) - self.assertEqual(None, resource.create()) + scheduler.TaskRunner(resource.create)() self.assertEqual(asc.InstanceGroup.CREATE_COMPLETE, resource.state) return resource @@ -121,7 +122,8 @@ class InstanceGroupTest(unittest.TestCase): self.m.ReplayAll() - self.assertRaises(exception.ResourceFailure, resource.create) + create = scheduler.TaskRunner(resource.create) + self.assertRaises(exception.ResourceFailure, create) self.assertEqual(asc.InstanceGroup.CREATE_FAILED, resource.state) self.m.VerifyAll() diff --git a/heat/tests/test_loadbalancer.py b/heat/tests/test_loadbalancer.py index 17021015..69103605 100644 --- a/heat/tests/test_loadbalancer.py +++ b/heat/tests/test_loadbalancer.py @@ -27,6 +27,7 @@ from heat.common import config from heat.common import context from heat.common import template_format from heat.engine import parser +from heat.engine import scheduler from heat.engine.resources import instance from heat.engine.resources import user from heat.engine.resources import loadbalancer as lb @@ -89,7 +90,7 @@ class LoadBalancerTest(unittest.TestCase): t['Resources'][resource_name], stack) self.assertEqual(None, resource.validate()) - self.assertEqual(None, resource.create()) + scheduler.TaskRunner(resource.create)() self.assertEqual(lb.LoadBalancer.CREATE_COMPLETE, resource.state) return resource diff --git a/heat/tests/test_nokey.py b/heat/tests/test_nokey.py index 0889af13..009bcaff 100644 --- a/heat/tests/test_nokey.py +++ b/heat/tests/test_nokey.py @@ -24,6 +24,7 @@ from heat.tests.v1_1 import fakes from heat.engine.resources import instance as instances from heat.common import template_format from heat.engine import parser +from heat.engine import scheduler from heat.openstack.common import uuidutils @@ -75,4 +76,4 @@ class nokeyTest(unittest.TestCase): self.fc.servers.list()[1]) self.m.ReplayAll() - self.assertEqual(instance.create(), None) + scheduler.TaskRunner(instance.create)() diff --git a/heat/tests/test_quantum.py b/heat/tests/test_quantum.py index cdaf651e..6ae8e72a 100644 --- a/heat/tests/test_quantum.py +++ b/heat/tests/test_quantum.py @@ -25,6 +25,7 @@ from heat.common import context from heat.common import exception from heat.common import template_format from heat.engine import properties +from heat.engine import scheduler from heat.engine.resources.quantum import net from heat.engine.resources.quantum import floatingip from heat.engine.resources.quantum import port @@ -131,7 +132,7 @@ class QuantumTest(unittest.TestCase): def create_net(self, t, stack, resource_name): resource = net.Net('test_net', t['Resources'][resource_name], stack) - self.assertEqual(None, resource.create()) + scheduler.TaskRunner(resource.create)() self.assertEqual(net.Net.CREATE_COMPLETE, resource.state) return resource @@ -251,7 +252,7 @@ class QuantumFloatingIPTest(unittest.TestCase): stack = self.parse_stack(t) fip = stack['floating_ip'] - self.assertEqual(None, fip.create()) + scheduler.TaskRunner(fip.create)() self.assertEqual(floatingip.FloatingIP.CREATE_COMPLETE, fip.state) fip.validate() @@ -286,7 +287,7 @@ class QuantumFloatingIPTest(unittest.TestCase): stack = self.parse_stack(t) p = stack['port_floating'] - self.assertEqual(None, p.create()) + scheduler.TaskRunner(p.create)() self.assertEqual(port.Port.CREATE_COMPLETE, p.state) p.validate() @@ -324,15 +325,15 @@ class QuantumFloatingIPTest(unittest.TestCase): stack = self.parse_stack(t) fip = stack['floating_ip'] - self.assertEqual(None, fip.create()) + scheduler.TaskRunner(fip.create)() self.assertEqual(floatingip.FloatingIP.CREATE_COMPLETE, fip.state) p = stack['port_floating'] - self.assertEqual(None, p.create()) + scheduler.TaskRunner(p.create)() self.assertEqual(port.Port.CREATE_COMPLETE, p.state) fipa = stack['floating_ip_assoc'] - self.assertEqual(None, fipa.create()) + scheduler.TaskRunner(fipa.create)() self.assertEqual(floatingip.FloatingIPAssociation.CREATE_COMPLETE, fipa.state) diff --git a/heat/tests/test_resource.py b/heat/tests/test_resource.py index fc38b1b0..690f30ae 100644 --- a/heat/tests/test_resource.py +++ b/heat/tests/test_resource.py @@ -21,6 +21,7 @@ from heat.common import context from heat.common import exception from heat.engine import parser from heat.engine import resource +from heat.engine import scheduler from heat.openstack.common import uuidutils from heat.tests import generic_resource as generic_rsrc @@ -229,7 +230,7 @@ class ResourceTest(unittest.TestCase): tmpl = {'Type': 'GenericResourceType', 'Properties': {'Foo': 'abc'}} res = generic_rsrc.GenericResource('test_resource', tmpl, self.stack) - self.assertEqual(None, res.create()) + scheduler.TaskRunner(res.create)() self.assertEqual(res.CREATE_COMPLETE, res.state) def test_create_fail_missing_req_prop(self): @@ -242,7 +243,8 @@ class ResourceTest(unittest.TestCase): res = generic_rsrc.GenericResource(rname, tmpl, self.stack) estr = 'Property error : test_resource: Property Foo not assigned' - self.assertRaises(exception.ResourceFailure, res.create) + create = scheduler.TaskRunner(res.create) + self.assertRaises(exception.ResourceFailure, create) self.assertEqual(res.CREATE_FAILED, res.state) def test_create_fail_prop_typo(self): @@ -255,7 +257,8 @@ class ResourceTest(unittest.TestCase): res = generic_rsrc.GenericResource(rname, tmpl, self.stack) estr = 'Property error : test_resource: Property Foo not assigned' - self.assertRaises(exception.ResourceFailure, res.create) + create = scheduler.TaskRunner(res.create) + self.assertRaises(exception.ResourceFailure, create) self.assertEqual(res.CREATE_FAILED, res.state) def test_update_ok(self): @@ -265,7 +268,7 @@ class ResourceTest(unittest.TestCase): tmpl = {'Type': 'GenericResourceType', 'Properties': {'Foo': 'abc'}} res = generic_rsrc.GenericResource('test_resource', tmpl, self.stack) - self.assertEqual(None, res.create()) + scheduler.TaskRunner(res.create)() self.assertEqual(res.CREATE_COMPLETE, res.state) utmpl = {'Type': 'GenericResourceType', 'Properties': {'Foo': 'xyz'}} @@ -285,7 +288,7 @@ class ResourceTest(unittest.TestCase): tmpl = {'Type': 'GenericResourceType', 'Properties': {'Foo': 'abc'}} res = generic_rsrc.GenericResource('test_resource', tmpl, self.stack) - self.assertEqual(None, res.create()) + scheduler.TaskRunner(res.create)() self.assertEqual(res.CREATE_COMPLETE, res.state) utmpl = {'Type': 'GenericResourceType', 'Properties': {}} @@ -301,7 +304,7 @@ class ResourceTest(unittest.TestCase): tmpl = {'Type': 'GenericResourceType', 'Properties': {'Foo': 'abc'}} res = generic_rsrc.GenericResource('test_resource', tmpl, self.stack) - self.assertEqual(None, res.create()) + scheduler.TaskRunner(res.create)() self.assertEqual(res.CREATE_COMPLETE, res.state) utmpl = {'Type': 'GenericResourceType', 'Properties': {'Food': 'xyz'}} @@ -326,7 +329,7 @@ class MetadataTest(unittest.TestCase): self.stack.store() self.res = generic_rsrc.GenericResource('metadata_resource', tmpl, self.stack) - self.res.create() + scheduler.TaskRunner(self.res.create)() def tearDown(self): self.stack.delete() diff --git a/heat/tests/test_s3.py b/heat/tests/test_s3.py index e4f80e13..24dc5ec1 100644 --- a/heat/tests/test_s3.py +++ b/heat/tests/test_s3.py @@ -26,6 +26,7 @@ from heat.common import template_format from heat.openstack.common.importutils import try_import from heat.engine.resources import s3 from heat.engine import parser +from heat.engine import scheduler from utils import skip_if swiftclient = try_import('swiftclient.client') @@ -70,7 +71,7 @@ class s3Test(unittest.TestCase): resource = s3.S3Bucket('test_resource', t['Resources'][resource_name], stack) - self.assertEqual(None, resource.create()) + scheduler.TaskRunner(resource.create)() self.assertEqual(s3.S3Bucket.CREATE_COMPLETE, resource.state) return resource diff --git a/heat/tests/test_swift.py b/heat/tests/test_swift.py index fe3f28a8..c9cb2c44 100644 --- a/heat/tests/test_swift.py +++ b/heat/tests/test_swift.py @@ -26,6 +26,7 @@ from heat.common import template_format from heat.openstack.common.importutils import try_import from heat.engine.resources import swift from heat.engine import parser +from heat.engine import scheduler from heat.tests.utils import skip_if swiftclient = try_import('swiftclient.client') @@ -72,7 +73,7 @@ class swiftTest(unittest.TestCase): 'test_resource', t['Resources'][resource_name], stack) - self.assertEqual(None, resource.create()) + scheduler.TaskRunner(resource.create)() self.assertEqual(swift.SwiftContainer.CREATE_COMPLETE, resource.state) return resource diff --git a/heat/tests/test_user.py b/heat/tests/test_user.py index b01a0d78..10046b21 100644 --- a/heat/tests/test_user.py +++ b/heat/tests/test_user.py @@ -25,6 +25,7 @@ from heat.common import context from heat.common import exception from heat.common import template_format from heat.engine import parser +from heat.engine import scheduler from heat.engine.resources import user from heat.tests import fakes @@ -74,7 +75,7 @@ class UserTest(unittest.TestCase): t['Resources'][resource_name], stack) self.assertEqual(None, resource.validate()) - self.assertEqual(None, resource.create()) + scheduler.TaskRunner(resource.create)() self.assertEqual(user.User.CREATE_COMPLETE, resource.state) return resource @@ -266,7 +267,7 @@ class AccessKeyTest(unittest.TestCase): t['Resources'][resource_name], stack) self.assertEqual(None, resource.validate()) - self.assertEqual(None, resource.create()) + scheduler.TaskRunner(resource.create)() self.assertEqual(user.AccessKey.CREATE_COMPLETE, resource.state) return resource @@ -332,7 +333,8 @@ class AccessKeyTest(unittest.TestCase): resource = user.AccessKey('HostKeys', t['Resources']['HostKeys'], stack) - self.assertRaises(exception.ResourceFailure, resource.create) + create = scheduler.TaskRunner(resource.create) + self.assertRaises(exception.ResourceFailure, create) self.assertEqual(user.AccessKey.CREATE_FAILED, resource.state) @@ -388,7 +390,7 @@ class AccessPolicyTest(unittest.TestCase): resource = user.AccessPolicy(resource_name, t['Resources'][resource_name], stack) - self.assertEqual(None, resource.create()) + scheduler.TaskRunner(resource.create)() self.assertEqual(user.User.CREATE_COMPLETE, resource.state) def test_accesspolicy_create_ok_empty(self): @@ -400,7 +402,7 @@ class AccessPolicyTest(unittest.TestCase): resource = user.AccessPolicy(resource_name, t['Resources'][resource_name], stack) - self.assertEqual(None, resource.create()) + scheduler.TaskRunner(resource.create)() self.assertEqual(user.User.CREATE_COMPLETE, resource.state) def test_accesspolicy_create_err_notfound(self): diff --git a/heat/tests/test_volume.py b/heat/tests/test_volume.py index 1a69a515..cec9ff4b 100644 --- a/heat/tests/test_volume.py +++ b/heat/tests/test_volume.py @@ -25,6 +25,7 @@ from heat.common import context from heat.common import exception from heat.common import template_format from heat.engine import parser +from heat.engine import scheduler from heat.engine.resources import volume as vol from heat.engine import clients from heat.tests.v1_1 import fakes @@ -74,7 +75,7 @@ class VolumeTest(unittest.TestCase): t['Resources'][resource_name], stack) self.assertEqual(resource.validate(), None) - self.assertEqual(resource.create(), None) + scheduler.TaskRunner(resource.create)() self.assertEqual(resource.state, vol.Volume.CREATE_COMPLETE) return resource @@ -83,7 +84,7 @@ class VolumeTest(unittest.TestCase): t['Resources'][resource_name], stack) self.assertEqual(resource.validate(), None) - self.assertEqual(resource.create(), None) + scheduler.TaskRunner(resource.create)() self.assertEqual(resource.state, vol.VolumeAttachment.CREATE_COMPLETE) return resource @@ -147,7 +148,8 @@ class VolumeTest(unittest.TestCase): resource = vol.Volume('DataVolume', t['Resources']['DataVolume'], stack) - self.assertRaises(exception.ResourceFailure, resource.create) + create = scheduler.TaskRunner(resource.create) + self.assertRaises(exception.ResourceFailure, create) self.m.VerifyAll() @@ -179,12 +181,13 @@ class VolumeTest(unittest.TestCase): t = self.load_template() stack = self.parse_stack(t, stack_name) - self.assertEqual(stack['DataVolume'].create(), None) + scheduler.TaskRunner(stack['DataVolume'].create)() self.assertEqual(fv.status, 'available') resource = vol.VolumeAttachment('MountPoint', t['Resources']['MountPoint'], stack) - self.assertRaises(exception.ResourceFailure, resource.create) + create = scheduler.TaskRunner(resource.create) + self.assertRaises(exception.ResourceFailure, create) self.m.VerifyAll() @@ -221,7 +224,7 @@ class VolumeTest(unittest.TestCase): t = self.load_template() stack = self.parse_stack(t, stack_name) - self.assertEqual(stack['DataVolume'].create(), None) + scheduler.TaskRunner(stack['DataVolume'].create)() self.assertEqual(fv.status, 'available') resource = self.create_attachment(t, stack, 'MountPoint') -- 2.45.2