From: Randall Burt Date: Wed, 10 Jul 2013 18:19:32 +0000 (-0500) Subject: Allow template resource use outside of Environment X-Git-Tag: 2014.1~313^2 X-Git-Url: https://review.fuel-infra.org/gitweb?a=commitdiff_plain;h=69715eb930814270c75e608dae4d3d1d010a41b0;p=openstack-build%2Fheat-build.git Allow template resource use outside of Environment In addition to using TemplateResource as a means to override another Resource implementation, this allows template authors to specify a template url as the Type of resource directly. This allows a user to define new Resource implementations in addition to overriding existing ones via Environments. implements blueprint provider-resource Change-Id: I11cf94dc062fcce0e4ff08765696447fc2411a7f --- diff --git a/heat/engine/attributes.py b/heat/engine/attributes.py index d5e0811c..f09146bb 100644 --- a/heat/engine/attributes.py +++ b/heat/engine/attributes.py @@ -112,8 +112,10 @@ class Attributes(collections.Mapping): @staticmethod def schema_from_outputs(json_snippet): - return dict(("Outputs.%s" % k, v.get("Description")) - for k, v in json_snippet.items()) + if json_snippet: + return dict((k, v.get("Description")) + for k, v in json_snippet.items()) + return {} def __getitem__(self, key): if key not in self: diff --git a/heat/engine/properties.py b/heat/engine/properties.py index 214abcd2..2ccdcde4 100644 --- a/heat/engine/properties.py +++ b/heat/engine/properties.py @@ -40,6 +40,14 @@ SCHEMA_TYPES = ( class Property(object): + + __param_type_map = { + parameters.STRING: STRING, + parameters.NUMBER: NUMBER, + parameters.COMMA_DELIMITED_LIST: LIST, + parameters.JSON: MAP + } + def __init__(self, schema, name=None): self.schema = schema self.name = name @@ -79,6 +87,37 @@ class Property(object): except ValueError: return float(value) + @staticmethod + def schema_from_param(param): + """ + Convert the param specification to a property schema definition + + :param param: parameter definition + :return: a property schema definition for param + """ + if parameters.TYPE not in param: + raise ValueError("Parameter does not define a type for conversion") + ret = { + TYPE: Property.__param_type_map.get(param.get(parameters.TYPE)) + } + if parameters.DEFAULT in param: + ret.update({DEFAULT: param[parameters.DEFAULT]}) + else: + ret.update({REQUIRED: "true"}) + if parameters.VALUES in param: + ret.update({VALUES: param[parameters.VALUES]}) + if parameters.PATTERN in param: + ret.update({PATTERN: param[parameters.PATTERN]}) + if parameters.MAX_LENGTH in param: + ret.update({MAX_LENGTH: param[parameters.MAX_LENGTH]}) + if parameters.MIN_LENGTH in param: + ret.update({MIN_LENGTH: param[parameters.MIN_LENGTH]}) + if parameters.MAX_VALUE in param: + ret.update({MAX_VALUE: param[parameters.MAX_VALUE]}) + if parameters.MIN_VALUE in param: + ret.update({MIN_VALUE: param[parameters.MIN_VALUE]}) + return ret + def _validate_integer(self, value): if value is None: value = self.has_default() and self.default() or 0 @@ -204,6 +243,20 @@ class Properties(collections.Mapping): else: self.error_prefix = parent_name + ': ' + @staticmethod + def schema_from_params(params_snippet): + """ + Convert a template snippet that defines parameters + into a properties schema + + :param params_snippet: parameter definition from a template + :returns: an equivalent properties schema for the specified params + """ + if params_snippet: + return dict((k, Property.schema_from_param(v)) for k, v + in params_snippet.items()) + return {} + def validate(self, with_value=True): for (key, prop) in self.props.items(): if with_value: diff --git a/heat/engine/resources/stack.py b/heat/engine/resources/stack.py index ed0cf8f5..4960f49b 100644 --- a/heat/engine/resources/stack.py +++ b/heat/engine/resources/stack.py @@ -13,6 +13,7 @@ # License for the specific language governing permissions and limitations # under the License. +from heat.common import exception from heat.common import template_format from heat.common import urlfetch from heat.engine import stack_resource @@ -48,6 +49,12 @@ class NestedStack(stack_resource.StackResource): def handle_delete(self): self.delete_nested() + def FnGetAtt(self, key): + if key and not key.startswith('Outputs.'): + raise exception.InvalidTemplateAttribute(resource=self.name, + key=key) + return self.get_output(key.partition('.')[-1]) + def FnGetRefId(self): return self.nested().identifier().arn() diff --git a/heat/engine/resources/template_resource.py b/heat/engine/resources/template_resource.py index 72e07543..fd3ce62b 100644 --- a/heat/engine/resources/template_resource.py +++ b/heat/engine/resources/template_resource.py @@ -13,10 +13,14 @@ # License for the specific language governing permissions and limitations # under the License. +from requests import exceptions + +from heat.common import template_format +from heat.common import urlfetch +from heat.engine import attributes +from heat.engine import properties from heat.engine import resource from heat.engine import stack_resource -from heat.engine import properties -from heat.common import template_format from heat.openstack.common import log as logging @@ -24,20 +28,41 @@ logger = logging.getLogger(__name__) class TemplateResource(stack_resource.StackResource): - '''A Nested Stack Resource representing another Resource.''' + ''' + A resource implemented by a nested stack. + + This implementation passes resource properties as parameters to the nested + stack. Outputs of the nested stack are exposed as attributes of this + resource. + ''' + def __init__(self, name, json_snippet, stack): self.template_name = stack.env.get_resource_type(json_snippet['Type'], name) + self._parsed_nested = None + self.stack = stack # on purpose don't pass in the environment so we get - # the official/facade class to copy it's schema. + # the official/facade class in case we need to copy it's schema. cls_facade = resource.get_class(json_snippet['Type']) - self.properties_schema = cls_facade.properties_schema - self.attributes_schema = cls_facade.attributes_schema + # if we're not overriding via the environment, mirror the template as + # a new resource + if cls_facade == self.__class__: + self.properties_schema = (properties.Properties + .schema_from_params(self.parsed_nested.get('Parameters'))) + self.attributes_schema = (attributes.Attributes + .schema_from_outputs(self.parsed_nested.get('Outputs'))) + # otherwise we are overriding a resource type via the environment + # and should mimic that type + else: + self.properties_schema = cls_facade.properties_schema + self.attributes_schema = cls_facade.attributes_schema super(TemplateResource, self).__init__(name, json_snippet, stack) def _to_parameters(self): - '''Convert CommaDelimitedList to List.''' + ''' + :return: parameter values for our nested stack based on our properties + ''' params = {} for n, v in iter(self.properties.props.items()): if not v.implemented(): @@ -54,17 +79,38 @@ class TemplateResource(stack_resource.StackResource): return params - def handle_create(self): - template_data = self.stack.t.files.get(self.template_name) - template = template_format.parse(template_data) + @property + def parsed_nested(self): + if not self._parsed_nested: + self._parsed_nested = template_format.parse(self.template_data) + return self._parsed_nested - return self.create_with_template(template, + @property + def template_data(self): + t_data = self.stack.t.files.get(self.template_name) + if not t_data and self.template_name.endswith((".yaml", ".template")): + try: + t_data = urlfetch.get(self.template_name) + except (exceptions.RequestException, IOError) as r_exc: + raise ValueError("Could not fetch remote template '%s': %s" % + (self.template_name, str(r_exc))) + else: + # TODO(Randall) Whoops, misunderstanding on my part; this + # doesn't actually persist to the db like I thought. + # Find a better way + self.stack.t.files[self.template_name] = t_data + return t_data + + def handle_create(self): + return self.create_with_template(self.parsed_nested, self._to_parameters()) def handle_delete(self): self.delete_nested() def FnGetRefId(self): + if not self.nested(): + return unicode(self.name) return self.nested().identifier().arn() diff --git a/heat/engine/stack_resource.py b/heat/engine/stack_resource.py index 0c4bc1d3..e322b240 100644 --- a/heat/engine/stack_resource.py +++ b/heat/engine/stack_resource.py @@ -33,7 +33,6 @@ class StackResource(resource.Resource): def __init__(self, name, json_snippet, stack): super(StackResource, self).__init__(name, json_snippet, stack) - self._outputs_to_attribs(json_snippet) self._nested = None def _outputs_to_attribs(self, json_snippet): @@ -161,12 +160,9 @@ class StackResource(resource.Resource): if stack is None: return None if op not in stack.outputs: - raise exception.InvalidTemplateAttribute( - resource=self.name, key=op) - + raise exception.InvalidTemplateAttribute(resource=self.name, + key=op) return stack.output(op) def _resolve_attribute(self, name): - if name.startswith('Outputs.'): - name = name.partition('.')[-1] return unicode(self.get_output(name)) diff --git a/heat/tests/templates/README b/heat/tests/templates/README index 2c2716a2..4c4c92ee 100644 --- a/heat/tests/templates/README +++ b/heat/tests/templates/README @@ -1,6 +1,6 @@ -These templates are required by test_template_format, where we don't want to -use a minimal template snippet (we want ideally to test the maximum possible -syntax to prove the format conversion works) +These templates are required by test_template_format and test_provider_template +where we don't want to use a minimal template snippet (we want ideally to test +the maximum possible syntax to prove the format conversion works) In general, tests should not depend on these templates, inline minimal template snippets are preferred. diff --git a/heat/tests/test_nested_stack.py b/heat/tests/test_nested_stack.py index 4c6667e6..5b7f2f8c 100644 --- a/heat/tests/test_nested_stack.py +++ b/heat/tests/test_nested_stack.py @@ -88,6 +88,10 @@ Outputs: self.assertEqual('bar', rsrc.FnGetAtt('Outputs.Foo')) self.assertRaises( exception.InvalidTemplateAttribute, rsrc.FnGetAtt, 'Foo') + self.assertRaises( + exception.InvalidTemplateAttribute, rsrc.FnGetAtt, 'Outputs.Bar') + self.assertRaises( + exception.InvalidTemplateAttribute, rsrc.FnGetAtt, 'Bar') rsrc.delete() self.assertTrue(rsrc.FnGetRefId().startswith(arn_prefix)) diff --git a/heat/tests/test_properties.py b/heat/tests/test_properties.py index f2eea9ba..c4788081 100644 --- a/heat/tests/test_properties.py +++ b/heat/tests/test_properties.py @@ -355,6 +355,26 @@ class PropertyTest(testtools.TestCase): p = properties.Property({'Type': 'List', 'Schema': list_schema}) self.assertRaises(TypeError, p.validate_data, [42, 'fish']) + def test_schema_from_param(self): + param = { + "Description": "WebServer EC2 instance type", + "Type": "String", + "Default": "m1.large", + "AllowedValues": ["t1.micro", "m1.small", "m1.large", "m1.xlarge", + "m2.xlarge", "m2.2xlarge", "m2.4xlarge", + "c1.medium", "c1.xlarge", "cc1.4xlarge"], + "ConstraintDescription": "must be a valid EC2 instance type." + } + expected = { + 'Default': 'm1.large', + 'Type': 'String', + 'AllowedValues': ['t1.micro', 'm1.small', 'm1.large', 'm1.xlarge', + 'm2.xlarge', 'm2.2xlarge', 'm2.4xlarge', + 'c1.medium', 'c1.xlarge', 'cc1.4xlarge'] + } + self.assertEqual(expected, + properties.Property.schema_from_param(param)) + class PropertiesTest(testtools.TestCase): def setUp(self): @@ -477,6 +497,156 @@ class PropertiesTest(testtools.TestCase): props = properties.Properties(schema, {'foo': None}) self.assertEqual(['one', 'two'], props['foo']) + def test_schema_from_params(self): + params_snippet = { + "DBUsername": { + "Type": "String", + "Description": "The WordPress database admin account username", + "Default": "admin", + "MinLength": "1", + "AllowedPattern": "[a-zA-Z][a-zA-Z0-9]*", + "NoEcho": "true", + "MaxLength": "16", + "ConstraintDescription": ("must begin with a letter and " + "contain only alphanumeric " + "characters.") + }, + "KeyName": { + "Type": "String", + "Description": ("Name of an existing EC2 KeyPair to enable " + "SSH access to the instances") + }, + "LinuxDistribution": { + "Default": "F17", + "Type": "String", + "Description": "Distribution of choice", + "AllowedValues": [ + "F18", + "F17", + "U10", + "RHEL-6.1", + "RHEL-6.2", + "RHEL-6.3" + ] + }, + "DBPassword": { + "Type": "String", + "Description": "The WordPress database admin account password", + "Default": "admin", + "MinLength": "1", + "AllowedPattern": "[a-zA-Z0-9]*", + "NoEcho": "true", + "MaxLength": "41", + "ConstraintDescription": ("must contain only alphanumeric " + "characters.") + }, + "DBName": { + "AllowedPattern": "[a-zA-Z][a-zA-Z0-9]*", + "Type": "String", + "Description": "The WordPress database name", + "MaxLength": "64", + "Default": "wordpress", + "MinLength": "1", + "ConstraintDescription": ("must begin with a letter and " + "contain only alphanumeric " + "characters.") + }, + "InstanceType": { + "Default": "m1.large", + "Type": "String", + "ConstraintDescription": "must be a valid EC2 instance type.", + "Description": "WebServer EC2 instance type", + "AllowedValues": [ + "t1.micro", + "m1.small", + "m1.large", + "m1.xlarge", + "m2.xlarge", + "m2.2xlarge", + "m2.4xlarge", + "c1.medium", + "c1.xlarge", + "cc1.4xlarge" + ] + }, + "DBRootPassword": { + "Type": "String", + "Description": "Root password for MySQL", + "Default": "admin", + "MinLength": "1", + "AllowedPattern": "[a-zA-Z0-9]*", + "NoEcho": "true", + "MaxLength": "41", + "ConstraintDescription": ("must contain only alphanumeric " + "characters.") + } + } + expected = { + "DBUsername": { + "Default": "admin", + "AllowedPattern": "[a-zA-Z][a-zA-Z0-9]*", + "MaxLength": "16", + "Type": "String", + "MinLength": "1" + }, + "LinuxDistribution": { + "Default": "F17", + "Type": "String", + "AllowedValues": [ + "F18", + "F17", + "U10", + "RHEL-6.1", + "RHEL-6.2", + "RHEL-6.3" + ] + }, + "InstanceType": { + "Default": "m1.large", + "Type": "String", + "AllowedValues": [ + "t1.micro", + "m1.small", + "m1.large", + "m1.xlarge", + "m2.xlarge", + "m2.2xlarge", + "m2.4xlarge", + "c1.medium", + "c1.xlarge", + "cc1.4xlarge" + ] + }, + "DBRootPassword": { + "Default": "admin", + "AllowedPattern": "[a-zA-Z0-9]*", + "MaxLength": "41", + "Type": "String", + "MinLength": "1" + }, + "KeyName": { + "Required": "true", + "Type": "String" + }, + "DBPassword": { + "Default": "admin", + "AllowedPattern": "[a-zA-Z0-9]*", + "MaxLength": "41", + "Type": "String", + "MinLength": "1" + }, + "DBName": { + "Default": "wordpress", + "AllowedPattern": "[a-zA-Z][a-zA-Z0-9]*", + "MaxLength": "64", + "Type": "String", + "MinLength": "1" + } + } + self.assertEqual(expected, + (properties.Properties + .schema_from_params(params_snippet))) + class PropertiesValidationTest(testtools.TestCase): def test_required(self): diff --git a/heat/tests/test_provider_template.py b/heat/tests/test_provider_template.py index 20ad7f8b..5e9957c4 100644 --- a/heat/tests/test_provider_template.py +++ b/heat/tests/test_provider_template.py @@ -12,6 +12,10 @@ # License for the specific language governing permissions and limitations # under the License. +import os +import yaml + +from heat.common import urlfetch from heat.engine import environment from heat.engine import parser @@ -33,6 +37,7 @@ class MyCloudResource(generic_rsrc.GenericResource): class ProviderTemplateTest(HeatTestCase): def setUp(self): super(ProviderTemplateTest, self).setUp() + setup_dummy_db() resource._register_class('OS::ResourceType', generic_rsrc.GenericResource) resource._register_class('myCloud::ResourceType', @@ -77,15 +82,6 @@ class ProviderTemplateTest(HeatTestCase): cls = resource.get_class('OS::ResourceType', 'fred', env) self.assertEqual(cls, generic_rsrc.GenericResource) - def test_get_template_resource(self): - # assertion: if the name matches {.yaml|.template} we get the - # TemplateResource class. - env_str = {'resource_registry': {'resources': {'fred': { - "OS::ResourceType": "some_magic.yaml"}}}} - env = environment.Environment(env_str) - cls = resource.get_class('OS::ResourceType', 'fred', env) - self.assertEqual(cls, template_resource.TemplateResource) - def test_to_parameters(self): """Tests property conversion to parameter values.""" setup_dummy_db() @@ -141,3 +137,58 @@ class ProviderTemplateTest(HeatTestCase): self.assertEqual(5, converted_params.get("ANum")) # verify Map conversion self.assertEqual(map_prop_val, converted_params.get("AMap")) + + def test_get_template_resource(self): + # assertion: if the name matches {.yaml|.template} we get the + # TemplateResource class. + env_str = {'resource_registry': {'resources': {'fred': { + "OS::ResourceType": "some_magic.yaml"}}}} + env = environment.Environment(env_str) + cls = resource.get_class('OS::ResourceType', 'fred', env) + self.assertEqual(cls, template_resource.TemplateResource) + + def test_template_as_resource(self): + """ + Test that the resulting resource has the right prop and attrib schema. + + Note that this test requires the Wordpress_Single_Instance.yaml + template in the templates directory since we want to test using a + non-trivial template. + """ + test_templ_name = "WordPress_Single_Instance.yaml" + path = os.path.join(os.path.dirname(os.path.realpath(__file__)), + 'templates', test_templ_name) + # check if its in the directory list vs. exists to work around + # case-insensitive file systems + self.assertIn(test_templ_name, os.listdir(os.path.dirname(path))) + with open(path) as test_templ_file: + test_templ = test_templ_file.read() + self.assertTrue(test_templ, "Empty test template") + self.m.StubOutWithMock(urlfetch, "get") + urlfetch.get(test_templ_name).AndReturn(test_templ) + parsed_test_templ = yaml.safe_load(test_templ) + self.m.ReplayAll() + json_snippet = { + "Type": test_templ_name, + "Properties": { + "KeyName": "mykeyname", + "DBName": "wordpress1", + "DBUsername": "wpdbuser", + "DBPassword": "wpdbpass", + "DBRootPassword": "wpdbrootpass", + "LinuxDistribution": "U10" + } + } + stack = parser.Stack(None, 'test_stack', parser.Template({}), + stack_id=uuidutils.generate_uuid()) + templ_resource = resource.Resource("test_templ_resource", json_snippet, + stack) + self.m.VerifyAll() + self.assertIsInstance(templ_resource, + template_resource.TemplateResource) + for prop in parsed_test_templ.get("Parameters", {}): + self.assertIn(prop, templ_resource.properties) + for attrib in parsed_test_templ.get("Outputs", {}): + self.assertIn(attrib, templ_resource.attributes) + for k, v in json_snippet.get("Properties").items(): + self.assertEqual(v, templ_resource.properties[k])