From f1c762b110100903121e13afc7870737161d2828 Mon Sep 17 00:00:00 2001 From: Steve Baker Date: Mon, 26 Nov 2012 12:52:47 +1300 Subject: [PATCH] Module for converting JSON to YAML, and parsing both convert_json_to_yaml is a utility function used for tests and file conversion utilities. parse_to_template will take any string, infer the format, and parse to a python structure. Currently it assumes the file is JSON if it starts with '{' otherwise it attempts to parse it as YAML. Change-Id: If15ccdaf912693f76b74bb1fe879145af1cb36b1 --- heat/engine/format.py | 74 +++++++++++++++++++++++++++++++++++++++ heat/tests/test_format.py | 73 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 147 insertions(+) create mode 100644 heat/engine/format.py create mode 100644 heat/tests/test_format.py diff --git a/heat/engine/format.py b/heat/engine/format.py new file mode 100644 index 00000000..693166aa --- /dev/null +++ b/heat/engine/format.py @@ -0,0 +1,74 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import re +import yaml +import json + + +def _construct_yaml_str(self, node): + # Override the default string handling function + # to always return unicode objects + return self.construct_scalar(node) +yaml.Loader.add_constructor(u'tag:yaml.org,2002:str', _construct_yaml_str) +yaml.SafeLoader.add_constructor(u'tag:yaml.org,2002:str', _construct_yaml_str) + + +def parse_to_template(tmpl_str): + ''' + Takes a string and returns a dict containing the parsed structure. + This includes determination of whether the string is using the + JSON or YAML format. + ''' + if tmpl_str.startswith('{'): + return json.loads(tmpl_str) + try: + return yaml.load(tmpl_str) + except yaml.scanner.ScannerError as e: + raise ValueError(e) + + +def convert_json_to_yaml(json_str): + '''Convert a string containing the AWS JSON template format + to an equivalent string containing the Heat YAML format.''' + + global key_order + # Replace AWS format version with Heat format version + json_str = re.sub('"AWSTemplateFormatVersion"\s*:\s*"[^"]+"\s*,', + '', json_str) + + # insert a sortable order into the key to preserve file ordering + key_order = 0 + + def order_key(matchobj): + global key_order + key = '%s"__%05d__order__%s" :' % ( + matchobj.group(1), + key_order, + matchobj.group(2)) + key_order = key_order + 1 + return key + key_re = re.compile('^(\s*)"([^"]+)"\s*:', re.M) + json_str = key_re.sub(order_key, json_str) + + # parse the string as json to a python structure + tpl = yaml.load(json_str) + + # dump python structure to yaml + yml = "HeatTemplateFormatVersion: '2012-12-12'\n" + yaml.safe_dump(tpl) + + # remove ordering from key names + yml = re.sub('__\d*__order__', '', yml) + return yml diff --git a/heat/tests/test_format.py b/heat/tests/test_format.py new file mode 100644 index 00000000..c8cfef38 --- /dev/null +++ b/heat/tests/test_format.py @@ -0,0 +1,73 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import json +import mox +import nose +from nose.plugins.attrib import attr +import os +import re +import unittest +import yaml + +from heat.engine import format +from heat.engine import parser + + +@attr(tag=['unit']) +class JsonToYamlTest(unittest.TestCase): + + def setUp(self): + self.expected_test_count = 29 + self.longMessage = True + + def test_convert_all_templates(self): + path = os.path.dirname(os.path.realpath(__file__)).\ + replace('heat/tests', 'templates') + + template_test_count = 0 + for (json_str, + yml_str, + file_name) in self.convert_all_json_to_yaml(path): + + self.compare_json_vs_yaml(json_str, yml_str, file_name) + template_test_count += 1 + + self.assertTrue(template_test_count >= self.expected_test_count, + 'Expected at least %d templates to be tested' % + self.expected_test_count) + + def compare_json_vs_yaml(self, json_str, yml_str, file_name): + yml = format.parse_to_template(yml_str) + + self.assertEqual(u'2012-12-12', yml[u'HeatTemplateFormatVersion'], + file_name) + self.assertFalse(u'AWSTemplateFormatVersion' in yml, file_name) + del(yml[u'HeatTemplateFormatVersion']) + + jsn = format.parse_to_template(json_str) + if u'AWSTemplateFormatVersion' in jsn: + del(jsn[u'AWSTemplateFormatVersion']) + + self.assertEqual(yml, jsn, file_name) + + def convert_all_json_to_yaml(self, dirpath): + for path in os.listdir(dirpath): + if not path.endswith('.template') and not path.endswith('.json'): + continue + f = open(os.path.join(dirpath, path), 'r') + json_str = f.read() + + yml_str = format.convert_json_to_yaml(json_str) + yield (json_str, yml_str, f.name) -- 2.45.2