Convert heat.common.template_format to use C yaml

Will fall back to pyyaml if CSafeLoader/CSafeDumper are not available.
This should increase template parsing performance by a factor of 9 and
reduce memory usage by an order of magnitude.

We also convert the one test that was directly calling yaml to use the
template_format.parse function, so that the same parsing is used
everywhere.

Change-Id: I0d14be2ef7fb2dccac43fc41b1d2d9e4395c6f77
Fixes bug #1214580

diff --git a/heat/common/template_format.py b/heat/common/template_format.py
index d6bd8e814e8847307a8464d7c520de0b7609182c..fe5c4e985098b50ea94e23fcc95cf09991c66847 100644 (file)
--- a/heat/common/template_format.py
+++ b/heat/common/template_format.py
@@ -20,21 +20,30 @@ import json
 HEAT_VERSIONS = (u'2012-12-12',)
 CFN_VERSIONS = (u'2010-09-09',)
 
+if hasattr(yaml, 'CSafeLoader'):
+    yaml_loader = yaml.CSafeLoader
+else:
+    yaml_loader = yaml.SafeLoader
+
+if hasattr(yaml, 'CSafeDumper'):
+    yaml_dumper = yaml.CSafeDumper
+else:
+    yaml_dumper = yaml.SafeDumper
+
 
 def _construct_yaml_str(self, node):
     # Override the default string handling function
     # to always return unicode objects
     return self.construct_scalar(node)
-yaml.Loader.add_constructor(u'tag:yaml.org,2002:str', _construct_yaml_str)
-yaml.SafeLoader.add_constructor(u'tag:yaml.org,2002:str', _construct_yaml_str)
+yaml_loader.add_constructor(u'tag:yaml.org,2002:str', _construct_yaml_str)
 # Unquoted dates like 2013-05-23 in yaml files get loaded as objects of type
 # datetime.data which causes problems in API layer when being processed by
 # openstack.common.jsonutils. Therefore, make unicode string out of timestamps
 # until jsonutils can handle dates.
-yaml.Loader.add_constructor(u'tag:yaml.org,2002:timestamp',
+yaml_loader.add_constructor(u'tag:yaml.org,2002:timestamp',
+                            _construct_yaml_str)
+yaml_loader.add_constructor(u'tag:yaml.org,2002:timestamp',
                             _construct_yaml_str)
-yaml.SafeLoader.add_constructor(u'tag:yaml.org,2002:timestamp',
-                                _construct_yaml_str)
 
 
 def parse(tmpl_str, add_template_sections=True):
@@ -47,7 +56,7 @@ def parse(tmpl_str, add_template_sections=True):
         tpl = json.loads(tmpl_str)
     else:
         try:
-            tpl = yaml.safe_load(tmpl_str)
+            tpl = yaml.load(tmpl_str, Loader=yaml_loader)
         except (yaml.scanner.ScannerError, yaml.parser.ParserError) as e:
             raise ValueError(e)
         else:
@@ -100,10 +109,11 @@ def convert_json_to_yaml(json_str):
     json_str = key_re.sub(order_key, json_str)
 
     # parse the string as json to a python structure
-    tpl = yaml.safe_load(json_str)
+    tpl = yaml.load(json_str, Loader=yaml_loader)
 
     # dump python structure to yaml
-    yml = "HeatTemplateFormatVersion: '2012-12-12'\n" + yaml.safe_dump(tpl)
+    tpl["HeatTemplateFormatVersion"] = '2012-12-12'
+    yml = yaml.dump(tpl, Dumper=yaml_dumper)
 
     # remove ordering from key names
     yml = re.sub('__\d*__order__', '', yml)

diff --git a/heat/tests/test_provider_template.py b/heat/tests/test_provider_template.py
index a20c9074b12a978d6f3255134f6887905c965446..af9b060766769aaa11c3bdedf2c0ef611c59e436 100644 (file)
--- a/heat/tests/test_provider_template.py
+++ b/heat/tests/test_provider_template.py
@@ -13,9 +13,9 @@
 #    under the License.
 
 import os
-import yaml
 
 from heat.common import urlfetch
+from heat.common import template_format
 
 from heat.engine import environment
 from heat.engine import parser
@@ -165,7 +165,7 @@ class ProviderTemplateTest(HeatTestCase):
         self.assertTrue(test_templ, "Empty test template")
         self.m.StubOutWithMock(urlfetch, "get")
         urlfetch.get(test_templ_name).AndReturn(test_templ)
-        parsed_test_templ = yaml.safe_load(test_templ)
+        parsed_test_templ = template_format.parse(test_templ)
         self.m.ReplayAll()
         json_snippet = {
             "Type": test_templ_name,