From: Clint Byrum <clint@fewbar.com>
Date: Tue, 20 Aug 2013 20:50:00 +0000 (-0700)
Subject: Convert heat.common.template_format to use C yaml
X-Git-Tag: 2014.1~164^2
X-Git-Url: https://review.fuel-infra.org/gitweb?a=commitdiff_plain;h=e72ec15e1c871bf8c49d7d21456fd32bfcc0a260;p=openstack-build%2Fheat-build.git

Convert heat.common.template_format to use C yaml

Will fall back to pyyaml if CSafeLoader/CSafeDumper are not available.
This should increase template parsing performance by a factor of 9 and
reduce memory usage by an order of magnitude.

We also convert the one test that was directly calling yaml to use the
template_format.parse function, so that the same parsing is used
everywhere.

Change-Id: I0d14be2ef7fb2dccac43fc41b1d2d9e4395c6f77
Fixes bug #1214580
---

diff --git a/heat/common/template_format.py b/heat/common/template_format.py
index d6bd8e81..fe5c4e98 100644
--- a/heat/common/template_format.py
+++ b/heat/common/template_format.py
@@ -20,21 +20,30 @@ import json
 HEAT_VERSIONS = (u'2012-12-12',)
 CFN_VERSIONS = (u'2010-09-09',)
 
+if hasattr(yaml, 'CSafeLoader'):
+    yaml_loader = yaml.CSafeLoader
+else:
+    yaml_loader = yaml.SafeLoader
+
+if hasattr(yaml, 'CSafeDumper'):
+    yaml_dumper = yaml.CSafeDumper
+else:
+    yaml_dumper = yaml.SafeDumper
+
 
 def _construct_yaml_str(self, node):
     # Override the default string handling function
     # to always return unicode objects
     return self.construct_scalar(node)
-yaml.Loader.add_constructor(u'tag:yaml.org,2002:str', _construct_yaml_str)
-yaml.SafeLoader.add_constructor(u'tag:yaml.org,2002:str', _construct_yaml_str)
+yaml_loader.add_constructor(u'tag:yaml.org,2002:str', _construct_yaml_str)
 # Unquoted dates like 2013-05-23 in yaml files get loaded as objects of type
 # datetime.data which causes problems in API layer when being processed by
 # openstack.common.jsonutils. Therefore, make unicode string out of timestamps
 # until jsonutils can handle dates.
-yaml.Loader.add_constructor(u'tag:yaml.org,2002:timestamp',
+yaml_loader.add_constructor(u'tag:yaml.org,2002:timestamp',
+                            _construct_yaml_str)
+yaml_loader.add_constructor(u'tag:yaml.org,2002:timestamp',
                             _construct_yaml_str)
-yaml.SafeLoader.add_constructor(u'tag:yaml.org,2002:timestamp',
-                                _construct_yaml_str)
 
 
 def parse(tmpl_str, add_template_sections=True):
@@ -47,7 +56,7 @@ def parse(tmpl_str, add_template_sections=True):
         tpl = json.loads(tmpl_str)
     else:
         try:
-            tpl = yaml.safe_load(tmpl_str)
+            tpl = yaml.load(tmpl_str, Loader=yaml_loader)
         except (yaml.scanner.ScannerError, yaml.parser.ParserError) as e:
             raise ValueError(e)
         else:
@@ -100,10 +109,11 @@ def convert_json_to_yaml(json_str):
     json_str = key_re.sub(order_key, json_str)
 
     # parse the string as json to a python structure
-    tpl = yaml.safe_load(json_str)
+    tpl = yaml.load(json_str, Loader=yaml_loader)
 
     # dump python structure to yaml
-    yml = "HeatTemplateFormatVersion: '2012-12-12'\n" + yaml.safe_dump(tpl)
+    tpl["HeatTemplateFormatVersion"] = '2012-12-12'
+    yml = yaml.dump(tpl, Dumper=yaml_dumper)
 
     # remove ordering from key names
     yml = re.sub('__\d*__order__', '', yml)
diff --git a/heat/tests/test_provider_template.py b/heat/tests/test_provider_template.py
index a20c9074..af9b0607 100644
--- a/heat/tests/test_provider_template.py
+++ b/heat/tests/test_provider_template.py
@@ -13,9 +13,9 @@
 #    under the License.
 
 import os
-import yaml
 
 from heat.common import urlfetch
+from heat.common import template_format
 
 from heat.engine import environment
 from heat.engine import parser
@@ -165,7 +165,7 @@ class ProviderTemplateTest(HeatTestCase):
         self.assertTrue(test_templ, "Empty test template")
         self.m.StubOutWithMock(urlfetch, "get")
         urlfetch.get(test_templ_name).AndReturn(test_templ)
-        parsed_test_templ = yaml.safe_load(test_templ)
+        parsed_test_templ = template_format.parse(test_templ)
         self.m.ReplayAll()
         json_snippet = {
             "Type": test_templ_name,