From 04d73a59ba00f0c73e04271a2c13f31f8f23cc51 Mon Sep 17 00:00:00 2001 From: Chris Alfonso Date: Mon, 2 Apr 2012 08:12:33 -0400 Subject: [PATCH] Adding the sqlalchemy db implmentation --- etc/heat-engine.conf | 2 + heat/common/config.py | 7 + heat/db/api.py | 27 +++- heat/db/sqlalchemy/api.py | 123 ++++++++++++++---- .../migrate_repo/versions/001_norwhal.py | 18 ++- heat/db/sqlalchemy/models.py | 23 ++-- heat/db/sqlalchemy/session.py | 97 ++++++++++++++ heat/engine/api/v1/stacks.py | 22 ++-- heat/engine/parser.py | 2 +- heat/engine/resources.py | 21 +-- tools/README | 3 + 11 files changed, 283 insertions(+), 62 deletions(-) create mode 100644 heat/db/sqlalchemy/session.py diff --git a/etc/heat-engine.conf b/etc/heat-engine.conf index c87c4f6a..0a3259b1 100644 --- a/etc/heat-engine.conf +++ b/etc/heat-engine.conf @@ -25,3 +25,5 @@ use_syslog = False # syslog_log_facility = LOG_LOCAL0 sql_connection = mysql://heat:heat@localhost/heat + +db_backend=heat.db.sqlalchemy.api diff --git a/heat/common/config.py b/heat/common/config.py index 0928b4ba..07c92160 100644 --- a/heat/common/config.py +++ b/heat/common/config.py @@ -51,6 +51,13 @@ class HeatEngineConfigOpts(HeatConfigOpts): db_opts = [ cfg.StrOpt('db_backend', default='heat.db.anydbm.api', help='The backend to use for db'), + cfg.StrOpt('sql_connection', + default='mysql://heat:heat@localhost/heat', + help='The SQLAlchemy connection string used to connect to the ' + 'database'), + cfg.IntOpt('sql_idle_timeout', + default=3600, + help='timeout before idle sql connections are reaped'), ] def __init__(self, **kwargs): diff --git a/heat/db/api.py b/heat/db/api.py index ad3d7b56..c4497030 100644 --- a/heat/db/api.py +++ b/heat/db/api.py @@ -26,12 +26,15 @@ The underlying driver is loaded as a :class:`LazyPluggable`. SQLAlchemy is currently the only supported backend. ''' -from heat.openstack.common import cfg from heat.common import utils def configure(conf): global IMPL + global SQL_CONNECTION + global SQL_IDLE_TIMEOUT IMPL = utils.import_object(conf.db_backend) + SQL_CONNECTION = conf.sql_connection + SQL_IDLE_TIMEOUT = conf.sql_idle_timeout def raw_template_get(context, template_id): return IMPL.raw_template_get(context, template_id) @@ -53,14 +56,24 @@ def parsed_template_create(context, values): return IMPL.parsed_template_create(context, values) -def state_get(context, state_id): - return IMPL.state_get(context, state_id) +def resource_get(context, resource_id): + return IMPL.resource_get(context, resource_id) -def state_get_all(context): - return IMPL.state_get_all(context) +def resource_get_all(context): + return IMPL.resource_get_all(context) -def state_create(context, values): - return IMPL.state_create(context, values) +def resource_create(context, values): + return IMPL.resource_create(context, values) + + +def stack_get(context, stack_id): + return IMPL.resource_get(context, resource_id) + +def stack_get_all(context): + return IMPL.stack_get_all(context) + +def stack_create(context, values): + return IMPL.stack_create(context, values) def event_get(context, event_id): diff --git a/heat/db/sqlalchemy/api.py b/heat/db/sqlalchemy/api.py index 3c0ac504..ad8c2d4a 100644 --- a/heat/db/sqlalchemy/api.py +++ b/heat/db/sqlalchemy/api.py @@ -15,11 +15,9 @@ '''Implementation of SQLAlchemy backend.''' -from nova.db.sqlalchemy.session import get_session -from nova import flags from nova import utils - -FLAGS = flags.FLAGS +from heat.db.sqlalchemy import models +from heat.db.sqlalchemy.session import get_session def model_query(context, *args, **kwargs): """Query helper that accounts for context's `read_deleted` field. @@ -51,45 +49,126 @@ def model_query(context, *args, **kwargs): return query -# a big TODO def raw_template_get(context, template_id): - return 'test return value' + result = model_query(context, models.RawTemplate).\ + filter_by(id=template_id).first() + + if not result: + raise Exception("raw template with id %s not found" % template_id) + + return result def raw_template_get_all(context): - pass + results = model_query(context, models.RawTemplate).all() -def raw_template_create(context, values): - pass + if not results: + raise Exception('no raw templates were found') + + return results +def raw_template_create(context, values): + raw_template_ref = models.RawTemplate() + raw_template_ref.update(values) + raw_template_ref.save() + return raw_template_ref def parsed_template_get(context, template_id): - pass + result = model_query(context, models.ParsedTemplate).\ + filter_by(id=template_id).first() + + if not result: + raise Exception("parsed template with id %s not found" % template_id) + + return result def parsed_template_get_all(context): - pass + results = model_query(context, models.ParsedTemplate).all() + + if not results: + raise Exception('no parsed templates were found') + + return results def parsed_template_create(context, values): - pass + parsed_template_ref = models.ParsedTemplate() + parsed_template_ref.update(values) + parsed_template_ref.save() + return parsed_template_ref + +def resource_get(context, resource_id): + result = model_query(context, models.Resource).\ + filter_by(id=resource_id).first() + + if not result: + raise Exception("resource with id %s not found" % resource_id) + return result -def state_get(context, state_id): - pass +def resource_get_all(context): + results = model_query(context, models.Resource).all() -def state_get_all(context): - pass + if not results: + raise Exception('no resources were found') + + return results -def state_create(context, values): - pass +def resource_create(context, values): + resource_ref = models.Resource() + resource_ref.update(values) + resource_ref.save() + return resource_ref +def stack_get(context, stack_id): + result = model_query(context, models.Stack).\ + filter_by(id=stack_id).first() + + if not result: + raise Exception("stack with id %s not found" % stack_id) + + return result + +def stack_get_all(context): + results = model_query(context, models.Stack).all() + + if not results: + raise Exception('no stacks were found') + + return results + +def stack_create(context, values): + stack_ref = models.Stack() + stack_ref.update(values) + stack_ref.save() + return stack_ref def event_get(context, event_id): - pass + result = model_query(context, models.Event).\ + filter_by(id=event_id).first() + + if not result: + raise Exception("event with id %s not found" % event_id) + + return result def event_get_all(context): - pass + results = model_query(context, models.Event).all() + + if not results: + raise Exception('no events were found') + + return results def event_get_all_by_stack(context, stack_id): - pass + results = model_query(context, models.Event).\ + filter_by(stack_id).all() + + if not results: + raise Exception("no events for stack_id %s were found" % stack_id) + + return results def event_create(context, values): - pass + event_ref = models.Event() + event_ref.update(values) + event_ref.save() + return event_ref diff --git a/heat/db/sqlalchemy/migrate_repo/versions/001_norwhal.py b/heat/db/sqlalchemy/migrate_repo/versions/001_norwhal.py index 852da9fd..3311d86d 100644 --- a/heat/db/sqlalchemy/migrate_repo/versions/001_norwhal.py +++ b/heat/db/sqlalchemy/migrate_repo/versions/001_norwhal.py @@ -13,9 +13,20 @@ def upgrade(migrate_engine): Column('template', Text()), ) + stack = Table( + 'stack', meta, + Column('id', Integer, primary_key=True), + Column('created_at', DateTime(timezone=False)), + Column('updated_at', DateTime(timezone=False)), + Column('name', String(length=255, convert_unicode=False, + assert_unicode=None, + unicode_error=None, _warn_on_bytestring=False)), + ) + event = Table( 'event', meta, Column('id', Integer, primary_key=True), + Column('stack_id', Integer, ForeignKey("stack.id"), nullable=False), Column('created_at', DateTime(timezone=False)), Column('updated_at', DateTime(timezone=False)), Column('name', String(length=255, convert_unicode=False, @@ -41,11 +52,12 @@ def upgrade(migrate_engine): parsedtemplate = Table( 'parsed_template', meta, Column('id', Integer, primary_key=True), - Column('resource_id', Integer()), + Column('resource_id', Integer, ForeignKey("resource.id"),\ + nullable=False), Column('template', Text()), ) - tables = [rawtemplate, event, resource, parsedtemplate] + tables = [rawtemplate, stack, event, resource, parsedtemplate] for table in tables: try: table.create() @@ -62,5 +74,5 @@ def downgrade(migrate_engine): resource = Table('resource', meta, autoload=True) parsedtemplate = Table('parsed_template', meta, autoload=True) - for table in (rawtemplate, event, resource, parsedtemplate): + for table in (rawtemplate, event, stack, parsedtemplate, resource): table.drop() diff --git a/heat/db/sqlalchemy/models.py b/heat/db/sqlalchemy/models.py index 886306bd..2ab13561 100644 --- a/heat/db/sqlalchemy/models.py +++ b/heat/db/sqlalchemy/models.py @@ -16,16 +16,14 @@ SQLAlchemy models for heat data. """ from sqlalchemy import * -from sqlalchemy.orm import relationship, backref, object_mapper from sqlalchemy.exc import IntegrityError from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.schema import ForeignKeyConstraint -from nova import flags +from nova import utils +from heat.db.sqlalchemy.session import get_session -FLAGS = flags.FLAGS BASE = declarative_base() -meta = MetaData() class HeatBase(object): """Base class for Heat Models.""" @@ -85,29 +83,38 @@ class HeatBase(object): local.update(joined) return local.iteritems() -class RawTemplate(Base, HeatBase): +class RawTemplate(BASE, HeatBase): """Represents an unparsed template which should be in JSON format.""" __tablename__ = 'raw_template' id = Column(Integer, primary_key=True) template = Text() -class ParsedTemplate(Base, HeatBase): +class ParsedTemplate(BASE, HeatBase): """Represents a parsed template.""" __tablename__ = 'parsed_template' id = Column(Integer, primary_key=True) resource_id = Column('resource_id', Integer) -class Event(Base, HeatBase): +class Stack(BASE, HeatBase): + """Represents an generated by the heat engine.""" + + __tablename__ = 'stack' + + id = Column(Integer, primary_key=True) + name = Column(String) + +class Event(BASE, HeatBase): """Represents an event generated by the heat engine.""" __tablename__ = 'event' id = Column(Integer, primary_key=True) + stack_id = Column(Integer) name = Column(String) -class Resource(Base, HeatBase): +class Resource(BASE, HeatBase): """Represents a resource created by the heat engine.""" __tablename__ = 'resource' diff --git a/heat/db/sqlalchemy/session.py b/heat/db/sqlalchemy/session.py new file mode 100644 index 00000000..596c7859 --- /dev/null +++ b/heat/db/sqlalchemy/session.py @@ -0,0 +1,97 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +"""Session Handling for SQLAlchemy backend.""" + +import sqlalchemy.interfaces +import sqlalchemy.orm +from sqlalchemy.exc import DisconnectionError + +import nova.exception +from heat.openstack.common import cfg +from heat.db import api as db_api + +_ENGINE = None +_MAKER = None + + +def get_session(autocommit=True, expire_on_commit=False): + """Return a SQLAlchemy session.""" + global _ENGINE, _MAKER + + if _MAKER is None or _ENGINE is None: + _ENGINE = get_engine() + _MAKER = get_maker(_ENGINE, autocommit, expire_on_commit) + + session = _MAKER() + session.query = nova.exception.wrap_db_error(session.query) + session.flush = nova.exception.wrap_db_error(session.flush) + return session + + +class SynchronousSwitchListener(sqlalchemy.interfaces.PoolListener): + + """Switch sqlite connections to non-synchronous mode""" + + def connect(self, dbapi_con, con_record): + dbapi_con.execute("PRAGMA synchronous = OFF") + + +class MySQLPingListener(object): + + """ + Ensures that MySQL connections checked out of the + pool are alive. + + Borrowed from: + http://groups.google.com/group/sqlalchemy/msg/a4ce563d802c929f + """ + + def checkout(self, dbapi_con, con_record, con_proxy): + try: + dbapi_con.cursor().execute('select 1') + except dbapi_con.OperationalError, ex: + if ex.args[0] in (2006, 2013, 2014, 2045, 2055): + LOG.warn('Got mysql server has gone away: %s', ex) + raise DisconnectionError("Database server went away") + else: + raise + + +def get_engine(): + """Return a SQLAlchemy engine.""" + connection_dict = sqlalchemy.engine.url.make_url(_get_sql_connection()) + engine_args = { + "pool_recycle": _get_sql_idle_timeout(), + "echo": False, + 'convert_unicode': True, + } + + if 'mysql' in connection_dict.drivername: + engine_args['listeners'] = [MySQLPingListener()] + + return sqlalchemy.create_engine(_get_sql_connection(), **engine_args) + + +def get_maker(engine, autocommit=True, expire_on_commit=False): + """Return a SQLAlchemy sessionmaker using the given engine.""" + return sqlalchemy.orm.sessionmaker(bind=engine, + autocommit=autocommit, + expire_on_commit=expire_on_commit) + +def _get_sql_connection(): + return db_api.SQL_CONNECTION + +def _get_sql_idle_timeout(): + return db_api.SQL_IDLE_TIMEOUT diff --git a/heat/engine/api/v1/stacks.py b/heat/engine/api/v1/stacks.py index bceaa79c..894a7d47 100644 --- a/heat/engine/api/v1/stacks.py +++ b/heat/engine/api/v1/stacks.py @@ -49,15 +49,15 @@ class StacksController(object): res = {'stacks': [] } for s in stack_db: mem = {} - mem['StackId'] = s - mem['StackName'] = s - mem['CreationTime'] = 'now' + mem['stack_id'] = s + mem['stack_name'] = s + mem['created_at'] = 'now' try: - mem['TemplateDescription'] = stack_db[s]['Description'] - mem['StackStatus'] = stack_db[s]['StackStatus'] + mem['template_description'] = stack_db[s]['Description'] + mem['stack_status'] = stack_db[s]['StackStatus'] except: - mem['TemplateDescription'] = 'No description' - mem['StackStatus'] = 'unknown' + mem['template_description'] = 'No description' + mem['stack_status'] = 'unknown' res['stacks'].append(mem) return res @@ -66,10 +66,10 @@ class StacksController(object): res = {'stacks': [] } if stack_db.has_key(id): mem = {} - mem['StackId'] = id - mem['StackName'] = id - mem['CreationTime'] = 'TODO' - mem['LastUpdatedTime'] = 'TODO' + mem['stack_id'] = id + mem['stack_name'] = id + mem['creation_at'] = 'TODO' + mem['updated_at'] = 'TODO' mem['NotificationARNs'] = 'TODO' mem['Outputs'] = [{'Description': 'TODO', 'OutputKey': 'TODO', 'OutputValue': 'TODO' }] mem['Parameters'] = stack_db[id]['Parameters'] diff --git a/heat/engine/parser.py b/heat/engine/parser.py index 38285e01..c13b6bd0 100644 --- a/heat/engine/parser.py +++ b/heat/engine/parser.py @@ -22,7 +22,7 @@ logger = logging.getLogger('heat.engine.parser') class Stack: def __init__(self, stack_name, template, parms=None): - + self.id = 0 self.t = template if self.t.has_key('Parameters'): self.parms = self.t['Parameters'] diff --git a/heat/engine/resources.py b/heat/engine/resources.py index 8010f69c..87ddf11d 100644 --- a/heat/engine/resources.py +++ b/heat/engine/resources.py @@ -22,7 +22,7 @@ from novaclient.v1_1 import client from heat.db import api as db_api from heat.common.config import HeatEngineConfigOpts - +import pdb db_api.configure(HeatEngineConfigOpts()) logger = logging.getLogger('heat.engine.resources') @@ -82,15 +82,16 @@ class Resource(object): def state_set(self, new_state, reason="state changed"): if new_state != self.state: ev = {} - ev['LogicalResourceId'] = self.name - ev['PhysicalResourceId'] = self.name - ev['StackId'] = self.stack.name - ev['StackName'] = self.stack.name - ev['ResourceStatus'] = new_state - ev['ResourceStatusReason'] = reason - ev['ResourceType'] = self.t['Type'] - ev['ResourceProperties'] = self.t['Properties'] - + ev['logical_resource_id'] = self.name + ev['physical_resource_id'] = self.name + ev['stack_id'] = self.stack.id + ev['stack_name'] = self.stack.name + ev['resource_status'] = new_state + ev['resource_status_reason'] = reason + ev['resource_type'] = self.t['Type'] + ev['resource_properties'] = self.t['Properties'] + new_stack = db_api.stack_create(None, ev) + ev['stack_id'] = new_stack.id db_api.event_create(None, ev) self.state = new_state diff --git a/tools/README b/tools/README index 8f178323..53d97f62 100644 --- a/tools/README +++ b/tools/README @@ -1,6 +1,9 @@ Files in this directory are general developer tools or examples of how to do certain activities. +If you're running on F16, make sure you first enable the preview yum repository +http://fedoraproject.org/wiki/Getting_started_with_OpenStack_on_Fedora_17#Preview_Repository_for_Fedora_16 + ----- Tools ----- -- 2.45.2