debian/patches/mysql-reconnect.patch

   1 diff --git a/neunton/common/exceptions.py b/neunton/common/exceptions.py
   2 index c99c254..e24f7bc 100644
   3 --- a/neunton/common/exceptions.py
   4 +++ b/neunton/common/exceptions.py
   5 @@ -235,3 +235,7 @@ class InvalidSharedSetting(QuantumException):
   6
   7  class InvalidExtenstionEnv(QuantumException):
   8      message = _("Invalid extension environment: %(reason)s")
   9 +
  10 +class DBError(Error):
  11 +    message = _("Database error")
  12 +
  13 diff --git a/neunton/db/api.py b/neunton/db/api.py
  14 index 238a9f9..737c748 100644
  15 --- a/neunton/db/api.py
  16 +++ b/neunton/db/api.py
  17 @@ -20,12 +20,16 @@
  18  import logging
  19  import time
  20
  21 +import time
  22 +
  23  import sqlalchemy as sql
  24  from sqlalchemy import create_engine
  25  from sqlalchemy.exc import DisconnectionError
  26 +from sqlalchemy.exc import OperationalError
  27  from sqlalchemy.orm import sessionmaker, exc
  28
  29  from quantum.db import model_base
  30 +from quantum.common.exceptions import DBError
  31
  32  LOG = logging.getLogger(__name__)
  33
  34 @@ -33,28 +37,61 @@ LOG = logging.getLogger(__name__)
  35  _ENGINE = None
  36  _MAKER = None
  37  BASE = model_base.BASE
  38 +OPTIONS = None
  39
  40 +def is_db_connection_error(args):
  41 +    """Return True if error in connecting to db."""
  42 +    # NOTE(adam_g): This is currently MySQL specific and needs to be extended
  43 +    #               to support Postgres and others.
  44 +    conn_err_codes = ('2002', '2003', '2006', '2013', '2014', '2045', '2055')
  45 +    for err_code in conn_err_codes:
  46 +        if args.find(err_code) != -1:
  47 +            return True
  48 +    return False
  49
  50 -class MySQLPingListener(object):
  51
  52 -    """
  53 -    Ensures that MySQL connections checked out of the
  54 -    pool are alive.
  55 +def wrap_db_error(f):
  56 +    """Function wrapper to capture DB errors
  57
  58 -    Borrowed from:
  59 -    http://groups.google.com/group/sqlalchemy/msg/a4ce563d802c929f
  60 -    """
  61 +    If an exception is thrown by the wrapped function,
  62 +    determine if it represents a database connection error.
  63 +    If so, retry the wrapped function, and repeat until it succeeds
  64 +    or we reach a configurable maximum number of retries.
  65 +    If it is not a connection error, or we exceeded the retry limit,
  66 +    raise a DBError.
  67
  68 -    def checkout(self, dbapi_con, con_record, con_proxy):
  69 -        try:
  70 -            dbapi_con.cursor().execute('select 1')
  71 -        except dbapi_con.OperationalError, ex:
  72 -            if ex.args[0] in (2006, 2013, 2014, 2045, 2055):
  73 -                LOG.warn('Got mysql server has gone away: %s', ex)
  74 -                raise DisconnectionError("Database server went away")
  75 -            else:
  76 +    """
  77 +    global OPTIONS
  78 +    def _wrap_db_error(*args, **kwargs):
  79 +        next_interval = OPTIONS.get('reconnect_interval', 1)
  80 +        remaining = OPTIONS.get('sql_max_retries', -1)
  81 +        if remaining == -1:
  82 +            remaining = 'infinite'
  83 +        while True:
  84 +            try:
  85 +                return f(*args, **kwargs)
  86 +            except OperationalError, e:
  87 +                if is_db_connection_error(e.args[0]):
  88 +                    if remaining == 0:
  89 +                        logging.warn('DB exceeded retry limit.')
  90 +                        raise DBError(e)
  91 +                    if remaining != 'infinite':
  92 +                        remaining -= 1
  93 +                    logging.warn('DB connection error, '
  94 +                                    'retrying in %i seconds.' % next_interval)
  95 +                    time.sleep(next_interval)
  96 +                    if OPTIONS.get('inc_reconnect_interval', True):
  97 +                        next_interval = min(next_interval * 2,
  98 +                                            OPTIONS.get('max_reconnect_interval', 60))
  99 +                else:
 100 +                    logging.warn('DB exception wrapped.')
 101 +                    raise DBError(e)
 102 +            except Exception, e:
 103                  raise
 104
 105 +    _wrap_db_error.func_name = f.func_name
 106 +    return _wrap_db_error
 107 +
 108
 109  def configure_db(options):
 110      """
 111 @@ -63,6 +100,8 @@ def configure_db(options):
 112
 113      :param options: Mapping of configuration options
 114      """
 115 +    global OPTIONS
 116 +    OPTIONS = options
 117      global _ENGINE
 118      if not _ENGINE:
 119          connection_dict = sql.engine.url.make_url(options['sql_connection'])
 120 @@ -72,9 +111,6 @@ def configure_db(options):
 121              'convert_unicode': True,
 122          }
 123
 124 -        if 'mysql' in connection_dict.drivername:
 125 -            engine_args['listeners'] = [MySQLPingListener()]
 126 -
 127          _ENGINE = create_engine(options['sql_connection'], **engine_args)
 128          base = options.get('base', BASE)
 129          if not register_models(base):
 130 @@ -101,10 +137,18 @@ def get_session(autocommit=True, expire_on_commit=False):
 131      global _MAKER, _ENGINE
 132      if not _MAKER:
 133          assert _ENGINE
 134 +        class OurQuery(sql.orm.query.Query):
 135 +            pass
 136 +        query = OurQuery
 137 +        query.all = wrap_db_error(query.all)
 138 +        query.first = wrap_db_error(query.first)
 139          _MAKER = sessionmaker(bind=_ENGINE,
 140                                autocommit=autocommit,
 141 -                              expire_on_commit=expire_on_commit)
 142 -    return _MAKER()
 143 +                              expire_on_commit=expire_on_commit,
 144 +                              query_cls=OurQuery)
 145 +    session = _MAKER()
 146 +    session.flush = wrap_db_error(session.flush)
 147 +    return session
 148
 149
 150  def retry_registration(remaining, reconnect_interval, base=BASE):