From 98d0a64ff1ec0b7839b52cf6f4a53d4d0765b7cf Mon Sep 17 00:00:00 2001 From: Dmitry Burmistrov Date: Thu, 7 Nov 2013 10:52:45 +0400 Subject: [PATCH] Added patches according to OSCI-846 --- debian/patches/fix-l3-agent.patch | 236 +++++++++++++++++++++++++++ debian/patches/mysql-reconnect.patch | 150 +++++++++++++++++ debian/patches/series | 2 + 3 files changed, 388 insertions(+) create mode 100644 debian/patches/fix-l3-agent.patch create mode 100644 debian/patches/mysql-reconnect.patch diff --git a/debian/patches/fix-l3-agent.patch b/debian/patches/fix-l3-agent.patch new file mode 100644 index 000000000..90fe1722f --- /dev/null +++ b/debian/patches/fix-l3-agent.patch @@ -0,0 +1,236 @@ +diff --git a/neunton/agent/l3_agent.py b/neunton/agent/l3_agent.py +index 79156cd..0037b80 100644 +--- a/neunton/agent/l3_agent.py ++++ b/neunton/agent/l3_agent.py +@@ -23,6 +23,7 @@ import eventlet + from eventlet import semaphore + import netaddr + from oslo.config import cfg ++from quantum.agent.linux.interface import OVSInterfaceDriver + + from neunton.agent.common import config + from neunton.agent.linux import external_process +@@ -233,7 +234,8 @@ class L3NATAgent(manager.Manager): + for c, r in self.metadata_nat_rules(): + ri.iptables_manager.ipv4['nat'].add_rule(c, r) + ri.iptables_manager.apply() +- self._spawn_metadata_proxy(ri) ++ if self.conf.use_namespaces: ++ self._spawn_metadata_proxy(ri) + + def _router_removed(self, router_id): + ri = self.router_info[router_id] +@@ -246,7 +248,8 @@ class L3NATAgent(manager.Manager): + for c, r in self.metadata_nat_rules(): + ri.iptables_manager.ipv4['nat'].remove_rule(c, r) + ri.iptables_manager.apply() +- self._destroy_metadata_proxy(ri) ++ if self.conf.use_namespaces: ++ self._destroy_metadata_proxy(ri) + del self.router_info[router_id] + self._destroy_router_namespace(ri.ns_name()) + +@@ -370,7 +373,9 @@ class L3NATAgent(manager.Manager): + ri.floating_ips.append(new_fip) + + def _get_ex_gw_port(self, ri): +- return ri.router.get('gw_port') ++ rv = ri.router.get('gw_port') ++ LOG.debug("***SV::_get_ex_gw_port: {0}".format(rv)) ++ return rv + + def _send_gratuitous_arp_packet(self, ri, interface_name, ip_address): + if self.conf.send_arp_for_ha > 0: +@@ -390,49 +395,78 @@ class L3NATAgent(manager.Manager): + LOG.error(_("Failed sending gratuitous ARP: %s"), str(e)) + + def get_internal_device_name(self, port_id): +- return (INTERNAL_DEV_PREFIX + port_id)[:self.driver.DEV_NAME_LEN] ++ devname = INTERNAL_DEV_PREFIX + port_id ++ LOG.debug("***SV::get_internal_device_name: {0}".format(devname)) ++ return devname[:self.driver.DEV_NAME_LEN] + + def get_external_device_name(self, port_id): +- return (EXTERNAL_DEV_PREFIX + port_id)[:self.driver.DEV_NAME_LEN] ++ devname = EXTERNAL_DEV_PREFIX + port_id ++ LOG.debug("***SV::get_external_device_name: {0}".format(devname)) ++ return devname[:self.driver.DEV_NAME_LEN] + +- def external_gateway_added(self, ri, ex_gw_port, internal_cidrs): ++ def get_effective_interface_name(self, ifname): ++ if not self.conf.use_namespaces: ++ return self.conf.external_network_bridge ++ else: ++ return ifname + ++ def external_gateway_added(self, ri, ex_gw_port, internal_cidrs): ++ ### SV: patched !!! + interface_name = self.get_external_device_name(ex_gw_port['id']) ++ effective_interface_name = self.get_effective_interface_name(interface_name) + ex_gw_ip = ex_gw_port['fixed_ips'][0]['ip_address'] +- if not ip_lib.device_exists(interface_name, +- root_helper=self.root_helper, +- namespace=ri.ns_name()): ++ ip_address = ex_gw_port['ip_cidr'].split('/')[0] ++ if not ip_lib.device_exists(interface_name, root_helper=self.root_helper, namespace=ri.ns_name()): + self.driver.plug(ex_gw_port['network_id'], + ex_gw_port['id'], interface_name, + ex_gw_port['mac_address'], + bridge=self.conf.external_network_bridge, + namespace=ri.ns_name(), +- prefix=EXTERNAL_DEV_PREFIX) +- self.driver.init_l3(interface_name, [ex_gw_port['ip_cidr']], +- namespace=ri.ns_name()) +- ip_address = ex_gw_port['ip_cidr'].split('/')[0] +- self._send_gratuitous_arp_packet(ri, interface_name, ip_address) ++ prefix=EXTERNAL_DEV_PREFIX ++ ) ++ ++ if not self.conf.use_namespaces: ++ LOG.info("Warning! You are not using network namespaces! " ++ "IP {ip} will be assigned to interface {iface}, instead {eiface}." ++ .format( ++ ip=ex_gw_port['ip_cidr'], ++ iface=interface_name, ++ eiface=effective_interface_name ++ ) ++ ) ++ net = netaddr.IPNetwork(ex_gw_port['ip_cidr']) ++ device = ip_lib.IPDevice(effective_interface_name, self.root_helper, namespace=ri.ns_name()) ++ device.addr.add(net.version, ex_gw_port['ip_cidr'], str(net.broadcast)) ++ else: ++ self.driver.init_l3(interface_name, [ex_gw_port['ip_cidr']], namespace=ri.ns_name()) ++ ++ self._send_gratuitous_arp_packet(ri, effective_interface_name, ip_address) + + gw_ip = ex_gw_port['subnet']['gateway_ip'] +- if ex_gw_port['subnet']['gateway_ip']: +- cmd = ['route', 'add', 'default', 'gw', gw_ip] +- if self.conf.use_namespaces: +- ip_wrapper = ip_lib.IPWrapper(self.root_helper, +- namespace=ri.ns_name()) +- ip_wrapper.netns.execute(cmd, check_exit_code=False) +- else: +- utils.execute(cmd, check_exit_code=False, +- root_helper=self.root_helper) ++ if interface_name == effective_interface_name: ++ # work in net.namespaces, or 1st assignation on this interface ++ if ex_gw_port['subnet']['gateway_ip']: ++ cmd = ['route', 'add', 'default', 'gw', gw_ip] ++ if self.conf.use_namespaces: ++ ip_wrapper = ip_lib.IPWrapper(self.root_helper, namespace=ri.ns_name()) ++ ip_wrapper.netns.execute(cmd, check_exit_code=False) ++ else: ++ utils.execute(cmd, check_exit_code=False, root_helper=self.root_helper) + +- for (c, r) in self.external_gateway_nat_rules(ex_gw_ip, +- internal_cidrs, +- interface_name): ++ for (c, r) in self.external_gateway_nat_rules(ex_gw_ip, internal_cidrs, effective_interface_name): + ri.iptables_manager.ipv4['nat'].add_rule(c, r) + ri.iptables_manager.apply() + + def external_gateway_removed(self, ri, ex_gw_port, internal_cidrs): +- + interface_name = self.get_external_device_name(ex_gw_port['id']) ++ effective_interface_name = self.get_effective_interface_name(interface_name) ++ ex_gw_ip = ex_gw_port['fixed_ips'][0]['ip_address'] ++ ++ if not self.conf.use_namespaces: ++ net = netaddr.IPNetwork(ex_gw_ip) ++ device = ip_lib.IPDevice(effective_interface_name, self.root_helper, namespace=ri.ns_name()) ++ device.addr.delete(net.version, ex_gw_ip) ++ + if ip_lib.device_exists(interface_name, + root_helper=self.root_helper, + namespace=ri.ns_name()): +@@ -441,9 +475,7 @@ class L3NATAgent(manager.Manager): + namespace=ri.ns_name(), + prefix=EXTERNAL_DEV_PREFIX) + +- ex_gw_ip = ex_gw_port['fixed_ips'][0]['ip_address'] +- for c, r in self.external_gateway_nat_rules(ex_gw_ip, internal_cidrs, +- interface_name): ++ for c, r in self.external_gateway_nat_rules(ex_gw_ip, internal_cidrs, effective_interface_name): + ri.iptables_manager.ipv4['nat'].remove_rule(c, r) + ri.iptables_manager.apply() + +@@ -463,6 +495,7 @@ class L3NATAgent(manager.Manager): + + def external_gateway_nat_rules(self, ex_gw_ip, internal_cidrs, + interface_name): ++ ### SV: need-patch !!! + rules = [('POSTROUTING', '! -i %(interface_name)s ' + '! -o %(interface_name)s -m conntrack ! ' + '--ctstate DNAT -j ACCEPT' % locals())] +@@ -475,7 +508,7 @@ class L3NATAgent(manager.Manager): + interface_name = self.get_internal_device_name(port_id) + if not ip_lib.device_exists(interface_name, + root_helper=self.root_helper, +- namespace=ri.ns_name()): ++ namespace=ri.ns_name()): # check existing MAC address of qr_* + self.driver.plug(network_id, port_id, interface_name, mac_address, + namespace=ri.ns_name(), + prefix=INTERNAL_DEV_PREFIX) +@@ -513,27 +546,38 @@ class L3NATAgent(manager.Manager): + return rules + + def floating_ip_added(self, ri, ex_gw_port, floating_ip, fixed_ip): ++ ### SV: patched !!! + ip_cidr = str(floating_ip) + '/32' + interface_name = self.get_external_device_name(ex_gw_port['id']) +- device = ip_lib.IPDevice(interface_name, self.root_helper, +- namespace=ri.ns_name()) ++ effective_interface_name = self.get_effective_interface_name(interface_name) ++ if not self.conf.use_namespaces: ++ LOG.info("Warning! You are not using network namespaces! " ++ "Floating IP {ip} will be assigned to interface {iface}, instead {eiface}." ++ .format( ++ ip=ip_cidr, ++ iface=interface_name, ++ eiface=effective_interface_name ++ ) ++ ) ++ device = ip_lib.IPDevice(effective_interface_name, self.root_helper, namespace=ri.ns_name()) + + if ip_cidr not in [addr['cidr'] for addr in device.addr.list()]: + net = netaddr.IPNetwork(ip_cidr) + device.addr.add(net.version, ip_cidr, str(net.broadcast)) +- self._send_gratuitous_arp_packet(ri, interface_name, floating_ip) ++ self._send_gratuitous_arp_packet(ri, effective_interface_name, floating_ip) + + for chain, rule in self.floating_forward_rules(floating_ip, fixed_ip): + ri.iptables_manager.ipv4['nat'].add_rule(chain, rule) + ri.iptables_manager.apply() + + def floating_ip_removed(self, ri, ex_gw_port, floating_ip, fixed_ip): ++ ### SV: patched !!! + ip_cidr = str(floating_ip) + '/32' + net = netaddr.IPNetwork(ip_cidr) + interface_name = self.get_external_device_name(ex_gw_port['id']) ++ effective_interface_name = self.get_effective_interface_name(interface_name) + +- device = ip_lib.IPDevice(interface_name, self.root_helper, +- namespace=ri.ns_name()) ++ device = ip_lib.IPDevice(effective_interface_name, self.root_helper, namespace=ri.ns_name()) + device.addr.delete(net.version, ip_cidr) + + for chain, rule in self.floating_forward_rules(floating_ip, fixed_ip): +@@ -642,7 +686,16 @@ class L3NATAgent(manager.Manager): + self.fullsync = True + + def after_start(self): +- LOG.info(_("L3 agent started")) ++ LOG.info(_("L3 agent started (Mirantis-patched)")) ++ LOG.debug("*Ext.bridge name is '{0}'".format(self.conf.external_network_bridge)) ++ if not self.conf.use_namespaces: ++ LOG.info("Warning!!!") ++ LOG.info("You are not using network namespaces! " ++ "Floating IPs will be assigned to interface {iface}".format( ++ iface=self.conf.external_network_bridge ++ ) ++ ) ++ + + def _update_routing_table(self, ri, operation, route): + cmd = ['ip', 'route', operation, 'to', route['destination'], diff --git a/debian/patches/mysql-reconnect.patch b/debian/patches/mysql-reconnect.patch new file mode 100644 index 000000000..8b0daeb96 --- /dev/null +++ b/debian/patches/mysql-reconnect.patch @@ -0,0 +1,150 @@ +diff --git a/neunton/common/exceptions.py b/neunton/common/exceptions.py +index c99c254..e24f7bc 100644 +--- a/neunton/common/exceptions.py ++++ b/neunton/common/exceptions.py +@@ -235,3 +235,7 @@ class InvalidSharedSetting(QuantumException): + + class InvalidExtenstionEnv(QuantumException): + message = _("Invalid extension environment: %(reason)s") ++ ++class DBError(Error): ++ message = _("Database error") ++ +diff --git a/neunton/db/api.py b/neunton/db/api.py +index 238a9f9..737c748 100644 +--- a/neunton/db/api.py ++++ b/neunton/db/api.py +@@ -20,12 +20,16 @@ + import logging + import time + ++import time ++ + import sqlalchemy as sql + from sqlalchemy import create_engine + from sqlalchemy.exc import DisconnectionError ++from sqlalchemy.exc import OperationalError + from sqlalchemy.orm import sessionmaker, exc + + from quantum.db import model_base ++from quantum.common.exceptions import DBError + + LOG = logging.getLogger(__name__) + +@@ -33,28 +37,61 @@ LOG = logging.getLogger(__name__) + _ENGINE = None + _MAKER = None + BASE = model_base.BASE ++OPTIONS = None + ++def is_db_connection_error(args): ++ """Return True if error in connecting to db.""" ++ # NOTE(adam_g): This is currently MySQL specific and needs to be extended ++ # to support Postgres and others. ++ conn_err_codes = ('2002', '2003', '2006', '2013', '2014', '2045', '2055') ++ for err_code in conn_err_codes: ++ if args.find(err_code) != -1: ++ return True ++ return False + +-class MySQLPingListener(object): + +- """ +- Ensures that MySQL connections checked out of the +- pool are alive. ++def wrap_db_error(f): ++ """Function wrapper to capture DB errors + +- Borrowed from: +- http://groups.google.com/group/sqlalchemy/msg/a4ce563d802c929f +- """ ++ If an exception is thrown by the wrapped function, ++ determine if it represents a database connection error. ++ If so, retry the wrapped function, and repeat until it succeeds ++ or we reach a configurable maximum number of retries. ++ If it is not a connection error, or we exceeded the retry limit, ++ raise a DBError. + +- def checkout(self, dbapi_con, con_record, con_proxy): +- try: +- dbapi_con.cursor().execute('select 1') +- except dbapi_con.OperationalError, ex: +- if ex.args[0] in (2006, 2013, 2014, 2045, 2055): +- LOG.warn('Got mysql server has gone away: %s', ex) +- raise DisconnectionError("Database server went away") +- else: ++ """ ++ global OPTIONS ++ def _wrap_db_error(*args, **kwargs): ++ next_interval = OPTIONS.get('reconnect_interval', 1) ++ remaining = OPTIONS.get('sql_max_retries', -1) ++ if remaining == -1: ++ remaining = 'infinite' ++ while True: ++ try: ++ return f(*args, **kwargs) ++ except OperationalError, e: ++ if is_db_connection_error(e.args[0]): ++ if remaining == 0: ++ logging.warn('DB exceeded retry limit.') ++ raise DBError(e) ++ if remaining != 'infinite': ++ remaining -= 1 ++ logging.warn('DB connection error, ' ++ 'retrying in %i seconds.' % next_interval) ++ time.sleep(next_interval) ++ if OPTIONS.get('inc_reconnect_interval', True): ++ next_interval = min(next_interval * 2, ++ OPTIONS.get('max_reconnect_interval', 60)) ++ else: ++ logging.warn('DB exception wrapped.') ++ raise DBError(e) ++ except Exception, e: + raise + ++ _wrap_db_error.func_name = f.func_name ++ return _wrap_db_error ++ + + def configure_db(options): + """ +@@ -63,6 +100,8 @@ def configure_db(options): + + :param options: Mapping of configuration options + """ ++ global OPTIONS ++ OPTIONS = options + global _ENGINE + if not _ENGINE: + connection_dict = sql.engine.url.make_url(options['sql_connection']) +@@ -72,9 +111,6 @@ def configure_db(options): + 'convert_unicode': True, + } + +- if 'mysql' in connection_dict.drivername: +- engine_args['listeners'] = [MySQLPingListener()] +- + _ENGINE = create_engine(options['sql_connection'], **engine_args) + base = options.get('base', BASE) + if not register_models(base): +@@ -101,10 +137,18 @@ def get_session(autocommit=True, expire_on_commit=False): + global _MAKER, _ENGINE + if not _MAKER: + assert _ENGINE ++ class OurQuery(sql.orm.query.Query): ++ pass ++ query = OurQuery ++ query.all = wrap_db_error(query.all) ++ query.first = wrap_db_error(query.first) + _MAKER = sessionmaker(bind=_ENGINE, + autocommit=autocommit, +- expire_on_commit=expire_on_commit) +- return _MAKER() ++ expire_on_commit=expire_on_commit, ++ query_cls=OurQuery) ++ session = _MAKER() ++ session.flush = wrap_db_error(session.flush) ++ return session + + + def retry_registration(remaining, reconnect_interval, base=BASE): diff --git a/debian/patches/series b/debian/patches/series index aa0010058..3c4b61d99 100644 --- a/debian/patches/series +++ b/debian/patches/series @@ -3,3 +3,5 @@ disable-udev-tests.patch bump-sqlalchemy-version.patch disable-ml2-notification-tests.patch remove-jsonrpclib.patch +fix-l3-agent.patch +mysql-reconnect.patch -- 2.32.3