]> review.fuel-infra Code Review - openstack-build/neutron-build.git/commitdiff
L3 Agent support for routers with HA and DVR
authorMichael Smith <michael.smith6@hp.com>
Fri, 5 Dec 2014 00:15:43 +0000 (16:15 -0800)
committerJohn Schwarz <jschwarz@redhat.com>
Tue, 6 Oct 2015 15:27:25 +0000 (18:27 +0300)
The main difference for DVR HA routers is where
the VRRP/keepalived logic is run and which ports
fall in the HA domain for DVR.  Instead of running
in the qrouter namespace, keepalived will run inside
the snat-namespace.  Therefore only snat ports will
fall under the control of the HA domain.

Partial-Bug: #1365473

Change-Id: If2962580397d39f72fd1fbbc1188a6958f00ff0c
Co-Authored-By: Michael Smith <michael.smith6@hp.com>
Co-Authored-By: Hardik Italia <hardik.italia@hp.com>
Co-Authored-By: Adolfo Duarte <adolfo.duarte@hp.com>
Co-Authored-By: John Schwarz <jschwarz@redhat.com>
neutron/agent/l3/agent.py
neutron/agent/l3/dvr_edge_ha_router.py [new file with mode: 0644]
neutron/agent/l3/dvr_edge_router.py
neutron/agent/l3/dvr_local_router.py
neutron/agent/l3/ha.py
neutron/agent/l3/ha_router.py
neutron/tests/functional/agent/test_l3_agent.py

index 570a6c420f6a2e5f0494a9a7d74f6d574dadefb4..cbf92ca58308eab3557c44336b3a19bb7772ce69 100644 (file)
@@ -25,6 +25,7 @@ from oslo_utils import timeutils
 
 from neutron.agent.common import utils as common_utils
 from neutron.agent.l3 import dvr
+from neutron.agent.l3 import dvr_edge_ha_router
 from neutron.agent.l3 import dvr_edge_router as dvr_router
 from neutron.agent.l3 import dvr_local_router as dvr_local_router
 from neutron.agent.l3 import ha
@@ -297,11 +298,6 @@ class L3NATAgent(firewall_l3_agent.FWaaSL3AgentRpcCallback,
                     raise Exception(msg)
 
     def _create_router(self, router_id, router):
-        # TODO(Carl) We need to support a router that is both HA and DVR.  The
-        # patch that enables it will replace these lines.  See bug #1365473.
-        if router.get('distributed') and router.get('ha'):
-            raise n_exc.DvrHaRouterNotSupported(router_id=router_id)
-
         args = []
         kwargs = {
             'router_id': router_id,
@@ -314,6 +310,13 @@ class L3NATAgent(firewall_l3_agent.FWaaSL3AgentRpcCallback,
         if router.get('distributed'):
             kwargs['agent'] = self
             kwargs['host'] = self.host
+
+        if router.get('distributed') and router.get('ha'):
+            if self.conf.agent_mode == l3_constants.L3_AGENT_MODE_DVR_SNAT:
+                kwargs['state_change_callback'] = self.enqueue_state_change
+                return dvr_edge_ha_router.DvrEdgeHaRouter(*args, **kwargs)
+
+        if router.get('distributed'):
             if self.conf.agent_mode == l3_constants.L3_AGENT_MODE_DVR_SNAT:
                 return dvr_router.DvrEdgeRouter(*args, **kwargs)
             else:
diff --git a/neutron/agent/l3/dvr_edge_ha_router.py b/neutron/agent/l3/dvr_edge_ha_router.py
new file mode 100644 (file)
index 0000000..5e254d6
--- /dev/null
@@ -0,0 +1,129 @@
+# Copyright (c) 2015 Hewlett-Packard Development Company, L.P.
+# All Rights Reserved.
+#
+#    Licensed under the Apache License, Version 2.0 (the "License"); you may
+#    not use this file except in compliance with the License. You may obtain
+#    a copy of the License at
+#
+#         http://www.apache.org/licenses/LICENSE-2.0
+#
+#    Unless required by applicable law or agreed to in writing, software
+#    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+#    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+#    License for the specific language governing permissions and limitations
+#    under the License.
+
+from oslo_log import log as logging
+
+from neutron.agent.l3.dvr_edge_router import DvrEdgeRouter
+from neutron.agent.l3 import dvr_snat_ns
+from neutron.agent.l3.ha_router import HaRouter
+from neutron.agent.l3.router_info import RouterInfo
+from neutron.common import constants as l3_constants
+
+LOG = logging.getLogger(__name__)
+
+
+class DvrEdgeHaRouter(DvrEdgeRouter, HaRouter):
+    """Router class which represents a centralized SNAT
+       DVR router with HA capabilities.
+    """
+
+    def __init__(self, agent, host, *args, **kwargs):
+        super(DvrEdgeHaRouter, self).__init__(agent, host,
+                                              *args, **kwargs)
+        self.enable_snat = None
+        self.snat_ports = None
+
+    @property
+    def ha_namespace(self):
+        if self.snat_namespace:
+            return self.snat_namespace.name
+        return None
+
+    def internal_network_added(self, port):
+        # Call RouterInfo's internal_network_added (Plugs the port, adds IP)
+        RouterInfo.internal_network_added(self, port)
+
+        for subnet in port['subnets']:
+            self._set_subnet_arp_info(subnet['id'])
+        self._snat_redirect_add_from_port(port)
+
+        if not self.get_ex_gw_port() or not self._is_this_snat_host():
+            return
+
+        sn_port = self.get_snat_port_for_internal_port(port)
+        if not sn_port:
+            return
+
+        self._plug_ha_router_port(
+            sn_port,
+            self._get_snat_int_device_name,
+            dvr_snat_ns.SNAT_INT_DEV_PREFIX)
+
+    def external_gateway_added(self, ex_gw_port, interface_name):
+        super(DvrEdgeHaRouter, self).external_gateway_added(
+            ex_gw_port, interface_name)
+        for port in self.get_snat_interfaces():
+            snat_interface_name = self._get_snat_int_device_name(port['id'])
+            self._disable_ipv6_addressing_on_interface(snat_interface_name)
+            self._add_vips(
+                self.get_snat_port_for_internal_port(port),
+                snat_interface_name)
+
+        self._add_gateway_vip(ex_gw_port, interface_name)
+        self._disable_ipv6_addressing_on_interface(interface_name)
+
+    def external_gateway_removed(self, ex_gw_port, interface_name):
+        for port in self.snat_ports:
+            snat_interface = self._get_snat_int_device_name(port['id'])
+            self.driver.unplug(snat_interface,
+                               namespace=self.ha_namespace,
+                               prefix=l3_constants.SNAT_INT_DEV_PREFIX)
+            self._clear_vips(snat_interface)
+        super(DvrEdgeHaRouter, self)._external_gateway_removed(
+            ex_gw_port, interface_name)
+        self._clear_vips(interface_name)
+
+    def external_gateway_updated(self, ex_gw_port, interface_name):
+        HaRouter.external_gateway_updated(self, ex_gw_port, interface_name)
+
+    def initialize(self, process_monitor):
+        self._create_snat_namespace()
+        super(DvrEdgeHaRouter, self).initialize(process_monitor)
+
+    def process(self, agent):
+        super(DvrEdgeHaRouter, self).process(agent)
+        if self.ha_port:
+            self.enable_keepalived()
+
+    def delete(self, agent):
+        super(DvrEdgeHaRouter, self).delete(agent)
+        if self.snat_namespace:
+            self.snat_namespace.delete()
+
+    def get_router_cidrs(self, device):
+        return RouterInfo.get_router_cidrs(self, device)
+
+    def _external_gateway_added(self, ex_gw_port, interface_name,
+                                ns_name, preserve_ips):
+        self._plug_external_gateway(ex_gw_port, interface_name, ns_name)
+
+    def _is_this_snat_host(self):
+        return (self.agent_conf.agent_mode
+                == l3_constants.L3_AGENT_MODE_DVR_SNAT)
+
+    def _dvr_internal_network_removed(self, port):
+        super(DvrEdgeHaRouter, self)._dvr_internal_network_removed(port)
+        sn_port = self.get_snat_port_for_internal_port(port, self.snat_ports)
+        if not sn_port:
+            return
+        self._clear_vips(self._get_snat_int_device_name(sn_port['id']))
+
+    def _plug_snat_port(self, port):
+        """Used by _create_dvr_gateway in DvrEdgeRouter."""
+        interface_name = self._get_snat_int_device_name(port['id'])
+        self.driver.plug(port['network_id'], port['id'],
+                         interface_name, port['mac_address'],
+                         namespace=self.snat_namespace.name,
+                         prefix=dvr_snat_ns.SNAT_INT_DEV_PREFIX)
index 2d4bd20e31ac802c12d7b2badf18b2c406f2ccae..72534d35ca237c5d8097a31da18b47256a4d2c57 100644 (file)
@@ -57,7 +57,7 @@ class DvrEdgeRouter(dvr_local_router.DvrLocalRouter):
                                         self.snat_namespace.name,
                                         preserve_ips=[])
 
-    def external_gateway_removed(self, ex_gw_port, interface_name):
+    def _external_gateway_removed(self, ex_gw_port, interface_name):
         super(DvrEdgeRouter, self).external_gateway_removed(ex_gw_port,
                                                             interface_name)
         if not self._is_this_snat_host() and not self.snat_namespace:
@@ -69,8 +69,12 @@ class DvrEdgeRouter(dvr_local_router.DvrLocalRouter):
                            bridge=self.agent_conf.external_network_bridge,
                            namespace=self.snat_namespace.name,
                            prefix=router.EXTERNAL_DEV_PREFIX)
-        self.snat_namespace.delete()
-        self.snat_namespace = None
+
+    def external_gateway_removed(self, ex_gw_port, interface_name):
+        self._external_gateway_removed(ex_gw_port, interface_name)
+        if self.snat_namespace:
+            self.snat_namespace.delete()
+            self.snat_namespace = None
 
     def internal_network_added(self, port):
         super(DvrEdgeRouter, self).internal_network_added(port)
@@ -115,18 +119,21 @@ class DvrEdgeRouter(dvr_local_router.DvrLocalRouter):
             self.driver.unplug(snat_interface, namespace=ns_name,
                                prefix=prefix)
 
+    def _plug_snat_port(self, port):
+        interface_name = self._get_snat_int_device_name(port['id'])
+        self._internal_network_added(
+            self.snat_namespace.name, port['network_id'],
+            port['id'], port['fixed_ips'],
+            port['mac_address'], interface_name,
+            dvr_snat_ns.SNAT_INT_DEV_PREFIX)
+
     def _create_dvr_gateway(self, ex_gw_port, gw_interface_name):
         """Create SNAT namespace."""
         snat_ns = self._create_snat_namespace()
         # connect snat_ports to br_int from SNAT namespace
         for port in self.get_snat_interfaces():
             # create interface_name
-            interface_name = self._get_snat_int_device_name(port['id'])
-            self._internal_network_added(
-                snat_ns.name, port['network_id'],
-                port['id'], port['fixed_ips'],
-                port['mac_address'], interface_name,
-                dvr_snat_ns.SNAT_INT_DEV_PREFIX)
+            self._plug_snat_port(port)
         self._external_gateway_added(ex_gw_port, gw_interface_name,
                                      snat_ns.name, preserve_ips=[])
         self.snat_iptables_manager = iptables_manager.IptablesManager(
index 1ecc79fc47736712cff7ba51e1f364cfefad9918..35a2c0af13f6cf4d599765e14e86785457ced6d8 100644 (file)
@@ -298,7 +298,9 @@ class DvrLocalRouter(dvr_router_base.DvrRouterBase):
         # external_gateway port or the agent_mode.
         for subnet in port['subnets']:
             self._set_subnet_arp_info(subnet['id'])
+        self._snat_redirect_add_from_port(port)
 
+    def _snat_redirect_add_from_port(self, port):
         ex_gw_port = self.get_ex_gw_port()
         if not ex_gw_port:
             return
index b66e1a37d2287ad502d6d6f29b4dab3c3e22946e..bafc7e71c637c8ba53e64634ccdefd900ee83a3f 100644 (file)
@@ -137,7 +137,11 @@ class AgentMixin(object):
             gateway_ips = ri._get_external_gw_ips(ri.ex_gw_port)
             if not ri.is_v6_gateway_set(gateway_ips):
                 interface_name = ri.get_external_device_name(ex_gw_port_id)
-                ri.driver.configure_ipv6_ra(ri.ns_name, interface_name)
+                if ri.router.get('distributed', False):
+                    namespace = ri.ha_namespace
+                else:
+                    namespace = ri.ns_name
+                ri.driver.configure_ipv6_ra(namespace, interface_name)
 
     def _update_metadata_proxy(self, ri, router_id, state):
         if state == 'master':
index b041da3899c764632f68700cd8738094934d4507..55d4eebb961be174a407091434bdda130d04867b 100644 (file)
@@ -69,6 +69,10 @@ class HaRouter(router.RouterInfo):
             LOG.error(_LE('Error while writing HA state for %s'),
                       self.router_id)
 
+    @property
+    def ha_namespace(self):
+        return self.ns_name
+
     def initialize(self, process_monitor):
         super(HaRouter, self).initialize(process_monitor)
         ha_port = self.router.get(n_consts.HA_INTERFACE_KEY)
@@ -89,7 +93,7 @@ class HaRouter(router.RouterInfo):
             keepalived.KeepalivedConf(),
             process_monitor,
             conf_path=self.agent_conf.ha_confs_path,
-            namespace=self.ns_name)
+            namespace=self.ha_namespace)
 
         config = self.keepalived_manager.config
 
@@ -138,19 +142,26 @@ class HaRouter(router.RouterInfo):
                          self.ha_port['id'],
                          interface_name,
                          self.ha_port['mac_address'],
-                         namespace=self.ns_name,
+                         namespace=self.ha_namespace,
                          prefix=HA_DEV_PREFIX)
         ip_cidrs = common_utils.fixed_ip_cidrs(self.ha_port['fixed_ips'])
         self.driver.init_l3(interface_name, ip_cidrs,
-                            namespace=self.ns_name,
+                            namespace=self.ha_namespace,
                             preserve_ips=[self._get_primary_vip()])
 
     def ha_network_removed(self):
         self.driver.unplug(self.get_ha_device_name(),
-                           namespace=self.ns_name,
+                           namespace=self.ha_namespace,
                            prefix=HA_DEV_PREFIX)
         self.ha_port = None
 
+    def _add_vips(self, port, interface_name):
+        for ip_cidr in common_utils.fixed_ip_cidrs(port['fixed_ips']):
+            try:
+                self._add_vip(ip_cidr, interface_name)
+            except keepalived.VIPDuplicateAddressException:
+                LOG.debug("%s has already been added to keepalive", ip_cidr)
+
     def _add_vip(self, ip_cidr, interface, scope=None):
         instance = self._get_keepalived_instance()
         instance.add_vip(ip_cidr, interface, scope)
@@ -223,7 +234,7 @@ class HaRouter(router.RouterInfo):
         a VIP to keepalived. This means that the IPv6 link local address
         will only be present on the master.
         """
-        device = ip_lib.IPDevice(interface_name, namespace=self.ns_name)
+        device = ip_lib.IPDevice(interface_name, namespace=self.ha_namespace)
         ipv6_lladdr = ip_lib.get_ipv6_lladdr(device.link.address)
 
         if self._should_delete_ipv6_lladdr(ipv6_lladdr):
@@ -233,8 +244,7 @@ class HaRouter(router.RouterInfo):
         self._add_vip(ipv6_lladdr, interface_name, scope='link')
 
     def _add_gateway_vip(self, ex_gw_port, interface_name):
-        for ip_cidr in common_utils.fixed_ip_cidrs(ex_gw_port['fixed_ips']):
-            self._add_vip(ip_cidr, interface_name)
+        self._add_vips(ex_gw_port, interface_name)
         self._add_default_gw_virtual_route(ex_gw_port, interface_name)
         self._add_extra_subnet_onlink_routes(ex_gw_port, interface_name)
 
@@ -254,20 +264,23 @@ class HaRouter(router.RouterInfo):
         for ip_cidr in ip_cidrs:
             self._add_vip(ip_cidr, interface_name)
 
-    def internal_network_added(self, port):
+    def _plug_ha_router_port(self, port, name_getter, prefix):
         port_id = port['id']
-        interface_name = self.get_internal_device_name(port_id)
+        interface_name = name_getter(port_id)
 
         self.driver.plug(port['network_id'],
                          port_id,
                          interface_name,
                          port['mac_address'],
-                         namespace=self.ns_name,
-                         prefix=router.INTERNAL_DEV_PREFIX)
+                         namespace=self.ha_namespace,
+                         prefix=prefix)
 
         self._disable_ipv6_addressing_on_interface(interface_name)
-        for ip_cidr in common_utils.fixed_ip_cidrs(port['fixed_ips']):
-            self._add_vip(ip_cidr, interface_name)
+        self._add_vips(port, interface_name)
+
+    def internal_network_added(self, port):
+        self._plug_ha_router_port(
+            port, self.get_internal_device_name, router.INTERNAL_DEV_PREFIX)
 
     def internal_network_removed(self, port):
         super(HaRouter, self).internal_network_removed(port)
@@ -279,7 +292,7 @@ class HaRouter(router.RouterInfo):
         return external_process.ProcessManager(
             self.agent_conf,
             '%s.monitor' % self.router_id,
-            self.ns_name,
+            self.ha_namespace,
             default_cmd_callback=self._get_state_change_monitor_callback())
 
     def _get_state_change_monitor_callback(self):
@@ -290,7 +303,7 @@ class HaRouter(router.RouterInfo):
             cmd = [
                 'neutron-keepalived-state-change',
                 '--router_id=%s' % self.router_id,
-                '--namespace=%s' % self.ns_name,
+                '--namespace=%s' % self.ha_namespace,
                 '--conf_dir=%s' % self.keepalived_manager.get_conf_dir(),
                 '--monitor_interface=%s' % ha_device,
                 '--monitor_cidr=%s' % ha_cidr,
@@ -317,7 +330,7 @@ class HaRouter(router.RouterInfo):
     def update_initial_state(self, callback):
         ha_device = ip_lib.IPDevice(
             self.get_ha_device_name(),
-            self.ns_name)
+            self.ha_namespace)
         addresses = ha_device.addr.list()
         cidrs = (address['cidr'] for address in addresses)
         ha_cidr = self._get_primary_vip()
@@ -341,7 +354,8 @@ class HaRouter(router.RouterInfo):
         self._disable_ipv6_addressing_on_interface(interface_name)
 
     def external_gateway_updated(self, ex_gw_port, interface_name):
-        self._plug_external_gateway(ex_gw_port, interface_name, self.ns_name)
+        self._plug_external_gateway(
+            ex_gw_port, interface_name, self.ha_namespace)
         ip_cidrs = common_utils.fixed_ip_cidrs(self.ex_gw_port['fixed_ips'])
         for old_gateway_cidr in ip_cidrs:
             self._remove_vip(old_gateway_cidr)
index 84b16dfb36245b86accab98ea1c71bd369bd0235..02c83a39544b25df11b87382bbf901533fa8cea4 100644 (file)
@@ -80,7 +80,7 @@ class L3AgentTestFramework(base.BaseSudoTestCase):
         agent_config.register_process_monitor_opts(config)
         return config
 
-    def _configure_agent(self, host):
+    def _configure_agent(self, host, agent_mode='dvr_snat'):
         conf = self._get_config_opts()
         l3_agent_main.register_opts(conf)
         conf.set_override(
@@ -107,10 +107,14 @@ class L3AgentTestFramework(base.BaseSudoTestCase):
         conf.set_override('external_pids',
                           get_temp_file_path('external/pids'))
         conf.set_override('host', host)
+        conf.set_override('agent_mode', agent_mode)
         agent = neutron_l3_agent.L3NATAgentWithStateReport(host, conf)
 
         return agent
 
+    def _get_agent_ovs_integration_bridge(self, agent):
+        return get_ovs_bridge(agent.conf.ovs_integration_bridge)
+
     def generate_router_info(self, enable_ha, ip_version=4, extra_routes=True,
                              enable_fip=True, enable_snat=True,
                              dual_stack=False, v6_ext_gw_with_sub=True):
@@ -286,11 +290,13 @@ class L3AgentTestFramework(base.BaseSudoTestCase):
         for extra_route in router.router['routes']:
             self.assertIn(extra_route, routes)
 
-    def _assert_onlink_subnet_routes(self, router, ip_versions):
+    def _assert_onlink_subnet_routes(
+            self, router, ip_versions, namespace=None):
+        ns_name = namespace or router.ns_name
         routes = []
         for ip_version in ip_versions:
             _routes = ip_lib.get_routing_table(ip_version,
-                                               namespace=router.ns_name)
+                                               namespace=ns_name)
             routes.extend(_routes)
         routes = set(route['destination'] for route in routes)
         extra_subnets = router.get_ex_gw_port()['extra_subnets']
@@ -317,9 +323,23 @@ class L3AgentTestFramework(base.BaseSudoTestCase):
 
     def fail_ha_router(self, router):
         device_name = router.get_ha_device_name()
-        ha_device = ip_lib.IPDevice(device_name, router.ns_name)
+        ha_device = ip_lib.IPDevice(device_name, router.ha_namespace)
         ha_device.link.set_down()
 
+    @classmethod
+    def _get_addresses_on_device(cls, namespace, interface):
+        return [address['cidr'] for address in
+                ip_lib.IPDevice(interface, namespace=namespace).addr.list()]
+
+    def _assert_no_ip_addresses_on_interface(self, namespace, interface):
+        self.assertEqual(
+            [], self._get_addresses_on_device(namespace, interface))
+
+    def _assert_ip_address_on_interface(self,
+                                        namespace, interface, ip_address):
+        self.assertIn(
+            ip_address, self._get_addresses_on_device(namespace, interface))
+
 
 class L3AgentTestCase(L3AgentTestFramework):
 
@@ -615,8 +635,10 @@ class L3AgentTestCase(L3AgentTestFramework):
         slaac = l3_constants.IPV6_SLAAC
         slaac_mode = {'ra_mode': slaac, 'address_mode': slaac}
         subnet_modes = [slaac_mode] * 2
-        self._add_internal_interface_by_subnet(router.router, count=2,
-                ip_version=6, ipv6_subnet_modes=subnet_modes)
+        self._add_internal_interface_by_subnet(router.router,
+                                               count=2,
+                                               ip_version=6,
+                                               ipv6_subnet_modes=subnet_modes)
         router.process(self.agent)
 
         if enable_ha:
@@ -712,15 +734,6 @@ class L3AgentTestCase(L3AgentTestFramework):
             router.router[l3_constants.HA_INTERFACE_KEY],
             ha_router_dev_name_getter, router.ns_name))
 
-    @classmethod
-    def _get_addresses_on_device(cls, namespace, interface):
-        return [address['cidr'] for address in
-                ip_lib.IPDevice(interface, namespace=namespace).addr.list()]
-
-    def _assert_no_ip_addresses_on_interface(self, namespace, interface):
-        self.assertEqual(
-            [], self._get_addresses_on_device(namespace, interface))
-
     def test_ha_router_conf_on_restarted_agent(self):
         router_info = self.generate_router_info(enable_ha=True)
         router1 = self.manage_router(self.agent, router_info)
@@ -777,9 +790,8 @@ class L3HATestFramework(L3AgentTestFramework):
         super(L3HATestFramework, self).setUp()
         self.failover_agent = self._configure_agent('agent2')
 
-        br_int_1 = get_ovs_bridge(self.agent.conf.ovs_integration_bridge)
-        br_int_2 = get_ovs_bridge(
-            self.failover_agent.conf.ovs_integration_bridge)
+        br_int_1 = self._get_agent_ovs_integration_bridge(self.agent)
+        br_int_2 = self._get_agent_ovs_integration_bridge(self.failover_agent)
 
         veth1, veth2 = self.useFixture(net_helpers.VethFixture()).ports
         br_int_1.add_port(veth1.name)
@@ -1013,6 +1025,9 @@ class TestDvrRouter(L3AgentTestFramework):
     def test_dvr_router_lifecycle_without_ha_with_snat_with_fips(self):
         self._dvr_router_lifecycle(enable_ha=False, enable_snat=True)
 
+    def test_dvr_router_lifecycle_ha_with_snat_with_fips(self):
+        self._dvr_router_lifecycle(enable_ha=True, enable_snat=True)
+
     def _helper_create_dvr_router_fips_for_ext_network(
             self, agent_mode, **dvr_router_kwargs):
         self.agent.conf.agent_mode = agent_mode
@@ -1054,7 +1069,9 @@ class TestDvrRouter(L3AgentTestFramework):
         self._validate_fips_for_external_network(router2, fip2_ns)
 
     def _dvr_router_lifecycle(self, enable_ha=False, enable_snat=False,
-                              custom_mtu=2000):
+                              custom_mtu=2000,
+                              ip_version=4,
+                              dual_stack=False):
         '''Test dvr router lifecycle
 
         :param enable_ha: sets the ha value for the router.
@@ -1089,9 +1106,28 @@ class TestDvrRouter(L3AgentTestFramework):
         # manage the router (create it, create namespaces,
         # attach interfaces, etc...)
         router = self.manage_router(self.agent, router_info)
+        if enable_ha:
+            port = router.get_ex_gw_port()
+            interface_name = router.get_external_device_name(port['id'])
+            self._assert_no_ip_addresses_on_interface(router.ha_namespace,
+                                                      interface_name)
+            utils.wait_until_true(lambda: router.ha_state == 'master')
+
+            # Keepalived notifies of a state transition when it starts,
+            # not when it ends. Thus, we have to wait until keepalived finishes
+            # configuring everything. We verify this by waiting until the last
+            # device has an IP address.
+            device = router.router[l3_constants.INTERFACE_KEY][-1]
+            device_exists = functools.partial(
+                self.device_exists_with_ips_and_mac,
+                device,
+                router.get_internal_device_name,
+                router.ns_name)
+            utils.wait_until_true(device_exists)
 
         self.assertTrue(self._namespace_exists(router.ns_name))
-        self.assertTrue(self._metadata_proxy_exists(self.agent.conf, router))
+        utils.wait_until_true(
+            lambda: self._metadata_proxy_exists(self.agent.conf, router))
         self._assert_internal_devices(router)
         self._assert_dvr_external_device(router)
         self._assert_dvr_gateway(router)
@@ -1101,13 +1137,25 @@ class TestDvrRouter(L3AgentTestFramework):
         self._assert_metadata_chains(router)
         self._assert_extra_routes(router)
         self._assert_rfp_fpr_mtu(router, custom_mtu)
+        if enable_snat:
+            ip_versions = [4, 6] if (ip_version == 6 or dual_stack) else [4]
+            snat_ns_name = dvr_snat_ns.SnatNamespace.get_snat_ns_name(
+                router.router_id)
+            self._assert_onlink_subnet_routes(
+                router, ip_versions, snat_ns_name)
 
         self._delete_router(self.agent, router.router_id)
         self._assert_interfaces_deleted_from_ovs()
         self._assert_router_does_not_exist(router)
+        self._assert_snat_namespace_does_not_exist(router)
 
-    def generate_dvr_router_info(
-        self, enable_ha=False, enable_snat=False, **kwargs):
+    def generate_dvr_router_info(self,
+                                 enable_ha=False,
+                                 enable_snat=False,
+                                 agent=None,
+                                 **kwargs):
+        if not agent:
+            agent = self.agent
         router = l3_test_common.prepare_router_data(
             enable_snat=enable_snat,
             enable_floating_ip=True,
@@ -1115,11 +1163,11 @@ class TestDvrRouter(L3AgentTestFramework):
             **kwargs)
         internal_ports = router.get(l3_constants.INTERFACE_KEY, [])
         router['distributed'] = True
-        router['gw_port_host'] = self.agent.conf.host
-        router['gw_port']['binding:host_id'] = self.agent.conf.host
+        router['gw_port_host'] = agent.conf.host
+        router['gw_port']['binding:host_id'] = agent.conf.host
         floating_ip = router['_floatingips'][0]
         floating_ip['floating_network_id'] = router['gw_port']['network_id']
-        floating_ip['host'] = self.agent.conf.host
+        floating_ip['host'] = agent.conf.host
         floating_ip['port_id'] = internal_ports[0]['id']
         floating_ip['status'] = 'ACTIVE'
 
@@ -1199,9 +1247,12 @@ class TestDvrRouter(L3AgentTestFramework):
         # that the correct ports and ip addresses exist in the
         # snat_ns_name namespace
         if self.agent.conf.agent_mode == 'dvr_snat':
-            self.assertTrue(self.device_exists_with_ips_and_mac(
-                external_port, router.get_external_device_name,
-                snat_ns_name))
+            device_exists = functools.partial(
+                self.device_exists_with_ips_and_mac,
+                external_port,
+                router.get_external_device_name,
+                snat_ns_name)
+            utils.wait_until_true(device_exists)
         # if the agent is in dvr mode then the snat_ns_name namespace
         # should not be present at all:
         elif self.agent.conf.agent_mode == 'dvr':
@@ -1484,3 +1535,104 @@ class TestDvrRouter(L3AgentTestFramework):
             self.agent.context,
             floating_agent_gw_port[0]['network_id'])
         self.assertFalse(self._namespace_exists(fip_ns))
+
+    def _mocked_dvr_ha_router(self, agent):
+        r_info = self.generate_dvr_router_info(enable_ha=True,
+                                               enable_snat=True,
+                                               agent=agent)
+
+        r_snat_ns_name = namespaces.build_ns_name(dvr_snat_ns.SNAT_NS_PREFIX,
+                                                  r_info['id'])
+
+        mocked_r_snat_ns_name = r_snat_ns_name + '@' + agent.host
+        r_ns_name = namespaces.build_ns_name(namespaces.NS_PREFIX,
+                                             r_info['id'])
+
+        mocked_r_ns_name = r_ns_name + '@' + agent.host
+
+        return r_info, mocked_r_ns_name, mocked_r_snat_ns_name
+
+    def _setup_dvr_ha_agents(self):
+        self.agent.conf.agent_mode = 'dvr_snat'
+
+        self.failover_agent = self._configure_agent('agent2')
+        self.failover_agent.conf.agent_mode = 'dvr_snat'
+
+    def _setup_dvr_ha_bridges(self):
+        br_int_1 = self._get_agent_ovs_integration_bridge(self.agent)
+        br_int_2 = self._get_agent_ovs_integration_bridge(self.failover_agent)
+
+        veth1, veth2 = self.useFixture(net_helpers.VethFixture()).ports
+        br_int_1.add_port(veth1.name)
+        br_int_2.add_port(veth2.name)
+
+    def _create_dvr_ha_router(self, agent):
+        get_ns_name = mock.patch.object(namespaces.RouterNamespace,
+                                        '_get_ns_name').start()
+        get_snat_ns_name = mock.patch.object(dvr_snat_ns.SnatNamespace,
+                                             'get_snat_ns_name').start()
+        (r_info,
+         mocked_r_ns_name,
+         mocked_r_snat_ns_name) = self._mocked_dvr_ha_router(agent)
+        get_ns_name.return_value = mocked_r_ns_name
+        get_snat_ns_name.return_value = mocked_r_snat_ns_name
+        router = self.manage_router(agent, r_info)
+        return router
+
+    def _assert_ip_addresses_in_dvr_ha_snat_namespace(self, router):
+        namespace = router.ha_namespace
+        ex_gw_port = router.get_ex_gw_port()
+        snat_port = router.get_snat_interfaces()[0]
+        ex_gw_port_name = router.get_external_device_name(
+            ex_gw_port['id'])
+        snat_port_name = router._get_snat_int_device_name(
+            snat_port['id'])
+
+        ip = ex_gw_port["fixed_ips"][0]['ip_address']
+        prefix_len = ex_gw_port["fixed_ips"][0]['prefixlen']
+        ex_gw_port_cidr = ip + "/" + str(prefix_len)
+        ip = snat_port["fixed_ips"][0]['ip_address']
+        prefix_len = snat_port["fixed_ips"][0]['prefixlen']
+        snat_port_cidr = ip + "/" + str(prefix_len)
+
+        self._assert_ip_address_on_interface(namespace,
+                                             ex_gw_port_name,
+                                             ex_gw_port_cidr)
+        self._assert_ip_address_on_interface(namespace,
+                                             snat_port_name,
+                                             snat_port_cidr)
+
+    def _assert_no_ip_addresses_in_dvr_ha_snat_namespace(self, router):
+        namespace = router.ha_namespace
+        ex_gw_port = router.get_ex_gw_port()
+        snat_port = router.get_snat_interfaces()[0]
+        ex_gw_port_name = router.get_external_device_name(
+            ex_gw_port['id'])
+        snat_port_name = router._get_snat_int_device_name(
+            snat_port['id'])
+
+        self._assert_no_ip_addresses_on_interface(namespace,
+                                                  snat_port_name)
+        self._assert_no_ip_addresses_on_interface(namespace,
+                                                  ex_gw_port_name)
+
+    def test_dvr_ha_router_failover(self):
+        self._setup_dvr_ha_agents()
+        self._setup_dvr_ha_bridges()
+
+        router1 = self._create_dvr_ha_router(self.agent)
+        router2 = self._create_dvr_ha_router(self.failover_agent)
+
+        utils.wait_until_true(lambda: router1.ha_state == 'master')
+        utils.wait_until_true(lambda: router2.ha_state == 'backup')
+
+        self._assert_ip_addresses_in_dvr_ha_snat_namespace(router1)
+        self._assert_no_ip_addresses_in_dvr_ha_snat_namespace(router2)
+
+        self.fail_ha_router(router1)
+
+        utils.wait_until_true(lambda: router2.ha_state == 'master')
+        utils.wait_until_true(lambda: router1.ha_state == 'backup')
+
+        self._assert_ip_addresses_in_dvr_ha_snat_namespace(router2)
+        self._assert_no_ip_addresses_in_dvr_ha_snat_namespace(router1)