From: Carl Baldwin Date: Tue, 27 Jan 2015 22:34:06 +0000 (+0000) Subject: Move ha router functionality from the agent to ha_router X-Git-Url: https://review.fuel-infra.org/gitweb?a=commitdiff_plain;h=9d1db4a01c4e617109d742e44e54fc9588a2b856;p=openstack-build%2Fneutron-build.git Move ha router functionality from the agent to ha_router This was pretty much a mechanical change. Most of the methods passed in ri as the first argument and so changing the way they're called was a simple matter of replacing self with ri. In the methods themselves, I first made sure that self was not referenced for anything that wasn't available in the router context and then replaced ri with self. Change-Id: I2f9e23543a10da256d33c4c32a50aa2b8c93989d Partially-Implements: bp/restructure-l3-agent --- diff --git a/neutron/agent/l3/agent.py b/neutron/agent/l3/agent.py index c38cd6dee..6020ce6a9 100644 --- a/neutron/agent/l3/agent.py +++ b/neutron/agent/l3/agent.py @@ -687,7 +687,7 @@ class L3NATAgent(firewall_l3_agent.FWaaSL3AgentRpcCallback, ip_cidr = common_utils.ip_to_cidr(fip_ip) if ri.is_ha: - self._add_vip(ri, ip_cidr, interface_name) + ri._add_vip(ip_cidr, interface_name) else: net = netaddr.IPNetwork(ip_cidr) try: @@ -714,7 +714,7 @@ class L3NATAgent(firewall_l3_agent.FWaaSL3AgentRpcCallback, def _remove_floating_ip(self, ri, device, ip_cidr): if ri.is_ha: - self._remove_vip(ri, ip_cidr) + ri._remove_vip(ip_cidr) else: net = netaddr.IPNetwork(ip_cidr) device.addr.delete(net.version, ip_cidr) @@ -726,7 +726,7 @@ class L3NATAgent(firewall_l3_agent.FWaaSL3AgentRpcCallback, def _get_router_cidrs(self, ri, device): if ri.is_ha: - return set(self._ha_get_existing_cidrs(ri, device.name)) + return set(ri._ha_get_existing_cidrs(device.name)) else: return set([addr['cidr'] for addr in device.addr.list()]) @@ -824,8 +824,8 @@ class L3NATAgent(firewall_l3_agent.FWaaSL3AgentRpcCallback, ri.ns_name, preserve_ips) if ri.is_ha: - self._ha_external_gateway_added(ri, ex_gw_port, interface_name) - self._ha_disable_addressing_on_interface(ri, interface_name) + ri._ha_external_gateway_added(ex_gw_port, interface_name) + ri._ha_disable_addressing_on_interface(interface_name) def external_gateway_updated(self, ri, ex_gw_port, interface_name): preserve_ips = [] @@ -847,7 +847,7 @@ class L3NATAgent(firewall_l3_agent.FWaaSL3AgentRpcCallback, ns_name, preserve_ips) if ri.is_ha: - self._ha_external_gateway_updated(ri, ex_gw_port, interface_name) + ri._ha_external_gateway_updated(ex_gw_port, interface_name) def _external_gateway_added(self, ri, ex_gw_port, interface_name, ns_name, preserve_ips): @@ -893,7 +893,7 @@ class L3NATAgent(firewall_l3_agent.FWaaSL3AgentRpcCallback, ns_name = ri.ns_name if ri.is_ha: - self._ha_external_gateway_removed(ri, interface_name) + ri._ha_external_gateway_removed(interface_name) self.driver.unplug(interface_name, bridge=self.conf.external_network_bridge, @@ -945,8 +945,8 @@ class L3NATAgent(firewall_l3_agent.FWaaSL3AgentRpcCallback, ri.is_ha) if ri.is_ha: - self._ha_disable_addressing_on_interface(ri, interface_name) - self._add_vip(ri, internal_cidr, interface_name) + ri._ha_disable_addressing_on_interface(interface_name) + ri._add_vip(internal_cidr, interface_name) ex_gw_port = self._get_ex_gw_port(ri) if ri.router['distributed'] and ex_gw_port: @@ -995,7 +995,7 @@ class L3NATAgent(firewall_l3_agent.FWaaSL3AgentRpcCallback, root_helper=self.root_helper, namespace=ri.ns_name): if ri.is_ha: - self._clear_vips(ri, interface_name) + ri._clear_vips(interface_name) self.driver.unplug(interface_name, namespace=ri.ns_name, prefix=INTERNAL_DEV_PREFIX) @@ -1204,7 +1204,7 @@ class L3NATAgent(firewall_l3_agent.FWaaSL3AgentRpcCallback, def routes_updated(self, ri): new_routes = ri.router['routes'] if ri.is_ha: - self._process_virtual_routes(ri, new_routes) + ri._process_virtual_routes(new_routes) return old_routes = ri.routes diff --git a/neutron/agent/l3/ha.py b/neutron/agent/l3/ha.py index 9e902d4a3..a344cd439 100644 --- a/neutron/agent/l3/ha.py +++ b/neutron/agent/l3/ha.py @@ -14,14 +14,9 @@ # under the License. import os -import signal -import netaddr from oslo.config import cfg -from neutron.agent.linux import ip_lib -from neutron.agent.linux import keepalived -from neutron.agent.metadata import driver as metadata_driver from neutron.common import constants as l3_constants from neutron.i18n import _LE from neutron.openstack.common import log as logging @@ -58,39 +53,6 @@ class AgentMixin(object): if not os.path.isdir(ha_full_path): os.makedirs(ha_full_path, 0o755) - def get_keepalived_manager(self, ri): - return keepalived.KeepalivedManager( - ri.router['id'], - keepalived.KeepalivedConf(), - conf_path=self.conf.ha_confs_path, - namespace=ri.ns_name, - root_helper=self.root_helper) - - def _init_keepalived_manager(self, ri): - ri.keepalived_manager = self.get_keepalived_manager(ri) - - config = ri.keepalived_manager.config - - interface_name = self.get_ha_device_name(ri.ha_port['id']) - ha_port_cidr = ri.ha_port['subnet']['cidr'] - instance = keepalived.KeepalivedInstance( - 'BACKUP', interface_name, ri.ha_vr_id, ha_port_cidr, - nopreempt=True, advert_int=self.conf.ha_vrrp_advert_int, - priority=ri.ha_priority) - instance.track_interfaces.append(interface_name) - - if self.conf.ha_vrrp_auth_password: - # TODO(safchain): use oslo.config types when it will be available - # in order to check the validity of ha_vrrp_auth_type - instance.set_authentication(self.conf.ha_vrrp_auth_type, - self.conf.ha_vrrp_auth_password) - - group = keepalived.KeepalivedGroup(ri.ha_vr_id) - group.add_instance(instance) - - config.add_group(group) - config.add_instance(instance) - def process_ha_router_added(self, ri): ha_port = ri.router.get(l3_constants.HA_INTERFACE_KEY) if not ha_port: @@ -99,140 +61,17 @@ class AgentMixin(object): return self._set_subnet_info(ha_port) - self.ha_network_added(ri, ha_port['network_id'], ha_port['id'], - ha_port['ip_cidr'], ha_port['mac_address']) + ri.ha_network_added(ha_port['network_id'], + ha_port['id'], + ha_port['ip_cidr'], + ha_port['mac_address']) ri.ha_port = ha_port - self._init_keepalived_manager(ri) - self._add_keepalived_notifiers(ri) + ri._init_keepalived_manager() + ri._add_keepalived_notifiers() def process_ha_router_removed(self, ri): - self.ha_network_removed(ri) - - def get_ha_device_name(self, port_id): - return (HA_DEV_PREFIX + port_id)[:self.driver.DEV_NAME_LEN] - - def ha_network_added(self, ri, network_id, port_id, internal_cidr, - mac_address): - interface_name = self.get_ha_device_name(port_id) - self.driver.plug(network_id, port_id, interface_name, mac_address, - namespace=ri.ns_name, - prefix=HA_DEV_PREFIX) - self.driver.init_l3(interface_name, [internal_cidr], - namespace=ri.ns_name) - - def ha_network_removed(self, ri): - interface_name = self.get_ha_device_name(ri.ha_port['id']) - self.driver.unplug(interface_name, namespace=ri.ns_name, - prefix=HA_DEV_PREFIX) - ri.ha_port = None - - def _add_vip(self, ri, ip_cidr, interface, scope=None): - instance = ri.keepalived_manager.config.get_instance(ri.ha_vr_id) - instance.add_vip(ip_cidr, interface, scope) - - def _remove_vip(self, ri, ip_cidr): - instance = ri.keepalived_manager.config.get_instance(ri.ha_vr_id) - instance.remove_vip_by_ip_address(ip_cidr) - - def _clear_vips(self, ri, interface): - instance = ri.keepalived_manager.config.get_instance(ri.ha_vr_id) - instance.remove_vips_vroutes_by_interface(interface) - - def _ha_get_existing_cidrs(self, ri, interface_name): - instance = ri.keepalived_manager.config.get_instance(ri.ha_vr_id) - return instance.get_existing_vip_ip_addresses(interface_name) - - def _add_keepalived_notifiers(self, ri): - callback = ( - metadata_driver.MetadataDriver._get_metadata_proxy_callback( - ri.router_id, self.conf)) - # TODO(mangelajo): use the process monitor in keepalived when - # keepalived stops killing/starting metadata - # proxy on its own - pm = ( - metadata_driver.MetadataDriver. - _get_metadata_proxy_process_manager(ri.router_id, - ri.ns_name, - self.conf)) - pid = pm.get_pid_file_name() - ri.keepalived_manager.add_notifier( - callback(pid), 'master', ri.ha_vr_id) - for state in ('backup', 'fault'): - ri.keepalived_manager.add_notifier( - ['kill', '-%s' % signal.SIGKILL, - '$(cat ' + pid + ')'], state, ri.ha_vr_id) - - def _ha_external_gateway_updated(self, ri, ex_gw_port, interface_name): - old_gateway_cidr = ri.ex_gw_port['ip_cidr'] - self._remove_vip(ri, old_gateway_cidr) - self._ha_external_gateway_added(ri, ex_gw_port, interface_name) - - def _add_default_gw_virtual_route(self, ri, ex_gw_port, interface_name): - gw_ip = ex_gw_port['subnet']['gateway_ip'] - if gw_ip: - instance = ri.keepalived_manager.config.get_instance(ri.ha_vr_id) - instance.virtual_routes = ( - [route for route in instance.virtual_routes - if route.destination != '0.0.0.0/0']) - instance.virtual_routes.append( - keepalived.KeepalivedVirtualRoute( - '0.0.0.0/0', gw_ip, interface_name)) - - def _ha_external_gateway_added(self, ri, ex_gw_port, interface_name): - self._add_vip(ri, ex_gw_port['ip_cidr'], interface_name) - self._add_default_gw_virtual_route(ri, ex_gw_port, interface_name) - - def _should_delete_ipv6_lladdr(self, ri, ipv6_lladdr): - """Only the master should have any IP addresses configured. - Let keepalived manage IPv6 link local addresses, the same way we let - it manage IPv4 addresses. In order to do that, we must delete - the address first as it is autoconfigured by the kernel. - """ - process = keepalived.KeepalivedManager.get_process( - self.conf, - ri.router_id, - self.root_helper, - ri.ns_name, - self.conf.ha_confs_path) - if process.active: - manager = self.get_keepalived_manager(ri) - conf = manager.get_conf_on_disk() - managed_by_keepalived = conf and ipv6_lladdr in conf - if managed_by_keepalived: - return False - return True - - def _ha_disable_addressing_on_interface(self, ri, interface_name): - """Disable IPv6 link local addressing on the device and add it as - a VIP to keepalived. This means that the IPv6 link local address - will only be present on the master. - """ - device = ip_lib.IPDevice(interface_name, self.root_helper, ri.ns_name) - ipv6_lladdr = self._get_ipv6_lladdr(device.link.address) - - if self._should_delete_ipv6_lladdr(ri, ipv6_lladdr): - device.addr.flush() - - self._remove_vip(ri, ipv6_lladdr) - self._add_vip(ri, ipv6_lladdr, interface_name, scope='link') - - def _get_ipv6_lladdr(self, mac_addr): - return '%s/64' % netaddr.EUI(mac_addr).ipv6_link_local() - - def _ha_external_gateway_removed(self, ri, interface_name): - self._clear_vips(ri, interface_name) - - def _process_virtual_routes(self, ri, new_routes): - instance = ri.keepalived_manager.config.get_instance(ri.ha_vr_id) - - # Filter out all of the old routes while keeping only the default route - instance.virtual_routes = [route for route in instance.virtual_routes - if route.destination == '0.0.0.0/0'] - for route in new_routes: - instance.virtual_routes.append(keepalived.KeepalivedVirtualRoute( - route['destination'], - route['nexthop'])) + ri.ha_network_removed() def get_ha_routers(self): return (router for router in self.router_info.values() if router.is_ha) diff --git a/neutron/agent/l3/ha_router.py b/neutron/agent/l3/ha_router.py index 168f06443..01b013b6d 100644 --- a/neutron/agent/l3/ha_router.py +++ b/neutron/agent/l3/ha_router.py @@ -12,13 +12,18 @@ # License for the specific language governing permissions and limitations # under the License. +import netaddr import shutil +import signal from neutron.agent.l3 import router_info as router +from neutron.agent.linux import ip_lib from neutron.agent.linux import keepalived +from neutron.agent.metadata import driver as metadata_driver from neutron.openstack.common import log as logging LOG = logging.getLogger(__name__) +HA_DEV_PREFIX = 'ha-' class HaRouter(router.RouterInfo): @@ -61,6 +66,44 @@ class HaRouter(router.RouterInfo): LOG.debug('Error while reading HA state for %s', self.router_id) return None + def get_keepalived_manager(self): + return keepalived.KeepalivedManager( + self.router['id'], + keepalived.KeepalivedConf(), + conf_path=self.agent_conf.ha_confs_path, + namespace=self.ns_name, + root_helper=self.root_helper) + + def _init_keepalived_manager(self): + # TODO(Carl) This looks a bit funny, doesn't it? + self.keepalived_manager = self.get_keepalived_manager() + + config = self.keepalived_manager.config + + interface_name = self.get_ha_device_name(self.ha_port['id']) + ha_port_cidr = self.ha_port['subnet']['cidr'] + instance = keepalived.KeepalivedInstance( + 'BACKUP', + interface_name, + self.ha_vr_id, + ha_port_cidr, + nopreempt=True, + advert_int=self.agent_conf.ha_vrrp_advert_int, + priority=self.ha_priority) + instance.track_interfaces.append(interface_name) + + if self.agent_conf.ha_vrrp_auth_password: + # TODO(safchain): use oslo.config types when it will be available + # in order to check the validity of ha_vrrp_auth_type + instance.set_authentication(self.agent_conf.ha_vrrp_auth_type, + self.agent_conf.ha_vrrp_auth_password) + + group = keepalived.KeepalivedGroup(self.ha_vr_id) + group.add_instance(instance) + + config.add_group(group) + config.add_instance(instance) + def spawn_keepalived(self): self.keepalived_manager.spawn_or_restart() @@ -68,3 +111,134 @@ class HaRouter(router.RouterInfo): self.keepalived_manager.disable() conf_dir = self.keepalived_manager.get_conf_dir() shutil.rmtree(conf_dir) + + def _add_keepalived_notifiers(self): + callback = ( + metadata_driver.MetadataDriver._get_metadata_proxy_callback( + self.router_id, self.agent_conf)) + # TODO(mangelajo): use the process monitor in keepalived when + # keepalived stops killing/starting metadata + # proxy on its own + pm = ( + metadata_driver.MetadataDriver. + _get_metadata_proxy_process_manager(self.router_id, + self.ns_name, + self.agent_conf)) + pid = pm.get_pid_file_name() + self.keepalived_manager.add_notifier( + callback(pid), 'master', self.ha_vr_id) + for state in ('backup', 'fault'): + self.keepalived_manager.add_notifier( + ['kill', '-%s' % signal.SIGKILL, + '$(cat ' + pid + ')'], state, self.ha_vr_id) + + def _get_keepalived_instance(self): + return self.keepalived_manager.config.get_instance(self.ha_vr_id) + + def get_ha_device_name(self, port_id): + return (HA_DEV_PREFIX + port_id)[:self.driver.DEV_NAME_LEN] + + def ha_network_added(self, network_id, port_id, internal_cidr, + mac_address): + interface_name = self.get_ha_device_name(port_id) + self.driver.plug(network_id, port_id, interface_name, mac_address, + namespace=self.ns_name, + prefix=HA_DEV_PREFIX) + self.driver.init_l3(interface_name, [internal_cidr], + namespace=self.ns_name) + + def ha_network_removed(self): + interface_name = self.get_ha_device_name(self.ha_port['id']) + self.driver.unplug(interface_name, namespace=self.ns_name, + prefix=HA_DEV_PREFIX) + self.ha_port = None + + def _add_vip(self, ip_cidr, interface, scope=None): + instance = self._get_keepalived_instance() + instance.add_vip(ip_cidr, interface, scope) + + def _remove_vip(self, ip_cidr): + instance = self._get_keepalived_instance() + instance.remove_vip_by_ip_address(ip_cidr) + + def _clear_vips(self, interface): + instance = self._get_keepalived_instance() + instance.remove_vips_vroutes_by_interface(interface) + + def _ha_get_existing_cidrs(self, interface_name): + instance = self._get_keepalived_instance() + return instance.get_existing_vip_ip_addresses(interface_name) + + def _ha_external_gateway_removed(self, interface_name): + self._clear_vips(interface_name) + + def _process_virtual_routes(self, new_routes): + instance = self._get_keepalived_instance() + + # Filter out all of the old routes while keeping only the default route + instance.virtual_routes = [route for route in instance.virtual_routes + if route.destination == '0.0.0.0/0'] + for route in new_routes: + instance.virtual_routes.append(keepalived.KeepalivedVirtualRoute( + route['destination'], + route['nexthop'])) + + def _add_default_gw_virtual_route(self, ex_gw_port, interface_name): + gw_ip = ex_gw_port['subnet']['gateway_ip'] + if gw_ip: + # TODO(Carl) This is repeated everywhere. A method would be nice. + instance = self._get_keepalived_instance() + instance.virtual_routes = ( + [route for route in instance.virtual_routes + if route.destination != '0.0.0.0/0']) + instance.virtual_routes.append( + keepalived.KeepalivedVirtualRoute( + '0.0.0.0/0', gw_ip, interface_name)) + + def _get_ipv6_lladdr(self, mac_addr): + return '%s/64' % netaddr.EUI(mac_addr).ipv6_link_local() + + def _should_delete_ipv6_lladdr(self, ipv6_lladdr): + """Only the master should have any IP addresses configured. + Let keepalived manage IPv6 link local addresses, the same way we let + it manage IPv4 addresses. In order to do that, we must delete + the address first as it is autoconfigured by the kernel. + """ + process = keepalived.KeepalivedManager.get_process( + self.agent_conf, + self.router_id, + self.root_helper, + self.ns_name, + self.agent_conf.ha_confs_path) + if process.active: + manager = self.get_keepalived_manager() + conf = manager.get_conf_on_disk() + managed_by_keepalived = conf and ipv6_lladdr in conf + if managed_by_keepalived: + return False + return True + + def _ha_disable_addressing_on_interface(self, interface_name): + """Disable IPv6 link local addressing on the device and add it as + a VIP to keepalived. This means that the IPv6 link local address + will only be present on the master. + """ + device = ip_lib.IPDevice(interface_name, + self.root_helper, + self.ns_name) + ipv6_lladdr = self._get_ipv6_lladdr(device.link.address) + + if self._should_delete_ipv6_lladdr(ipv6_lladdr): + device.addr.flush() + + self._remove_vip(ipv6_lladdr) + self._add_vip(ipv6_lladdr, interface_name, scope='link') + + def _ha_external_gateway_added(self, ex_gw_port, interface_name): + self._add_vip(ex_gw_port['ip_cidr'], interface_name) + self._add_default_gw_virtual_route(ex_gw_port, interface_name) + + def _ha_external_gateway_updated(self, ex_gw_port, interface_name): + old_gateway_cidr = self.ex_gw_port['ip_cidr'] + self._remove_vip(old_gateway_cidr) + self._ha_external_gateway_added(ex_gw_port, interface_name) diff --git a/neutron/tests/functional/agent/test_l3_agent.py b/neutron/tests/functional/agent/test_l3_agent.py index 7caae4157..03bfaf052 100755 --- a/neutron/tests/functional/agent/test_l3_agent.py +++ b/neutron/tests/functional/agent/test_l3_agent.py @@ -145,16 +145,16 @@ class L3AgentTestFramework(base.BaseOVSLinuxTestCase): def get_expected_keepalive_configuration(self, router): ha_confs_path = self.agent.conf.ha_confs_path router_id = router.router_id - ha_device_name = self.agent.get_ha_device_name(router.ha_port['id']) + ha_device_name = router.get_ha_device_name(router.ha_port['id']) ha_device_cidr = router.ha_port['ip_cidr'] external_port = self.agent._get_ex_gw_port(router) - ex_port_ipv6 = self.agent._get_ipv6_lladdr( + ex_port_ipv6 = router._get_ipv6_lladdr( external_port['mac_address']) external_device_name = self.agent.get_external_device_name( external_port['id']) external_device_cidr = external_port['ip_cidr'] internal_port = router.router[l3_constants.INTERFACE_KEY][0] - int_port_ipv6 = self.agent._get_ipv6_lladdr( + int_port_ipv6 = router._get_ipv6_lladdr( internal_port['mac_address']) internal_device_name = self.agent.get_internal_device_name( internal_port['id']) @@ -406,7 +406,7 @@ class L3AgentTestCase(L3AgentTestFramework): def _assert_ha_device(self, router): self.assertTrue(self.device_exists_with_ip_mac( router.router[l3_constants.HA_INTERFACE_KEY], - self.agent.get_ha_device_name, router.ns_name)) + router.get_ha_device_name, router.ns_name)) def _assert_no_ip_addresses_on_interface(self, router, interface): device = ip_lib.IPDevice(interface, self.root_helper, router.ns_name) @@ -441,7 +441,7 @@ class L3HATestFramework(L3AgentTestFramework): helpers.wait_until_true(lambda: router1.ha_state == 'master') helpers.wait_until_true(lambda: router2.ha_state == 'backup') - device_name = self.agent.get_ha_device_name( + device_name = router1.get_ha_device_name( router1.router[l3_constants.HA_INTERFACE_KEY]['id']) ha_device = ip_lib.IPDevice(device_name, self.root_helper, router1.ns_name) diff --git a/neutron/tests/unit/test_l3_agent.py b/neutron/tests/unit/test_l3_agent.py index a3bc72f9f..a1cb633ae 100644 --- a/neutron/tests/unit/test_l3_agent.py +++ b/neutron/tests/unit/test_l3_agent.py @@ -1060,8 +1060,8 @@ class TestBasicRouterOperations(base.BaseTestCase): device = mock.MagicMock() device.name.return_value = 'eth2' addresses = ['15.1.2.2/24', '15.1.2.3/32'] - agent._ha_get_existing_cidrs = mock.MagicMock() - agent._ha_get_existing_cidrs.return_value = addresses + ri._ha_get_existing_cidrs = mock.MagicMock() + ri._ha_get_existing_cidrs.return_value = addresses self.assertEqual(set(addresses), agent._get_router_cidrs(ri, device)) # TODO(mrsmith): refactor for DVR cases