From d9155521539e5e67f0984c180267fc91cc17c738 Mon Sep 17 00:00:00 2001 From: Assaf Muller Date: Tue, 5 Aug 2014 23:47:18 +0300 Subject: [PATCH] Introduces a keepalived manager for HA This patch introduces a keepalived manager which will be used for the blueprint blueprint l3-high-availability. The manager can create a keepalived.conf compliant configuration, start, stop and restart the service, as well as create keepalived notification scripts. Implements: blueprint l3-high-availability Change-Id: I1ba9f332778f27de950d9e97d4fb4a337f6f26da Co-Authored-By: Assaf Muller --- etc/neutron/rootwrap.d/l3.filters | 4 + neutron/agent/linux/external_process.py | 16 +- neutron/agent/linux/keepalived.py | 371 ++++++++++++++++++ .../functional/agent/linux/test_keepalived.py | 60 +++ .../tests/unit/agent/linux/test_keepalived.py | 241 ++++++++++++ 5 files changed, 685 insertions(+), 7 deletions(-) create mode 100644 neutron/agent/linux/keepalived.py create mode 100644 neutron/tests/functional/agent/linux/test_keepalived.py create mode 100644 neutron/tests/unit/agent/linux/test_keepalived.py diff --git a/etc/neutron/rootwrap.d/l3.filters b/etc/neutron/rootwrap.d/l3.filters index 89195fccf..8a662b954 100644 --- a/etc/neutron/rootwrap.d/l3.filters +++ b/etc/neutron/rootwrap.d/l3.filters @@ -42,3 +42,7 @@ iptables-save: CommandFilter, iptables-save, root iptables-restore: CommandFilter, iptables-restore, root ip6tables-save: CommandFilter, ip6tables-save, root ip6tables-restore: CommandFilter, ip6tables-restore, root + +# Keepalived +keepalived: CommandFilter, keepalived, root +kill_keepalived: KillFilter, root, /usr/sbin/keepalived, -HUP, -15, -9 diff --git a/neutron/agent/linux/external_process.py b/neutron/agent/linux/external_process.py index af260e10a..672d70355 100644 --- a/neutron/agent/linux/external_process.py +++ b/neutron/agent/linux/external_process.py @@ -37,7 +37,7 @@ class ProcessManager(object): Note: The manager expects uuid to be in cmdline. """ def __init__(self, conf, uuid, root_helper='sudo', - namespace=None, service=None): + namespace=None, service=None, pids_path=None): self.conf = conf self.uuid = uuid self.root_helper = root_helper @@ -46,6 +46,7 @@ class ProcessManager(object): self.service_pid_fname = 'pid.' + service else: self.service_pid_fname = 'pid' + self.pids_path = pids_path or self.conf.external_pids def enable(self, cmd_callback, reload_cfg=False): if not self.active: @@ -67,18 +68,19 @@ class ProcessManager(object): utils.execute(cmd, self.root_helper) # In the case of shutting down, remove the pid file if sig == '9': - utils.remove_conf_file(self.conf.external_pids, + utils.remove_conf_file(self.pids_path, self.uuid, self.service_pid_fname) elif pid: - LOG.debug(_('Process for %(uuid)s pid %(pid)d is stale, ignoring ' - 'command'), {'uuid': self.uuid, 'pid': pid}) + LOG.debug('Process for %(uuid)s pid %(pid)d is stale, ignoring ' + 'signal %(signal)s', {'uuid': self.uuid, 'pid': pid, + 'signal': sig}) else: - LOG.debug(_('No process started for %s'), self.uuid) + LOG.debug('No process started for %s', self.uuid) def get_pid_file_name(self, ensure_pids_dir=False): """Returns the file name for a given kind of config file.""" - return utils.get_conf_file_name(self.conf.external_pids, + return utils.get_conf_file_name(self.pids_path, self.uuid, self.service_pid_fname, ensure_pids_dir) @@ -86,7 +88,7 @@ class ProcessManager(object): @property def pid(self): """Last known pid for this external process spawned for this uuid.""" - return utils.get_value_from_conf_file(self.conf.external_pids, + return utils.get_value_from_conf_file(self.pids_path, self.uuid, self.service_pid_fname, int) diff --git a/neutron/agent/linux/keepalived.py b/neutron/agent/linux/keepalived.py new file mode 100644 index 000000000..cb4aeebcf --- /dev/null +++ b/neutron/agent/linux/keepalived.py @@ -0,0 +1,371 @@ +# Copyright (C) 2014 eNovance SAS +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import itertools +import os +import stat + +from oslo.config import cfg + +from neutron.agent.linux import external_process +from neutron.agent.linux import utils +from neutron.common import exceptions +from neutron.openstack.common.gettextutils import _LW +from neutron.openstack.common import log as logging + +VALID_STATES = ['MASTER', 'BACKUP'] +VALID_NOTIFY_STATES = ['master', 'backup', 'fault'] +VALID_AUTH_TYPES = ['AH', 'PASS'] +HA_DEFAULT_PRIORITY = 50 + +LOG = logging.getLogger(__name__) + + +class InvalidInstanceStateException(exceptions.NeutronException): + message = (_('Invalid instance state: %%(state)s, valid states are: ' + '%(valid_states)s') % + {'valid_states': ', '.join(VALID_STATES)}) + + +class InvalidNotifyStateException(exceptions.NeutronException): + message = (_('Invalid notify state: %%(state)s, valid states are: ' + '%(valid_notify_states)s') % + {'valid_notify_states': ', '.join(VALID_NOTIFY_STATES)}) + + +class InvalidAuthenticationTypeExecption(exceptions.NeutronException): + message = (_('Invalid authentication type: %%(auth_type)s, ' + 'valid types are: %(valid_auth_types)s') % + {'valid_auth_types': ', '.join(VALID_AUTH_TYPES)}) + + +class KeepalivedVipAddress(object): + """A virtual address entry of a keepalived configuration.""" + + def __init__(self, ip_address, interface_name): + self.ip_address = ip_address + self.interface_name = interface_name + + def build_config(self): + return '%s dev %s' % (self.ip_address, self.interface_name) + + +class KeepalivedVirtualRoute(object): + """A virtual route entry of a keepalived configuration.""" + + def __init__(self, destination, nexthop, interface_name=None): + self.destination = destination + self.nexthop = nexthop + self.interface_name = interface_name + + def build_config(self): + output = '%s via %s' % (self.destination, self.nexthop) + if self.interface_name: + output += ' dev %s' % self.interface_name + return output + + +class KeepalivedGroup(object): + """Group section of a keepalived configuration.""" + + def __init__(self, ha_vr_id): + self.ha_vr_id = ha_vr_id + self.name = 'VG_%s' % ha_vr_id + self.instance_names = set() + self.notifiers = {} + + def add_instance(self, instance): + self.instance_names.add(instance.name) + + def set_notify(self, state, path): + if state not in VALID_NOTIFY_STATES: + raise InvalidNotifyStateException(state=state) + self.notifiers[state] = path + + def build_config(self): + return itertools.chain(['vrrp_sync_group %s {' % self.name, + ' group {'], + (' %s' % i for i in self.instance_names), + [' }'], + (' notify_%s "%s"' % (state, path) + for state, path in self.notifiers.items()), + ['}']) + + +class KeepalivedInstance(object): + """Instance section of a keepalived configuration.""" + + def __init__(self, state, interface, vrouter_id, + priority=HA_DEFAULT_PRIORITY, advert_int=None, + mcast_src_ip=None, nopreempt=False): + self.name = 'VR_%s' % vrouter_id + + if state not in VALID_STATES: + raise InvalidInstanceStateException(state=state) + + self.state = state + self.interface = interface + self.vrouter_id = vrouter_id + self.priority = priority + self.nopreempt = nopreempt + self.advert_int = advert_int + self.mcast_src_ip = mcast_src_ip + self.track_interfaces = [] + self.vips = [] + self.virtual_routes = [] + self.authentication = tuple() + + def set_authentication(self, auth_type, password): + if auth_type not in VALID_AUTH_TYPES: + raise InvalidAuthenticationTypeExecption(auth_type=auth_type) + + self.authentication = (auth_type, password) + + def remove_vips_vroutes_by_interface(self, interface_name): + self.vips = [vip for vip in self.vips + if vip.interface_name != interface_name] + + self.virtual_routes = [vroute for vroute in self.virtual_routes + if vroute.interface_name != interface_name] + + def remove_vip_by_ip_address(self, ip_address): + self.vips = [vip for vip in self.vips + if vip.ip_address != ip_address] + + def _build_track_interface_config(self): + return itertools.chain( + [' track_interface {'], + (' %s' % i for i in self.track_interfaces), + [' }']) + + def _build_vips_config(self): + vips_sorted = sorted(self.vips, key=lambda vip: vip.ip_address) + first_address = vips_sorted.pop(0) + + vips_result = [' virtual_ipaddress {', + ' %s' % first_address.build_config(), + ' }'] + if vips_sorted: + vips_result.extend( + itertools.chain([' virtual_ipaddress_excluded {'], + (' %s' % vip.build_config() + for vip in vips_sorted), + [' }'])) + + return vips_result + + def _build_virtual_routes_config(self): + return itertools.chain([' virtual_routes {'], + (' %s' % route.build_config() + for route in self.virtual_routes), + [' }']) + + def build_config(self): + config = ['vrrp_instance %s {' % self.name, + ' state %s' % self.state, + ' interface %s' % self.interface, + ' virtual_router_id %s' % self.vrouter_id, + ' priority %s' % self.priority] + + if self.nopreempt: + config.append(' nopreempt') + + if self.advert_int: + config.append(' advert_int %s' % self.advert_int) + + if self.authentication: + auth_type, password = self.authentication + authentication = [' authentication {', + ' auth_type %s' % auth_type, + ' auth_pass %s' % password, + ' }'] + config.extend(authentication) + + if self.mcast_src_ip: + config.append(' mcast_src_ip %s' % self.mcast_src_ip) + + if self.track_interfaces: + config.extend(self._build_track_interface_config()) + + if self.vips: + config.extend(self._build_vips_config()) + + if self.virtual_routes: + config.extend(self._build_virtual_routes_config()) + + config.append('}') + + return config + + +class KeepalivedConf(object): + """A keepalived configuration.""" + + def __init__(self): + self.reset() + + def reset(self): + self.groups = {} + self.instances = {} + + def add_group(self, group): + self.groups[group.ha_vr_id] = group + + def get_group(self, ha_vr_id): + return self.groups.get(ha_vr_id) + + def add_instance(self, instance): + self.instances[instance.vrouter_id] = instance + + def get_instance(self, vrouter_id): + return self.instances.get(vrouter_id) + + def build_config(self): + config = [] + + for group in self.groups.values(): + config.extend(group.build_config()) + + for instance in self.instances.values(): + config.extend(instance.build_config()) + + return config + + def get_config_str(self): + """Generates and returns the keepalived configuration. + + :return: Keepalived configuration string. + """ + return '\n'.join(self.build_config()) + + +class KeepalivedNotifierMixin(object): + def _get_notifier_path(self, state): + return self._get_full_config_file_path('notify_%s.sh' % state) + + def _write_notify_script(self, state, script): + name = self._get_notifier_path(state) + utils.replace_file(name, script) + st = os.stat(name) + os.chmod(name, st.st_mode | stat.S_IEXEC) + + return name + + def _prepend_shebang(self, script): + return '#!/usr/bin/env bash\n%s' % script + + def _append_state(self, script, state): + state_path = self._get_full_config_file_path('state') + return '%s\necho -n %s > %s' % (script, state, state_path) + + def add_notifier(self, script, state, ha_vr_id): + """Add a master, backup or fault notifier. + + These notifiers are executed when keepalived invokes a state + transition. Write a notifier to disk and add it to the + configuration. + """ + + script_with_prefix = self._prepend_shebang(' '.join(script)) + full_script = self._append_state(script_with_prefix, state) + self._write_notify_script(state, full_script) + + group = self.config.get_group(ha_vr_id) + group.set_notify(state, self._get_notifier_path(state)) + + def get_conf_dir(self): + confs_dir = os.path.abspath(os.path.normpath(self.conf_path)) + conf_dir = os.path.join(confs_dir, self.resource_id) + return conf_dir + + def _get_full_config_file_path(self, filename, ensure_conf_dir=True): + conf_dir = self.get_conf_dir() + if ensure_conf_dir and not os.path.isdir(conf_dir): + os.makedirs(conf_dir, 0o755) + return os.path.join(conf_dir, filename) + + +class KeepalivedManager(KeepalivedNotifierMixin): + """Wrapper for keepalived. + + This wrapper permits to write keepalived config files, to start/restart + keepalived process. + + """ + + def __init__(self, resource_id, config, conf_path='/tmp', + namespace=None, root_helper=None): + self.resource_id = resource_id + self.config = config + self.namespace = namespace + self.root_helper = root_helper + self.conf_path = conf_path + self.conf = cfg.CONF + self.process = None + self.spawned = False + + def _output_config_file(self): + config_str = self.config.get_config_str() + config_path = self._get_full_config_file_path('keepalived.conf') + utils.replace_file(config_path, config_str) + + return config_path + + def spawn(self): + config_path = self._output_config_file() + + self.process = external_process.ProcessManager( + self.conf, + self.resource_id, + self.root_helper, + self.namespace, + pids_path=self.conf_path) + + def callback(pid_file): + cmd = ['keepalived', '-P', + '-f', config_path, + '-p', pid_file, + '-r', '%s-vrrp' % pid_file] + return cmd + + self.process.enable(callback) + + self.spawned = True + LOG.debug('Keepalived spawned with config %s', config_path) + + def spawn_or_restart(self): + if self.process: + self.restart() + else: + self.spawn() + + def restart(self): + if self.process.active: + self._output_config_file() + self.process.reload_cfg() + else: + LOG.warn(_LW('A previous instance of keepalived seems to be dead, ' + 'unable to restart it, a new instance will be ' + 'spawned')) + self.process.disable() + self.spawn() + + def disable(self): + if self.process: + self.process.disable(sig='15') + self.spawned = False + + def revive(self): + if self.spawned and not self.process.active: + self.restart() diff --git a/neutron/tests/functional/agent/linux/test_keepalived.py b/neutron/tests/functional/agent/linux/test_keepalived.py new file mode 100644 index 000000000..d37a798da --- /dev/null +++ b/neutron/tests/functional/agent/linux/test_keepalived.py @@ -0,0 +1,60 @@ +# Copyright (c) 2014 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo.config import cfg + +from neutron.agent.common import config +from neutron.agent.linux import external_process +from neutron.agent.linux import keepalived +from neutron.openstack.common import log as logging +from neutron.tests.functional import base as functional_base +from neutron.tests.unit.agent.linux import test_keepalived + +LOG = logging.getLogger(__name__) + + +class KeepalivedManagerTestCase(functional_base.BaseSudoTestCase, + test_keepalived.KeepalivedConfBaseMixin): + def setUp(self): + super(KeepalivedManagerTestCase, self).setUp() + self.check_sudo_enabled() + self._configure() + + def _configure(self): + cfg.CONF.set_override('debug', True) + config.setup_logging(cfg.CONF) + config.register_root_helper(cfg.CONF) + cfg.CONF.set_override('root_helper', self.root_helper, group='AGENT') + + def test_keepalived_spawn(self): + expected_config = self._get_config() + manager = keepalived.KeepalivedManager('router1', expected_config, + conf_path=cfg.CONF.state_path, + root_helper=self.root_helper) + self.addCleanup(manager.disable) + + manager.spawn() + process = external_process.ProcessManager( + cfg.CONF, + 'router1', + self.root_helper, + namespace=None, + pids_path=cfg.CONF.state_path) + self.assertTrue(process.active) + + config_path = manager._get_full_config_file_path('keepalived.conf') + with open(config_path, 'r') as config_file: + config_contents = config_file.read() + self.assertEqual(expected_config.get_config_str(), config_contents) diff --git a/neutron/tests/unit/agent/linux/test_keepalived.py b/neutron/tests/unit/agent/linux/test_keepalived.py new file mode 100644 index 000000000..9bd6c4f2e --- /dev/null +++ b/neutron/tests/unit/agent/linux/test_keepalived.py @@ -0,0 +1,241 @@ +# Copyright (C) 2014 eNovance SAS +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from neutron.agent.linux import keepalived +from neutron.tests import base + +# Keepalived user guide: +# http://www.keepalived.org/pdf/UserGuide.pdf + + +class KeepalivedConfBaseMixin(object): + + def _get_config(self): + config = keepalived.KeepalivedConf() + + group1 = keepalived.KeepalivedGroup(1) + group2 = keepalived.KeepalivedGroup(2) + + group1.set_notify('master', '/tmp/script.sh') + + instance1 = keepalived.KeepalivedInstance('MASTER', 'eth0', 1, + advert_int=5) + instance1.set_authentication('AH', 'pass123') + instance1.track_interfaces.append("eth0") + + vip_address1 = keepalived.KeepalivedVipAddress('192.168.1.0/24', + 'eth1') + + vip_address2 = keepalived.KeepalivedVipAddress('192.168.2.0/24', + 'eth2') + + vip_address3 = keepalived.KeepalivedVipAddress('192.168.3.0/24', + 'eth2') + + vip_address_ex = keepalived.KeepalivedVipAddress('192.168.55.0/24', + 'eth10') + + instance1.vips.append(vip_address1) + instance1.vips.append(vip_address2) + instance1.vips.append(vip_address3) + instance1.vips.append(vip_address_ex) + + virtual_route = keepalived.KeepalivedVirtualRoute("0.0.0.0/0", + "192.168.1.1", + "eth1") + instance1.virtual_routes.append(virtual_route) + + group1.add_instance(instance1) + + instance2 = keepalived.KeepalivedInstance('MASTER', 'eth4', 2, + mcast_src_ip='224.0.0.1') + instance2.track_interfaces.append("eth4") + + vip_address1 = keepalived.KeepalivedVipAddress('192.168.3.0/24', + 'eth6') + + instance2.vips.append(vip_address1) + instance2.vips.append(vip_address2) + instance2.vips.append(vip_address_ex) + + group2.add_instance(instance2) + + config.add_group(group1) + config.add_instance(instance1) + config.add_group(group2) + config.add_instance(instance2) + + return config + + +class KeepalivedConfTestCase(base.BaseTestCase, + KeepalivedConfBaseMixin): + + expected = """vrrp_sync_group VG_1 { + group { + VR_1 + } + notify_master "/tmp/script.sh" +} +vrrp_sync_group VG_2 { + group { + VR_2 + } +} +vrrp_instance VR_1 { + state MASTER + interface eth0 + virtual_router_id 1 + priority 50 + advert_int 5 + authentication { + auth_type AH + auth_pass pass123 + } + track_interface { + eth0 + } + virtual_ipaddress { + 192.168.1.0/24 dev eth1 + } + virtual_ipaddress_excluded { + 192.168.2.0/24 dev eth2 + 192.168.3.0/24 dev eth2 + 192.168.55.0/24 dev eth10 + } + virtual_routes { + 0.0.0.0/0 via 192.168.1.1 dev eth1 + } +} +vrrp_instance VR_2 { + state MASTER + interface eth4 + virtual_router_id 2 + priority 50 + mcast_src_ip 224.0.0.1 + track_interface { + eth4 + } + virtual_ipaddress { + 192.168.2.0/24 dev eth2 + } + virtual_ipaddress_excluded { + 192.168.3.0/24 dev eth6 + 192.168.55.0/24 dev eth10 + } +}""" + + def test_config_generation(self): + config = self._get_config() + self.assertEqual(self.expected, config.get_config_str()) + + def test_config_with_reset(self): + config = self._get_config() + self.assertEqual(self.expected, config.get_config_str()) + + config.reset() + self.assertEqual('', config.get_config_str()) + + +class KeepalivedStateExceptionTestCase(base.BaseTestCase): + def test_state_exception(self): + group = keepalived.KeepalivedGroup('group2') + + invalid_notify_state = 'a seal walks' + self.assertRaises(keepalived.InvalidNotifyStateException, + group.set_notify, + invalid_notify_state, '/tmp/script.sh') + + invalid_vrrp_state = 'into a club' + self.assertRaises(keepalived.InvalidInstanceStateException, + keepalived.KeepalivedInstance, + invalid_vrrp_state, 'eth0', 33) + + invalid_auth_type = '[hip, hip]' + instance = keepalived.KeepalivedInstance('MASTER', 'eth0', 1) + self.assertRaises(keepalived.InvalidAuthenticationTypeExecption, + instance.set_authentication, + invalid_auth_type, 'some_password') + + +class KeepalivedInstanceTestCase(base.BaseTestCase, + KeepalivedConfBaseMixin): + def test_remove_adresses_by_interface(self): + config = self._get_config() + instance = config.get_instance(1) + instance.remove_vips_vroutes_by_interface('eth2') + instance.remove_vips_vroutes_by_interface('eth10') + + expected = """vrrp_sync_group VG_1 { + group { + VR_1 + } + notify_master "/tmp/script.sh" +} +vrrp_sync_group VG_2 { + group { + VR_2 + } +} +vrrp_instance VR_1 { + state MASTER + interface eth0 + virtual_router_id 1 + priority 50 + advert_int 5 + authentication { + auth_type AH + auth_pass pass123 + } + track_interface { + eth0 + } + virtual_ipaddress { + 192.168.1.0/24 dev eth1 + } + virtual_routes { + 0.0.0.0/0 via 192.168.1.1 dev eth1 + } +} +vrrp_instance VR_2 { + state MASTER + interface eth4 + virtual_router_id 2 + priority 50 + mcast_src_ip 224.0.0.1 + track_interface { + eth4 + } + virtual_ipaddress { + 192.168.2.0/24 dev eth2 + } + virtual_ipaddress_excluded { + 192.168.3.0/24 dev eth6 + 192.168.55.0/24 dev eth10 + } +}""" + + self.assertEqual(expected, config.get_config_str()) + + +class KeepalivedVirtualRouteTestCase(base.BaseTestCase): + def test_virtual_route_with_dev(self): + route = keepalived.KeepalivedVirtualRoute('0.0.0.0/0', '1.2.3.4', + 'eth0') + self.assertEqual('0.0.0.0/0 via 1.2.3.4 dev eth0', + route.build_config()) + + def test_virtual_route_without_dev(self): + route = keepalived.KeepalivedVirtualRoute('50.0.0.0/8', '1.2.3.4') + self.assertEqual('50.0.0.0/8 via 1.2.3.4', route.build_config()) -- 2.45.2