From 995c35221bd9d51a71022902a00a1d9e23449787 Mon Sep 17 00:00:00 2001 From: Moshe Levi Date: Fri, 7 Aug 2015 17:35:48 +0300 Subject: [PATCH] SR-IOV: Add Agent QoS driver to support bandwidth limit This patch adds SR-IOV agent driver which uses eswitch manager to set VF rate limit. It also updates the agent to call port_delete api of the extension manager to cleanup when port is deleted. Partially-Implements: blueprint ml2-sriov-qos-with-bwlimiting Change-Id: I364fc8158e502d4dcc3510d6157f12969961a11d --- doc/source/devref/quality_of_service.rst | 27 +++++- .../mech_sriov/agent/eswitch_manager.py | 14 +++ .../agent/extension_drivers/__init__.py | 0 .../agent/extension_drivers/qos_driver.py | 84 +++++++++++++++++ .../mech_sriov/agent/sriov_nic_agent.py | 30 +++++- .../mech_sriov/mech_driver/mech_driver.py | 3 + .../agent/extension_drivers/__init__.py | 0 .../extension_drivers/test_qos_driver.py | 92 +++++++++++++++++++ .../mech_sriov/agent/test_eswitch_manager.py | 20 ++++ .../mech_sriov/agent/test_sriov_nic_agent.py | 24 ++++- setup.cfg | 1 + 11 files changed, 287 insertions(+), 8 deletions(-) create mode 100755 neutron/plugins/ml2/drivers/mech_sriov/agent/extension_drivers/__init__.py create mode 100755 neutron/plugins/ml2/drivers/mech_sriov/agent/extension_drivers/qos_driver.py create mode 100755 neutron/tests/unit/plugins/ml2/drivers/mech_sriov/agent/extension_drivers/__init__.py create mode 100755 neutron/tests/unit/plugins/ml2/drivers/mech_sriov/agent/extension_drivers/test_qos_driver.py diff --git a/doc/source/devref/quality_of_service.rst b/doc/source/devref/quality_of_service.rst index 87b6999dc..0418aa2a3 100644 --- a/doc/source/devref/quality_of_service.rst +++ b/doc/source/devref/quality_of_service.rst @@ -253,8 +253,13 @@ with them. Agent backends -------------- -At the moment, QoS is supported by Open vSwitch backend only, so -QosOVSAgentDriver is the only driver that implements QosAgentDriver interface. +At the moment, QoS is supported by Open vSwitch and SR-IOV ml2 drivers. + +Each agent backend defines a QoS driver that implements the QosAgentDriver +interface: + +* Open vSwitch (QosOVSAgentDriver); +* SR-IOV (QosSRIOVAgentDriver). Open vSwitch @@ -277,6 +282,24 @@ More details about HTB in `the blog post `_. +SR-IOV +~~~~~~ + +SR-IOV bandwidth limit implementation relies on the new pci_lib function: + +* set_vf_max_rate + +As the name of the function suggests, the limit is applied on a Virtual +Function (VF). + +ip link interface has the following limitation for bandwidth limit: it uses +Mbps as units of bandwidth measurement, not kbps, and does not support float +numbers. So in case the limit is set to something less than 1000 kbps, it's set +to 1 Mbps only. If the limit is set to something that does not divide to 1000 +kbps chunks, then the effective limit is rounded to the nearest integer Mbps +value. + + Configuration ============= diff --git a/neutron/plugins/ml2/drivers/mech_sriov/agent/eswitch_manager.py b/neutron/plugins/ml2/drivers/mech_sriov/agent/eswitch_manager.py index 0bfb0e0f8..c42679437 100644 --- a/neutron/plugins/ml2/drivers/mech_sriov/agent/eswitch_manager.py +++ b/neutron/plugins/ml2/drivers/mech_sriov/agent/eswitch_manager.py @@ -330,3 +330,17 @@ class ESwitchManager(object): {"device_mac": device_mac, "pci_slot": pci_slot}) embedded_switch = None return embedded_switch + + def get_pci_slot_by_mac(self, device_mac): + """Get pci slot by mac. + + Get pci slot by device mac + @param device_mac: device mac + """ + result = None + for pci_slot, embedded_switch in self.pci_slot_map.items(): + used_device_mac = embedded_switch.get_pci_device(pci_slot) + if used_device_mac == device_mac: + result = pci_slot + break + return result diff --git a/neutron/plugins/ml2/drivers/mech_sriov/agent/extension_drivers/__init__.py b/neutron/plugins/ml2/drivers/mech_sriov/agent/extension_drivers/__init__.py new file mode 100755 index 000000000..e69de29bb diff --git a/neutron/plugins/ml2/drivers/mech_sriov/agent/extension_drivers/qos_driver.py b/neutron/plugins/ml2/drivers/mech_sriov/agent/extension_drivers/qos_driver.py new file mode 100755 index 000000000..8c30817a1 --- /dev/null +++ b/neutron/plugins/ml2/drivers/mech_sriov/agent/extension_drivers/qos_driver.py @@ -0,0 +1,84 @@ +# Copyright 2015 Mellanox Technologies, Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +from oslo_log import log as logging + +from neutron.agent.l2.extensions import qos +from neutron.i18n import _LE, _LI, _LW +from neutron.plugins.ml2.drivers.mech_sriov.agent.common import ( + exceptions as exc) +from neutron.plugins.ml2.drivers.mech_sriov.agent import eswitch_manager as esm +from neutron.plugins.ml2.drivers.mech_sriov.mech_driver import ( + mech_driver) + +LOG = logging.getLogger(__name__) + + +class QosSRIOVAgentDriver(qos.QosAgentDriver): + + _SUPPORTED_RULES = ( + mech_driver.SriovNicSwitchMechanismDriver.supported_qos_rule_types) + + def __init__(self): + super(QosSRIOVAgentDriver, self).__init__() + self.eswitch_mgr = None + + def initialize(self): + self.eswitch_mgr = esm.ESwitchManager() + + def create(self, port, qos_policy): + self._handle_rules('create', port, qos_policy) + + def update(self, port, qos_policy): + self._handle_rules('update', port, qos_policy) + + def delete(self, port, qos_policy): + # TODO(QoS): consider optimizing flushing of all QoS rules from the + # port by inspecting qos_policy.rules contents + self._delete_bandwidth_limit(port) + + def _handle_rules(self, action, port, qos_policy): + for rule in qos_policy.rules: + if rule.rule_type in self._SUPPORTED_RULES: + handler_name = ("".join(("_", action, "_", rule.rule_type))) + handler = getattr(self, handler_name) + handler(port, rule) + else: + LOG.warning(_LW('Unsupported QoS rule type for %(rule_id)s: ' + '%(rule_type)s; skipping'), + {'rule_id': rule.id, 'rule_type': rule.rule_type}) + + def _create_bandwidth_limit(self, port, rule): + self._update_bandwidth_limit(port, rule) + + def _update_bandwidth_limit(self, port, rule): + pci_slot = port['profile'].get('pci_slot') + device = port['device'] + self._set_vf_max_rate(device, pci_slot, rule.max_kbps) + + def _delete_bandwidth_limit(self, port): + pci_slot = port['profile'].get('pci_slot') + device = port['device'] + self._set_vf_max_rate(device, pci_slot) + + def _set_vf_max_rate(self, device, pci_slot, max_kbps=0): + if self.eswitch_mgr.device_exists(device, pci_slot): + try: + self.eswitch_mgr.set_device_max_rate( + device, pci_slot, max_kbps) + except exc.SriovNicError: + LOG.exception( + _LE("Failed to set device %s max rate"), device) + else: + LOG.info(_LI("No device with MAC %s defined on agent."), device) diff --git a/neutron/plugins/ml2/drivers/mech_sriov/agent/sriov_nic_agent.py b/neutron/plugins/ml2/drivers/mech_sriov/agent/sriov_nic_agent.py index 7bf297955..13210aa51 100644 --- a/neutron/plugins/ml2/drivers/mech_sriov/agent/sriov_nic_agent.py +++ b/neutron/plugins/ml2/drivers/mech_sriov/agent/sriov_nic_agent.py @@ -26,6 +26,7 @@ from oslo_log import log as logging import oslo_messaging from oslo_service import loopingcall +from neutron.agent.l2.extensions import manager as ext_manager from neutron.agent import rpc as agent_rpc from neutron.agent import securitygroups_rpc as sg_rpc from neutron.common import config as common_config @@ -34,7 +35,7 @@ from neutron.common import topics from neutron.common import utils as n_utils from neutron import context from neutron.i18n import _LE, _LI, _LW -from neutron.plugins.ml2.drivers.mech_sriov.agent.common import config # noqa +from neutron.plugins.ml2.drivers.mech_sriov.agent.common import config from neutron.plugins.ml2.drivers.mech_sriov.agent.common \ import exceptions as exc from neutron.plugins.ml2.drivers.mech_sriov.agent import eswitch_manager as esm @@ -72,12 +73,13 @@ class SriovNicSwitchAgent(object): polling_interval): self.polling_interval = polling_interval + self.conf = cfg.CONF self.setup_eswitch_mgr(physical_devices_mappings, exclude_devices) configurations = {'device_mappings': physical_devices_mappings} self.agent_state = { 'binary': 'neutron-sriov-nic-agent', - 'host': cfg.CONF.host, + 'host': self.conf.host, 'topic': n_constants.L2_AGENT_TOPIC, 'configurations': configurations, 'agent_type': n_constants.AGENT_TYPE_NIC_SWITCH, @@ -92,6 +94,10 @@ class SriovNicSwitchAgent(object): self.sg_agent = sg_rpc.SecurityGroupAgentRpc(self.context, self.sg_plugin_rpc) self._setup_rpc() + self.ext_manager = self._create_agent_extension_manager( + self.connection) + # The initialization is complete; we can start receiving messages + self.connection.consume_in_threads() # Initialize iteration counter self.iter_num = 0 @@ -111,7 +117,8 @@ class SriovNicSwitchAgent(object): [topics.SECURITY_GROUP, topics.UPDATE]] self.connection = agent_rpc.create_consumers(self.endpoints, self.topic, - consumers) + consumers, + start_listening=False) report_interval = cfg.CONF.AGENT.report_interval if report_interval: @@ -129,6 +136,12 @@ class SriovNicSwitchAgent(object): except Exception: LOG.exception(_LE("Failed reporting state!")) + def _create_agent_extension_manager(self, connection): + ext_manager.register_opts(self.conf) + mgr = ext_manager.AgentExtensionsManager(self.conf) + mgr.initialize(connection, 'sriov') + return mgr + def setup_eswitch_mgr(self, device_mappings, exclude_devices={}): self.eswitch_mgr = esm.ESwitchManager() self.eswitch_mgr.discover_devices(device_mappings, exclude_devices) @@ -225,6 +238,7 @@ class SriovNicSwitchAgent(object): profile.get('pci_slot'), device_details['admin_state_up'], spoofcheck) + self.ext_manager.handle_port(self.context, device_details) else: LOG.info(_LI("Device with MAC %s not defined on plugin"), device) @@ -235,6 +249,16 @@ class SriovNicSwitchAgent(object): for device in devices: LOG.info(_LI("Removing device with mac_address %s"), device) try: + pci_slot = self.eswitch_mgr.get_pci_slot_by_mac(device) + if pci_slot: + profile = {'pci_slot': pci_slot} + port = {'device': device, 'profile': profile} + self.ext_manager.delete_port(self.context, port) + else: + LOG.warning(_LW("Failed to find pci slot for device " + "%(device)s; skipping extension port " + "cleanup"), device) + dev_details = self.plugin_rpc.update_device_down(self.context, device, self.agent_id, diff --git a/neutron/plugins/ml2/drivers/mech_sriov/mech_driver/mech_driver.py b/neutron/plugins/ml2/drivers/mech_sriov/mech_driver/mech_driver.py index 50a95e226..dcb7e52d3 100644 --- a/neutron/plugins/ml2/drivers/mech_sriov/mech_driver/mech_driver.py +++ b/neutron/plugins/ml2/drivers/mech_sriov/mech_driver/mech_driver.py @@ -24,6 +24,7 @@ from neutron.plugins.common import constants as p_const from neutron.plugins.ml2 import driver_api as api from neutron.plugins.ml2.drivers.mech_sriov.mech_driver \ import exceptions as exc +from neutron.services.qos import qos_consts LOG = log.getLogger(__name__) @@ -61,6 +62,8 @@ class SriovNicSwitchMechanismDriver(api.MechanismDriver): """ + supported_qos_rule_types = [qos_consts.RULE_TYPE_BANDWIDTH_LIMIT] + def __init__(self, agent_type=constants.AGENT_TYPE_NIC_SWITCH, vif_type=portbindings.VIF_TYPE_HW_VEB, diff --git a/neutron/tests/unit/plugins/ml2/drivers/mech_sriov/agent/extension_drivers/__init__.py b/neutron/tests/unit/plugins/ml2/drivers/mech_sriov/agent/extension_drivers/__init__.py new file mode 100755 index 000000000..e69de29bb diff --git a/neutron/tests/unit/plugins/ml2/drivers/mech_sriov/agent/extension_drivers/test_qos_driver.py b/neutron/tests/unit/plugins/ml2/drivers/mech_sriov/agent/extension_drivers/test_qos_driver.py new file mode 100755 index 000000000..7ccb74507 --- /dev/null +++ b/neutron/tests/unit/plugins/ml2/drivers/mech_sriov/agent/extension_drivers/test_qos_driver.py @@ -0,0 +1,92 @@ +# Copyright 2015 Mellanox Technologies, Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or +# implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import mock +from oslo_utils import uuidutils + +from neutron import context +from neutron.objects.qos import policy +from neutron.objects.qos import rule +from neutron.plugins.ml2.drivers.mech_sriov.agent.common import exceptions +from neutron.plugins.ml2.drivers.mech_sriov.agent.extension_drivers import ( + qos_driver) +from neutron.tests import base + + +class QosSRIOVAgentDriverTestCase(base.BaseTestCase): + + ASSIGNED_MAC = '00:00:00:00:00:66' + PCI_SLOT = '0000:06:00.1' + + def setUp(self): + super(QosSRIOVAgentDriverTestCase, self).setUp() + self.context = context.get_admin_context() + self.qos_driver = qos_driver.QosSRIOVAgentDriver() + self.qos_driver.initialize() + self.qos_driver.eswitch_mgr = mock.Mock() + self.qos_driver.eswitch_mgr.set_device_max_rate = mock.Mock() + self.max_rate_mock = self.qos_driver.eswitch_mgr.set_device_max_rate + self.rule = self._create_bw_limit_rule_obj() + self.qos_policy = self._create_qos_policy_obj([self.rule]) + self.port = self._create_fake_port() + + def _create_bw_limit_rule_obj(self): + rule_obj = rule.QosBandwidthLimitRule() + rule_obj.id = uuidutils.generate_uuid() + rule_obj.max_kbps = 2 + rule_obj.max_burst_kbps = 200 + rule_obj.obj_reset_changes() + return rule_obj + + def _create_qos_policy_obj(self, rules): + policy_dict = {'id': uuidutils.generate_uuid(), + 'tenant_id': uuidutils.generate_uuid(), + 'name': 'test', + 'description': 'test', + 'shared': False, + 'rules': rules} + policy_obj = policy.QosPolicy(self.context, **policy_dict) + policy_obj.obj_reset_changes() + return policy_obj + + def _create_fake_port(self): + return {'port_id': uuidutils.generate_uuid(), + 'profile': {'pci_slot': self.PCI_SLOT}, + 'device': self.ASSIGNED_MAC} + + def test_create_rule(self): + self.qos_driver.create(self.port, self.qos_policy) + self.max_rate_mock.assert_called_once_with( + self.ASSIGNED_MAC, self.PCI_SLOT, self.rule.max_kbps) + + def test_update_rule(self): + self.qos_driver.update(self.port, self.qos_policy) + self.max_rate_mock.assert_called_once_with( + self.ASSIGNED_MAC, self.PCI_SLOT, self.rule.max_kbps) + + def test_delete_rules(self): + self.qos_driver.delete(self.port, self.qos_policy) + self.max_rate_mock.assert_called_once_with( + self.ASSIGNED_MAC, self.PCI_SLOT, 0) + + def test__set_vf_max_rate_captures_sriov_failure(self): + self.max_rate_mock.side_effect = exceptions.SriovNicError() + self.qos_driver._set_vf_max_rate(self.ASSIGNED_MAC, self.PCI_SLOT) + + def test__set_vf_max_rate_unknown_device(self): + with mock.patch.object(self.qos_driver.eswitch_mgr, 'device_exists', + return_value=False): + self.qos_driver._set_vf_max_rate(self.ASSIGNED_MAC, self.PCI_SLOT) + self.assertFalse(self.max_rate_mock.called) diff --git a/neutron/tests/unit/plugins/ml2/drivers/mech_sriov/agent/test_eswitch_manager.py b/neutron/tests/unit/plugins/ml2/drivers/mech_sriov/agent/test_eswitch_manager.py index e131dc1eb..a2b480c70 100644 --- a/neutron/tests/unit/plugins/ml2/drivers/mech_sriov/agent/test_eswitch_manager.py +++ b/neutron/tests/unit/plugins/ml2/drivers/mech_sriov/agent/test_eswitch_manager.py @@ -194,6 +194,26 @@ class TestESwitchManagerApi(base.BaseTestCase): 'device_mac': self.WRONG_MAC}) self.assertFalse(result) + @mock.patch("neutron.plugins.ml2.drivers.mech_sriov.agent.pci_lib." + "PciDeviceIPWrapper.get_assigned_macs", + return_value=[ASSIGNED_MAC]) + @mock.patch("neutron.plugins.ml2.drivers.mech_sriov.agent." + "eswitch_manager.PciOsWrapper.is_assigned_vf", + return_value=True) + def test_get_pci_slot_by_existing_mac(self, *args): + pci_slot = self.eswitch_mgr.get_pci_slot_by_mac(self.ASSIGNED_MAC) + self.assertIsNotNone(pci_slot) + + @mock.patch("neutron.plugins.ml2.drivers.mech_sriov.agent.pci_lib." + "PciDeviceIPWrapper.get_assigned_macs", + return_value=[ASSIGNED_MAC]) + @mock.patch("neutron.plugins.ml2.drivers.mech_sriov.agent." + "eswitch_manager.PciOsWrapper.is_assigned_vf", + return_value=True) + def test_get_pci_slot_by_not_existing_mac(self, *args): + pci_slot = self.eswitch_mgr.get_pci_slot_by_mac(self.WRONG_MAC) + self.assertIsNone(pci_slot) + class TestEmbSwitch(base.BaseTestCase): DEV_NAME = "eth2" diff --git a/neutron/tests/unit/plugins/ml2/drivers/mech_sriov/agent/test_sriov_nic_agent.py b/neutron/tests/unit/plugins/ml2/drivers/mech_sriov/agent/test_sriov_nic_agent.py index ccbb04435..8ebc73ce5 100644 --- a/neutron/tests/unit/plugins/ml2/drivers/mech_sriov/agent/test_sriov_nic_agent.py +++ b/neutron/tests/unit/plugins/ml2/drivers/mech_sriov/agent/test_sriov_nic_agent.py @@ -49,7 +49,13 @@ class TestSriovAgent(base.BaseTestCase): self.agent = sriov_nic_agent.SriovNicSwitchAgent({}, {}, 0) - def test_treat_devices_removed_with_existed_device(self): + @mock.patch("neutron.plugins.ml2.drivers.mech_sriov.agent.pci_lib." + "PciDeviceIPWrapper.get_assigned_macs", + return_value=[DEVICE_MAC]) + @mock.patch("neutron.plugins.ml2.drivers.mech_sriov.agent." + "eswitch_manager.PciOsWrapper.is_assigned_vf", + return_value=True) + def test_treat_devices_removed_with_existed_device(self, *args): agent = sriov_nic_agent.SriovNicSwitchAgent({}, {}, 0) devices = [DEVICE_MAC] with mock.patch.object(agent.plugin_rpc, @@ -63,7 +69,13 @@ class TestSriovAgent(base.BaseTestCase): self.assertFalse(resync) self.assertTrue(fn_udd.called) - def test_treat_devices_removed_with_not_existed_device(self): + @mock.patch("neutron.plugins.ml2.drivers.mech_sriov.agent.pci_lib." + "PciDeviceIPWrapper.get_assigned_macs", + return_value=[DEVICE_MAC]) + @mock.patch("neutron.plugins.ml2.drivers.mech_sriov.agent." + "eswitch_manager.PciOsWrapper.is_assigned_vf", + return_value=True) + def test_treat_devices_removed_with_not_existed_device(self, *args): agent = sriov_nic_agent.SriovNicSwitchAgent({}, {}, 0) devices = [DEVICE_MAC] with mock.patch.object(agent.plugin_rpc, @@ -77,7 +89,13 @@ class TestSriovAgent(base.BaseTestCase): self.assertFalse(resync) self.assertTrue(fn_udd.called) - def test_treat_devices_removed_failed(self): + @mock.patch("neutron.plugins.ml2.drivers.mech_sriov.agent.pci_lib." + "PciDeviceIPWrapper.get_assigned_macs", + return_value=[DEVICE_MAC]) + @mock.patch("neutron.plugins.ml2.drivers.mech_sriov.agent." + "eswitch_manager.PciOsWrapper.is_assigned_vf", + return_value=True) + def test_treat_devices_removed_failed(self, *args): agent = sriov_nic_agent.SriovNicSwitchAgent({}, {}, 0) devices = [DEVICE_MAC] with mock.patch.object(agent.plugin_rpc, diff --git a/setup.cfg b/setup.cfg index c9ff7b7c0..3ae7a7230 100644 --- a/setup.cfg +++ b/setup.cfg @@ -200,6 +200,7 @@ neutron.agent.l2.extensions = qos = neutron.agent.l2.extensions.qos:QosAgentExtension neutron.qos.agent_drivers = ovs = neutron.plugins.ml2.drivers.openvswitch.agent.extension_drivers.qos_driver:QosOVSAgentDriver + sriov = neutron.plugins.ml2.drivers.mech_sriov.agent.extension_drivers.qos_driver:QosSRIOVAgentDriver # These are for backwards compat with Icehouse notification_driver configuration values oslo.messaging.notify.drivers = neutron.openstack.common.notifier.log_notifier = oslo_messaging.notify._impl_log:LogDriver -- 2.45.2