From: Ihar Hrachyshka Date: Thu, 23 Jul 2015 08:32:53 +0000 (+0200) Subject: Merge remote-tracking branch 'origin/feature/qos' into merge-branch X-Git-Url: https://review.fuel-infra.org/gitweb?a=commitdiff_plain;h=d3708de0cb1dacc610bc9585a55dd260a862104d;p=openstack-build%2Fneutron-build.git Merge remote-tracking branch 'origin/feature/qos' into merge-branch Also applied the following fixes: === 1. cleaned up some pylint failures that were not spotted before: Module neutron.objects.qos.policy: Metaclass class method __new__ should have 'mcs' as first argument Module neutron.objects.qos.rule: Lambda may not be necessary === 2. Revert "Introduce the AFTER_READ callback for ports and networks" This reverts commit e3dba1424114575581c153e02227282e036ad0a2. We don't use callbacks to extend resources anymore, instead relying on ml2 extension drivers. No need for the patch to achieve QoS, and it also breaks test_delete_subnet_with_callback that was added in master recently. === 3. updated requirements.txt and test-requirements.txt based on: https://review.openstack.org/#/c/204398/ to avoid requirements gate checks failing due to incompatible requirements comparing to global-requirements.txt Change-Id: I744ab2d8327a428a5467f2d07d073a5f8c333520 --- d3708de0cb1dacc610bc9585a55dd260a862104d diff --cc neutron/db/api.py index dec09bd35,c1619c51b..ba9c70a3a --- a/neutron/db/api.py +++ b/neutron/db/api.py @@@ -17,9 -17,9 +17,10 @@@ import contextli import six from oslo_config import cfg +from oslo_db import api as oslo_db_api from oslo_db import exception as os_db_exception from oslo_db.sqlalchemy import session + from oslo_utils import uuidutils from sqlalchemy import exc from sqlalchemy import orm diff --cc neutron/db/migration/alembic_migrations/versions/HEADS index 407b2c232,000000000..8a0a139d7 mode 100644,000000..100644 --- a/neutron/db/migration/alembic_migrations/versions/HEADS +++ b/neutron/db/migration/alembic_migrations/versions/HEADS @@@ -1,3 -1,0 +1,3 @@@ ++48153cb5f051 +4ffceebfada - 8675309a5c4f +kilo diff --cc neutron/db/migration/alembic_migrations/versions/liberty/expand/48153cb5f051_qos_db_changes.py index 000000000,d042ef83f..9a8fb102a mode 000000,100755..100755 --- a/neutron/db/migration/alembic_migrations/versions/liberty/expand/48153cb5f051_qos_db_changes.py +++ b/neutron/db/migration/alembic_migrations/versions/liberty/expand/48153cb5f051_qos_db_changes.py @@@ -1,0 -1,77 +1,77 @@@ + # Copyright 2015 Huawei Technologies India Pvt Ltd, Inc + # + # Licensed under the Apache License, Version 2.0 (the "License"); you may + # not use this file except in compliance with the License. You may obtain + # a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + # License for the specific language governing permissions and limitations + # under the License. + # + + """qos db changes + + Revision ID: 48153cb5f051 -Revises: 599c6a226151 ++Revises: 8675309a5c4f + Create Date: 2015-06-24 17:03:34.965101 + + """ + + # revision identifiers, used by Alembic. + revision = '48153cb5f051' -down_revision = '52c5312f6baf' ++down_revision = '8675309a5c4f' + + from alembic import op + import sqlalchemy as sa + + from neutron.api.v2 import attributes as attrs + + + def upgrade(): + op.create_table( + 'qos_policies', + sa.Column('id', sa.String(length=36), primary_key=True), + sa.Column('name', sa.String(length=attrs.NAME_MAX_LEN)), + sa.Column('description', sa.String(length=attrs.DESCRIPTION_MAX_LEN)), + sa.Column('shared', sa.Boolean()), + sa.Column('tenant_id', sa.String(length=attrs.TENANT_ID_MAX_LEN), + index=True)) + + op.create_table( + 'qos_network_policy_bindings', + sa.Column('policy_id', sa.String(length=36), + sa.ForeignKey('qos_policies.id', ondelete='CASCADE'), + nullable=False), + sa.Column('network_id', sa.String(length=36), + sa.ForeignKey('networks.id', ondelete='CASCADE'), + nullable=False, unique=True)) + + op.create_table( + 'qos_port_policy_bindings', + sa.Column('policy_id', sa.String(length=36), + sa.ForeignKey('qos_policies.id', ondelete='CASCADE'), + nullable=False), + sa.Column('port_id', sa.String(length=36), + sa.ForeignKey('ports.id', ondelete='CASCADE'), + nullable=False, unique=True)) + + op.create_table( + 'qos_rules', + sa.Column('id', sa.String(length=36), primary_key=True), + sa.Column('qos_policy_id', sa.String(length=36), + sa.ForeignKey('qos_policies.id', ondelete='CASCADE'), + nullable=False), + sa.Column('type', sa.String(length=255))) + + op.create_table( + 'qos_bandwidth_limit_rules', + sa.Column('id', sa.String(length=36), + sa.ForeignKey('qos_rules.id', ondelete='CASCADE'), + nullable=False, + primary_key=True), + sa.Column('max_kbps', sa.Integer()), + sa.Column('max_burst_kbps', sa.Integer())) diff --cc neutron/db/migration/models/head.py index 139f532be,78da3b1ef..057fc1b2f --- a/neutron/db/migration/models/head.py +++ b/neutron/db/migration/models/head.py @@@ -41,8 -39,8 +41,9 @@@ from neutron.db import model_bas from neutron.db import models_v2 # noqa from neutron.db import portbindings_db # noqa from neutron.db import portsecurity_db # noqa + from neutron.db.qos import models as qos_models # noqa from neutron.db import quota_db # noqa +from neutron.db import rbac_db_models # noqa from neutron.db import securitygroups_db # noqa from neutron.db import servicetype_db # noqa from neutron.ipam.drivers.neutrondb_ipam import db_models # noqa diff --cc neutron/objects/qos/policy.py index 000000000,09ba2b59b..3f0ba35ce mode 000000,100644..100644 --- a/neutron/objects/qos/policy.py +++ b/neutron/objects/qos/policy.py @@@ -1,0 -1,114 +1,114 @@@ + # Copyright 2015 Red Hat, Inc. + # All Rights Reserved. + # + # Licensed under the Apache License, Version 2.0 (the "License"); you may + # not use this file except in compliance with the License. You may obtain + # a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + # License for the specific language governing permissions and limitations + # under the License. + + import abc + + from oslo_versionedobjects import base as obj_base + from oslo_versionedobjects import fields as obj_fields + import six + + from neutron.common import exceptions + from neutron.common import utils + from neutron.db import api as db_api + from neutron.db.qos import api as qos_db_api + from neutron.db.qos import models as qos_db_model + from neutron.extensions import qos as qos_extension + from neutron.objects import base + from neutron.objects.qos import rule as rule_obj_impl + + + class QosRulesExtenderMeta(abc.ABCMeta): + - def __new__(cls, *args, **kwargs): - cls_ = super(QosRulesExtenderMeta, cls).__new__(cls, *args, **kwargs) ++ def __new__(mcs, name, bases, dct): ++ cls = super(QosRulesExtenderMeta, mcs).__new__(mcs, name, bases, dct) + - cls_.rule_fields = {} ++ cls.rule_fields = {} + for rule in qos_extension.VALID_RULE_TYPES: + rule_cls_name = 'Qos%sRule' % utils.camelize(rule) + field = '%s_rules' % rule - cls_.fields[field] = obj_fields.ListOfObjectsField(rule_cls_name) - cls_.rule_fields[field] = rule_cls_name ++ cls.fields[field] = obj_fields.ListOfObjectsField(rule_cls_name) ++ cls.rule_fields[field] = rule_cls_name + - cls_.synthetic_fields = list(cls_.rule_fields.keys()) ++ cls.synthetic_fields = list(cls.rule_fields.keys()) + - return cls_ ++ return cls + + + @obj_base.VersionedObjectRegistry.register + @six.add_metaclass(QosRulesExtenderMeta) + class QosPolicy(base.NeutronObject): + + db_model = qos_db_model.QosPolicy + + port_binding_model = qos_db_model.QosPortPolicyBinding + network_binding_model = qos_db_model.QosNetworkPolicyBinding + + fields = { + 'id': obj_fields.UUIDField(), + 'tenant_id': obj_fields.UUIDField(), + 'name': obj_fields.StringField(), + 'description': obj_fields.StringField(), + 'shared': obj_fields.BooleanField() + } + + fields_no_update = ['id', 'tenant_id'] + + def obj_load_attr(self, attrname): + if attrname not in self.rule_fields: + raise exceptions.ObjectActionError( + action='obj_load_attr', reason='unable to load %s' % attrname) + + rule_cls = getattr(rule_obj_impl, self.rule_fields[attrname]) + rules = rule_cls.get_rules_by_policy(self._context, self.id) + setattr(self, attrname, rules) + self.obj_reset_changes([attrname]) + + @classmethod + def _get_object_policy(cls, context, model, **kwargs): + # TODO(QoS): we should make sure we use public functions + binding_db_obj = db_api._find_object(context, model, **kwargs) + # TODO(QoS): rethink handling missing binding case + if binding_db_obj: + return cls.get_by_id(context, binding_db_obj['policy_id']) + + @classmethod + def get_network_policy(cls, context, network_id): + return cls._get_object_policy(context, cls.network_binding_model, + network_id=network_id) + + @classmethod + def get_port_policy(cls, context, port_id): + return cls._get_object_policy(context, cls.port_binding_model, + port_id=port_id) + + def attach_network(self, network_id): + qos_db_api.create_policy_network_binding(self._context, + policy_id=self.id, + network_id=network_id) + + def attach_port(self, port_id): + qos_db_api.create_policy_port_binding(self._context, + policy_id=self.id, + port_id=port_id) + + def detach_network(self, network_id): + qos_db_api.delete_policy_network_binding(self._context, + policy_id=self.id, + network_id=network_id) + + def detach_port(self, port_id): + qos_db_api.delete_policy_port_binding(self._context, + policy_id=self.id, + port_id=port_id) diff --cc neutron/objects/qos/rule.py index 000000000,b9aead64b..efe8c5335 mode 000000,100644..100644 --- a/neutron/objects/qos/rule.py +++ b/neutron/objects/qos/rule.py @@@ -1,0 -1,158 +1,157 @@@ + # Copyright 2015 Huawei Technologies India Pvt Ltd, Inc. + # All Rights Reserved. + # + # Licensed under the Apache License, Version 2.0 (the "License"); you may + # not use this file except in compliance with the License. You may obtain + # a copy of the License at + # + # http://www.apache.org/licenses/LICENSE-2.0 + # + # Unless required by applicable law or agreed to in writing, software + # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT + # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the + # License for the specific language governing permissions and limitations + # under the License. + + import abc + + from oslo_versionedobjects import base as obj_base + from oslo_versionedobjects import fields as obj_fields + import six + + from neutron.db import api as db_api + from neutron.db.qos import models as qos_db_model + from neutron.extensions import qos as qos_extension + from neutron.objects import base + + + @six.add_metaclass(abc.ABCMeta) + class QosRule(base.NeutronObject): + + base_db_model = qos_db_model.QosRule + + fields = { + 'id': obj_fields.UUIDField(), + 'type': obj_fields.StringField(), + 'qos_policy_id': obj_fields.UUIDField() + } + + fields_no_update = ['id', 'tenant_id', 'qos_policy_id'] + + # each rule subclass should redefine it + rule_type = None + + _core_fields = list(fields.keys()) + + _common_fields = ['id'] + + @classmethod + def _is_common_field(cls, field): + return field in cls._common_fields + + @classmethod + def _is_core_field(cls, field): + return field in cls._core_fields + + @classmethod + def _is_addn_field(cls, field): + return not cls._is_core_field(field) or cls._is_common_field(field) + + @staticmethod + def _filter_fields(fields, func): + return { + key: val for key, val in fields.items() + if func(key) + } + + def _get_changed_core_fields(self): + fields = self.obj_get_changes() + return self._filter_fields(fields, self._is_core_field) + + def _get_changed_addn_fields(self): + fields = self.obj_get_changes() - return self._filter_fields( - fields, lambda key: self._is_addn_field(key)) ++ return self._filter_fields(fields, self._is_addn_field) + + def _copy_common_fields(self, from_, to_): + for field in self._common_fields: + to_[field] = from_[field] + + @classmethod + def get_objects(cls, context, **kwargs): + # TODO(QoS): support searching for subtype fields + db_objs = db_api.get_objects(context, cls.base_db_model, **kwargs) + return [cls.get_by_id(context, db_obj['id']) for db_obj in db_objs] + + @classmethod + def get_by_id(cls, context, id): + obj = super(QosRule, cls).get_by_id(context, id) + + if obj: + # the object above does not contain fields from base QosRule yet, + # so fetch it and mix its fields into the object + base_db_obj = db_api.get_object(context, cls.base_db_model, id) + for field in cls._core_fields: + setattr(obj, field, base_db_obj[field]) + + obj.obj_reset_changes() + return obj + + # TODO(QoS): create and update are not transactional safe + def create(self): + + # TODO(QoS): enforce that type field value is bound to specific class + self.type = self.rule_type + + # create base qos_rule + core_fields = self._get_changed_core_fields() + base_db_obj = db_api.create_object( + self._context, self.base_db_model, core_fields) + + # create type specific qos_..._rule + addn_fields = self._get_changed_addn_fields() + self._copy_common_fields(core_fields, addn_fields) + addn_db_obj = db_api.create_object( + self._context, self.db_model, addn_fields) + + # merge two db objects into single neutron one + self.from_db_object(base_db_obj, addn_db_obj) + + def update(self): + updated_db_objs = [] + + # TODO(QoS): enforce that type field cannot be changed + + # update base qos_rule, if needed + core_fields = self._get_changed_core_fields() + if core_fields: + base_db_obj = db_api.update_object( + self._context, self.base_db_model, self.id, core_fields) + updated_db_objs.append(base_db_obj) + + addn_fields = self._get_changed_addn_fields() + if addn_fields: + addn_db_obj = db_api.update_object( + self._context, self.db_model, self.id, addn_fields) + updated_db_objs.append(addn_db_obj) + + # update neutron object with values from both database objects + self.from_db_object(*updated_db_objs) + + # delete is the same, additional rule object cleanup is done thru cascading + + @classmethod + def get_rules_by_policy(cls, context, policy_id): + return cls.get_objects(context, qos_policy_id=policy_id) + + + @obj_base.VersionedObjectRegistry.register + class QosBandwidthLimitRule(QosRule): + + db_model = qos_db_model.QosBandwidthLimitRule + + rule_type = qos_extension.RULE_TYPE_BANDWIDTH_LIMIT + + fields = { + 'max_kbps': obj_fields.IntegerField(nullable=True), + 'max_burst_kbps': obj_fields.IntegerField(nullable=True) + } diff --cc neutron/plugins/common/constants.py index edf52f593,b20e551c7..e5aa166d1 --- a/neutron/plugins/common/constants.py +++ b/neutron/plugins/common/constants.py @@@ -22,7 -22,7 +22,8 @@@ FIREWALL = "FIREWALL VPN = "VPN" METERING = "METERING" L3_ROUTER_NAT = "L3_ROUTER_NAT" +FLAVORS = "FLAVORS" + QOS = "QOS" # Maps extension alias to service type EXT_TO_SERVICE_MAPPING = { @@@ -33,7 -33,7 +34,8 @@@ 'vpnaas': VPN, 'metering': METERING, 'router': L3_ROUTER_NAT, - 'flavors': FLAVORS ++ 'flavors': FLAVORS, + 'qos': QOS, } # Service operation status constants diff --cc neutron/tests/unit/common/test_utils.py index 81634f979,1f5cfb2e4..20e764bfa --- a/neutron/tests/unit/common/test_utils.py +++ b/neutron/tests/unit/common/test_utils.py @@@ -667,15 -665,12 +667,26 @@@ class TestDelayedStringRenderer(base.Ba self.assertTrue(my_func.called) +class TestEnsureDir(base.BaseTestCase): + @mock.patch('os.makedirs') + def test_ensure_dir_no_fail_if_exists(self, makedirs): + error = OSError() + error.errno = errno.EEXIST + makedirs.side_effect = error + utils.ensure_dir("/etc/create/concurrently") + + @mock.patch('os.makedirs') + def test_ensure_dir_calls_makedirs(self, makedirs): + utils.ensure_dir("/etc/create/directory") + makedirs.assert_called_once_with("/etc/create/directory", 0o755) ++ ++ + class TestCamelize(base.BaseTestCase): + def test_camelize(self): + data = {'bandwidth_limit': 'BandwidthLimit', + 'test': 'Test', + 'some__more__dashes': 'SomeMoreDashes', + 'a_penguin_walks_into_a_bar': 'APenguinWalksIntoABar'} + + for s, expected in data.items(): + self.assertEqual(expected, utils.camelize(s)) diff --cc neutron/tests/unit/plugins/ml2/test_plugin.py index 7766e6cfb,8df72127b..aa9cc520d --- a/neutron/tests/unit/plugins/ml2/test_plugin.py +++ b/neutron/tests/unit/plugins/ml2/test_plugin.py @@@ -1630,3 -1587,82 +1630,75 @@@ class TestMl2PluginCreateUpdateDeletePo # run the transaction balancing function defined in this test plugin.delete_port(self.context, 'fake_id') self.assertTrue(self.notify.call_count) + + + class TestMl2PluginCreateUpdateNetwork(base.BaseTestCase): + def setUp(self): + super(TestMl2PluginCreateUpdateNetwork, self).setUp() + self.context = mock.MagicMock() + self.notify_p = mock.patch('neutron.callbacks.registry.notify') + self.notify = self.notify_p.start() + + def _ensure_transaction_is_closed(self): + transaction = self.context.session.begin(subtransactions=True) + enter = transaction.__enter__.call_count + exit = transaction.__exit__.call_count + self.assertEqual(enter, exit) + + def _create_plugin_for_create_update_network(self): + plugin = ml2_plugin.Ml2Plugin() + plugin.extension_manager = mock.Mock() + plugin.type_manager = mock.Mock() + plugin.mechanism_manager = mock.Mock() + plugin.notifier = mock.Mock() + mock.patch('neutron.extensions.providernet.' + '_raise_if_updates_provider_attributes').start() + - # Only check transaction is closed when not reading since we don't - # care much about reads in these tests. + self.notify.side_effect = ( - lambda r, e, t, **kwargs: None if e == events.AFTER_READ - else self._ensure_transaction_is_closed()) ++ lambda r, e, t, **kwargs: self._ensure_transaction_is_closed()) + + return plugin + + def test_create_network_rpc_outside_transaction(self): - # TODO(QoS): Figure out why it passes locally but fails in gate - self.skipTest("Gate is voodoo failing") + with mock.patch.object(ml2_plugin.Ml2Plugin, '__init__') as init,\ + mock.patch.object(base_plugin.NeutronDbPluginV2, + 'create_network'): + init.return_value = None + + plugin = self._create_plugin_for_create_update_network() + + plugin.create_network(self.context, mock.MagicMock()) + + kwargs = {'context': self.context, 'network': mock.ANY} + self.notify.assert_called_once_with('network', 'after_create', + plugin, **kwargs) + + def test_create_network_bulk_rpc_outside_transaction(self): - # TODO(QoS): Figure out why it passes locally but fails in gate - self.skipTest("Gate is voodoo failing") + with mock.patch.object(ml2_plugin.Ml2Plugin, '__init__') as init,\ + mock.patch.object(base_plugin.NeutronDbPluginV2, + 'create_network'): + init.return_value = None + + plugin = self._create_plugin_for_create_update_network() + + plugin.create_network_bulk(self.context, + {'networks': + [mock.MagicMock(), mock.MagicMock()]}) + + self.assertEqual(2, self.notify.call_count) + + def test_update_network_rpc_outside_transaction(self): + with mock.patch.object(ml2_plugin.Ml2Plugin, '__init__') as init,\ + mock.patch.object(base_plugin.NeutronDbPluginV2, + 'update_network'): + init.return_value = None + plugin = self._create_plugin_for_create_update_network() + + plugin.update_network(self.context, 'fake_id', mock.MagicMock()) + + kwargs = { + 'context': self.context, + 'network': mock.ANY, + } - self.notify.assert_called_with('network', 'after_update', ++ self.notify.assert_called_once_with('network', 'after_update', + plugin, **kwargs) diff --cc requirements.txt index 2d7c1ad36,2e7a8452f..c7577f0b5 --- a/requirements.txt +++ b/requirements.txt @@@ -26,16 -25,17 +26,17 @@@ stevedore>=1.5.0 # Apache-2. oslo.concurrency>=2.1.0 # Apache-2.0 oslo.config>=1.11.0 # Apache-2.0 oslo.context>=0.2.0 # Apache-2.0 -oslo.db>=1.10.0 # Apache-2.0 +oslo.db>=1.12.0 # Apache-2.0 oslo.i18n>=1.5.0 # Apache-2.0 -oslo.log>=1.2.0 # Apache-2.0 -oslo.messaging!=1.12.0,>=1.8.0 # Apache-2.0 -oslo.middleware!=2.0.0,>=1.2.0 # Apache-2.0 +oslo.log>=1.6.0 # Apache-2.0 - oslo.messaging>=1.16.0 # Apache-2.0 ++oslo.messaging!=1.17.0,!=1.17.1,>=1.16.0 # Apache-2.0 +oslo.middleware>=2.4.0 # Apache-2.0 oslo.policy>=0.5.0 # Apache-2.0 oslo.rootwrap>=2.0.0 # Apache-2.0 oslo.serialization>=1.4.0 # Apache-2.0 oslo.service>=0.1.0 # Apache-2.0 -oslo.utils>=1.6.0 # Apache-2.0 +oslo.utils>=1.9.0 # Apache-2.0 + oslo.versionedobjects>=0.3.0,!=0.5.0 python-novaclient>=2.22.0 diff --cc test-requirements.txt index b2c8bc9bb,6693ab22e..a39721dd3 --- a/test-requirements.txt +++ b/test-requirements.txt @@@ -6,8 -6,7 +6,7 @@@ hacking<0.11,>=0.10. cliff>=1.13.0 # Apache-2.0 coverage>=3.6 fixtures>=1.3.1 - mock!=1.1.4,>=1.1;python_version!='2.6' - mock==1.0.1;python_version=='2.6' -mock>=1.0 ++mock>=1.2 python-subunit>=0.0.18 requests-mock>=0.6.0 # Apache-2.0 sphinx!=1.2.0,!=1.3b1,<1.3,>=1.1.2