From: Zhiteng Huang Date: Thu, 2 Jan 2014 01:31:24 +0000 (+0800) Subject: Pull latest scheduler change from Oslo X-Git-Url: https://review.fuel-infra.org/gitweb?a=commitdiff_plain;h=8ba08f0aa61d42d618f7772f83c7129c671bde65;p=openstack-build%2Fcinder-build.git Pull latest scheduler change from Oslo The RetryFilter is now part of Oslo, although it's been renamed to IgnoreAttemptedFilter. Thanks to entry_point, we are able to maintain backwards compatibility after pulling that commit from Oslo. Commits in this change: * 66fe978 2013-12-10 | Change IgnoreAttemptedHostFilter to expect 'retry' key (attempt-retry) * 135dd00 2013-12-10 | Remove start index 0 in range() * 45658e2 2013-12-09 | Fix violations of H302:import only modules * 70004c6 2013-12-04 | Add IgnoreAttemptedHostsFilter to oslo * 880acf7 2013-11-14 | Change capabilities filters to use resource type (capfilter_message) * 06e9d98 2013-11-10 | Add some log messages to capabilities_filter.py * 3970d46 2013-11-02 | Fix typos in oslo * 8718763 2013-08-19 | Replace list with dict in AvailabilityZoneFilter.host_passes * c0d052a 2013-07-12 | python3: Add basic compatibility support. * e3545f8 2013-06-02 | Enable hacking H402 test * 484a1df 2013-05-30 | Enable hacking H403 test * 35660da 2013-05-30 | Enable hacking H401 test * 5dcc43b 2013-05-05 | Break out common functionality for filters and weights * 1f2aba5 2013-05-03 | Renames filter to base_filter and weight to base_weight Change-Id: Ibeb685ef60e44cb6388fc460ee6a78255ed3dbae --- diff --git a/cinder/openstack/common/scheduler/filter.py b/cinder/openstack/common/scheduler/base_filter.py similarity index 64% rename from cinder/openstack/common/scheduler/filter.py rename to cinder/openstack/common/scheduler/base_filter.py index 52c18afa3..e31e7a270 100644 --- a/cinder/openstack/common/scheduler/filter.py +++ b/cinder/openstack/common/scheduler/base_filter.py @@ -17,9 +17,7 @@ Filter support """ -import inspect - -from stevedore import extension +from cinder.openstack.common.scheduler import base_handler class BaseFilter(object): @@ -33,7 +31,7 @@ class BaseFilter(object): def filter_all(self, filter_obj_list, filter_properties): """Yield objects that pass the filter. - Can be overriden in a subclass, if you need to base filtering + Can be overridden in a subclass, if you need to base filtering decisions on all objects. Otherwise, one can just override _filter_one() to filter a single object. """ @@ -42,27 +40,11 @@ class BaseFilter(object): yield obj -class BaseFilterHandler(object): - """ Base class to handle loading filter classes. +class BaseFilterHandler(base_handler.BaseHandler): + """Base class to handle loading filter classes. This class should be subclassed where one needs to use filters. """ - def __init__(self, filter_class_type, filter_namespace): - self.namespace = filter_namespace - self.filter_class_type = filter_class_type - self.filter_manager = extension.ExtensionManager(filter_namespace) - - def _is_correct_class(self, obj): - """Return whether an object is a class of the correct type and - is not prefixed with an underscore. - """ - return (inspect.isclass(obj) and - not obj.__name__.startswith('_') and - issubclass(obj, self.filter_class_type)) - - def get_all_classes(self): - return [x.plugin for x in self.filter_manager - if self._is_correct_class(x.plugin)] def get_filtered_objects(self, filter_classes, objs, filter_properties): diff --git a/cinder/openstack/common/scheduler/base_handler.py b/cinder/openstack/common/scheduler/base_handler.py new file mode 100644 index 000000000..1808d2c61 --- /dev/null +++ b/cinder/openstack/common/scheduler/base_handler.py @@ -0,0 +1,45 @@ +# Copyright (c) 2011-2013 OpenStack Foundation. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. +""" +A common base for handling extension classes. + +Used by BaseFilterHandler and BaseWeightHandler +""" + +import inspect + +from stevedore import extension + + +class BaseHandler(object): + """Base class to handle loading filter and weight classes.""" + def __init__(self, modifier_class_type, modifier_namespace): + self.namespace = modifier_namespace + self.modifier_class_type = modifier_class_type + self.extension_manager = extension.ExtensionManager(modifier_namespace) + + def _is_correct_class(self, cls): + """Return whether an object is a class of the correct type and + is not prefixed with an underscore. + """ + return (inspect.isclass(cls) and + not cls.__name__.startswith('_') and + issubclass(cls, self.modifier_class_type)) + + def get_all_classes(self): + # We use a set, as some classes may have an entrypoint of their own, + # and also be returned by a function such as 'all_filters' for example + return [ext.plugin for ext in self.extension_manager if + self._is_correct_class(ext.plugin)] diff --git a/cinder/openstack/common/scheduler/weight.py b/cinder/openstack/common/scheduler/base_weight.py similarity index 73% rename from cinder/openstack/common/scheduler/weight.py rename to cinder/openstack/common/scheduler/base_weight.py index 82f1d25ee..a6ba75d36 100644 --- a/cinder/openstack/common/scheduler/weight.py +++ b/cinder/openstack/common/scheduler/base_weight.py @@ -17,9 +17,7 @@ Pluggable Weighing support """ -import inspect - -from stevedore import extension +from cinder.openstack.common.scheduler import base_handler class WeighedObject(object): @@ -36,7 +34,7 @@ class BaseWeigher(object): """Base class for pluggable weighers.""" def _weight_multiplier(self): """How weighted this weigher should be. Normally this would - be overriden in a subclass based on a config value. + be overridden in a subclass based on a config value. """ return 1.0 @@ -56,26 +54,9 @@ class BaseWeigher(object): self._weigh_object(obj.obj, weight_properties)) -class BaseWeightHandler(object): +class BaseWeightHandler(base_handler.BaseHandler): object_class = WeighedObject - def __init__(self, weighed_object_type, weight_namespace): - self.namespace = weight_namespace - self.weighed_object_type = weighed_object_type - self.weight_manager = extension.ExtensionManager(weight_namespace) - - def _is_correct_class(self, obj): - """Return whether an object is a class of the correct type and - is not prefixed with an underscore. - """ - return (inspect.isclass(obj) and - not obj.__name__.startswith('_') and - issubclass(obj, self.weighed_object_type)) - - def get_all_classes(self): - return [x.plugin for x in self.weight_manager - if self._is_correct_class(x.plugin)] - def get_weighed_objects(self, weigher_classes, obj_list, weighing_properties): """Return a sorted (highest score first) list of WeighedObjects.""" diff --git a/cinder/openstack/common/scheduler/filters/__init__.py b/cinder/openstack/common/scheduler/filters/__init__.py index 40bf096a4..7c77d9cdf 100644 --- a/cinder/openstack/common/scheduler/filters/__init__.py +++ b/cinder/openstack/common/scheduler/filters/__init__.py @@ -18,12 +18,12 @@ Scheduler host filters """ from cinder.openstack.common import log as logging -from cinder.openstack.common.scheduler import filter +from cinder.openstack.common.scheduler import base_filter LOG = logging.getLogger(__name__) -class BaseHostFilter(filter.BaseFilter): +class BaseHostFilter(base_filter.BaseFilter): """Base class for host filters.""" def _filter_one(self, obj, filter_properties): """Return True if the object passes the filter, otherwise False.""" @@ -36,6 +36,6 @@ class BaseHostFilter(filter.BaseFilter): raise NotImplementedError() -class HostFilterHandler(filter.BaseFilterHandler): +class HostFilterHandler(base_filter.BaseFilterHandler): def __init__(self, namespace): super(HostFilterHandler, self).__init__(BaseHostFilter, namespace) diff --git a/cinder/openstack/common/scheduler/filters/availability_zone_filter.py b/cinder/openstack/common/scheduler/filters/availability_zone_filter.py index 0c3ca1ef7..1afc64c03 100644 --- a/cinder/openstack/common/scheduler/filters/availability_zone_filter.py +++ b/cinder/openstack/common/scheduler/filters/availability_zone_filter.py @@ -22,7 +22,7 @@ class AvailabilityZoneFilter(filters.BaseHostFilter): def host_passes(self, host_state, filter_properties): spec = filter_properties.get('request_spec', {}) - props = spec.get('resource_properties', []) + props = spec.get('resource_properties', {}) availability_zone = props.get('availability_zone') if availability_zone: diff --git a/cinder/openstack/common/scheduler/filters/capabilities_filter.py b/cinder/openstack/common/scheduler/filters/capabilities_filter.py index 41b638a2c..7e7953c14 100644 --- a/cinder/openstack/common/scheduler/filters/capabilities_filter.py +++ b/cinder/openstack/common/scheduler/filters/capabilities_filter.py @@ -13,11 +13,13 @@ # License for the specific language governing permissions and limitations # under the License. +import six + +from cinder.openstack.common.gettextutils import _ # noqa from cinder.openstack.common import log as logging from cinder.openstack.common.scheduler import filters from cinder.openstack.common.scheduler.filters import extra_specs_ops - LOG = logging.getLogger(__name__) @@ -25,13 +27,14 @@ class CapabilitiesFilter(filters.BaseHostFilter): """HostFilter to work with resource (instance & volume) type records.""" def _satisfies_extra_specs(self, capabilities, resource_type): - """Check that the capabilities provided by the services - satisfy the extra specs associated with the instance type""" + """Check that the capabilities provided by the services satisfy + the extra specs associated with the resource type. + """ extra_specs = resource_type.get('extra_specs', []) if not extra_specs: return True - for key, req in extra_specs.iteritems(): + for key, req in six.iteritems(extra_specs): # Either not scope format, or in capabilities scope scope = key.split(':') if len(scope) > 1 and scope[0] != "capabilities": @@ -40,7 +43,7 @@ class CapabilitiesFilter(filters.BaseHostFilter): del scope[0] cap = capabilities - for index in range(0, len(scope)): + for index in range(len(scope)): try: cap = cap.get(scope[index], None) except AttributeError: @@ -48,16 +51,20 @@ class CapabilitiesFilter(filters.BaseHostFilter): if cap is None: return False if not extra_specs_ops.match(cap, req): + LOG.debug(_("extra_spec requirement '%(req)s' does not match " + "'%(cap)s'"), {'req': req, 'cap': cap}) return False return True def host_passes(self, host_state, filter_properties): - """Return a list of hosts that can create instance_type.""" + """Return a list of hosts that can create resource_type.""" # Note(zhiteng) Currently only Cinder and Nova are using # this filter, so the resource type is either instance or # volume. resource_type = filter_properties.get('resource_type') if not self._satisfies_extra_specs(host_state.capabilities, resource_type): + LOG.debug(_("%(host_state)s fails resource_type extra_specs " + "requirements"), {'host_state': host_state}) return False return True diff --git a/cinder/scheduler/filters/retry_filter.py b/cinder/openstack/common/scheduler/filters/ignore_attempted_hosts_filter.py similarity index 55% rename from cinder/scheduler/filters/retry_filter.py rename to cinder/openstack/common/scheduler/filters/ignore_attempted_hosts_filter.py index cd660b771..d288ead20 100644 --- a/cinder/scheduler/filters/retry_filter.py +++ b/cinder/openstack/common/scheduler/filters/ignore_attempted_hosts_filter.py @@ -1,4 +1,4 @@ -# Copyright (c) 2012 OpenStack Foundation +# Copyright (c) 2011 OpenStack Foundation. # All Rights Reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); you may @@ -13,34 +13,44 @@ # License for the specific language governing permissions and limitations # under the License. +from cinder.openstack.common.gettextutils import _ # noqa from cinder.openstack.common import log as logging from cinder.openstack.common.scheduler import filters LOG = logging.getLogger(__name__) -class RetryFilter(filters.BaseHostFilter): - """Filter out nodes that have already been attempted for scheduling - purposes +class IgnoreAttemptedHostsFilter(filters.BaseHostFilter): + """Filter out previously attempted hosts + + A host passes this filter if it has not already been attempted for + scheduling. The scheduler needs to add previously attempted hosts + to the 'retry' key of filter_properties in order for this to work + correctly. For example: + { + 'retry': { + 'hosts': ['host1', 'host2'], + 'num_attempts': 3, + } + } """ def host_passes(self, host_state, filter_properties): """Skip nodes that have already been attempted.""" - retry = filter_properties.get('retry', None) - if not retry: + attempted = filter_properties.get('retry', None) + if not attempted: # Re-scheduling is disabled - LOG.debug("Re-scheduling is disabled") + LOG.debug(_("Re-scheduling is disabled.")) return True - hosts = retry.get('hosts', []) + hosts = attempted.get('hosts', []) host = host_state.host passes = host not in hosts pass_msg = "passes" if passes else "fails" LOG.debug(_("Host %(host)s %(pass_msg)s. Previously tried hosts: " - "%(hosts)s") % - {'host': host, 'pass_msg': pass_msg, 'hosts': hosts}) - - # Host passes if it's not in the list of previously attempted hosts: + "%(hosts)s") % {'host': host, + 'pass_msg': pass_msg, + 'hosts': hosts}) return passes diff --git a/cinder/openstack/common/scheduler/filters/json_filter.py b/cinder/openstack/common/scheduler/filters/json_filter.py index 370f23b2a..ce0a365ee 100644 --- a/cinder/openstack/common/scheduler/filters/json_filter.py +++ b/cinder/openstack/common/scheduler/filters/json_filter.py @@ -16,6 +16,8 @@ import operator +import six + from cinder.openstack.common import jsonutils from cinder.openstack.common.scheduler import filters @@ -51,7 +53,7 @@ class JsonFilter(filters.BaseHostFilter): return self._op_compare(args, operator.gt) def _in(self, args): - """First term is in set of remaining terms""" + """First term is in set of remaining terms.""" return self._op_compare(args, operator.contains) def _less_than_equal(self, args): @@ -117,7 +119,7 @@ class JsonFilter(filters.BaseHostFilter): for arg in query[1:]: if isinstance(arg, list): arg = self._process_filter(arg, host_state) - elif isinstance(arg, basestring): + elif isinstance(arg, six.string_types): arg = self._parse_string(arg, host_state) if arg is not None: cooked_args.append(arg) diff --git a/cinder/openstack/common/scheduler/weights/__init__.py b/cinder/openstack/common/scheduler/weights/__init__.py index a2743577d..f4f5f4202 100644 --- a/cinder/openstack/common/scheduler/weights/__init__.py +++ b/cinder/openstack/common/scheduler/weights/__init__.py @@ -18,10 +18,10 @@ Scheduler host weights """ -from cinder.openstack.common.scheduler import weight +from cinder.openstack.common.scheduler import base_weight -class WeighedHost(weight.WeighedObject): +class WeighedHost(base_weight.WeighedObject): def to_dict(self): return { 'weight': self.weight, @@ -33,12 +33,12 @@ class WeighedHost(weight.WeighedObject): (self.obj.host, self.weight)) -class BaseHostWeigher(weight.BaseWeigher): +class BaseHostWeigher(base_weight.BaseWeigher): """Base class for host weights.""" pass -class HostWeightHandler(weight.BaseWeightHandler): +class HostWeightHandler(base_weight.BaseWeightHandler): object_class = WeighedHost def __init__(self, namespace): diff --git a/cinder/tests/scheduler/test_host_filters.py b/cinder/tests/scheduler/test_host_filters.py index a6ab78ec1..231943aae 100644 --- a/cinder/tests/scheduler/test_host_filters.py +++ b/cinder/tests/scheduler/test_host_filters.py @@ -90,26 +90,3 @@ class HostFiltersTestCase(test.TestCase): 'updated_at': None, 'service': service}) self.assertTrue(filt_cls.host_passes(host, filter_properties)) - - def test_retry_filter_disabled(self): - # Test case where retry/re-scheduling is disabled. - filt_cls = self.class_map['RetryFilter']() - host = fakes.FakeHostState('host1', {}) - filter_properties = {} - self.assertTrue(filt_cls.host_passes(host, filter_properties)) - - def test_retry_filter_pass(self): - # Node not previously tried. - filt_cls = self.class_map['RetryFilter']() - host = fakes.FakeHostState('host1', {}) - retry = dict(num_attempts=2, hosts=['host2']) - filter_properties = dict(retry=retry) - self.assertTrue(filt_cls.host_passes(host, filter_properties)) - - def test_retry_filter_fail(self): - # Node was already tried. - filt_cls = self.class_map['RetryFilter']() - host = fakes.FakeHostState('host1', {}) - retry = dict(num_attempts=1, hosts=['host1']) - filter_properties = dict(retry=retry) - self.assertFalse(filt_cls.host_passes(host, filter_properties)) diff --git a/setup.cfg b/setup.cfg index 8b6deed4e..5ce712286 100644 --- a/setup.cfg +++ b/setup.cfg @@ -44,7 +44,7 @@ cinder.scheduler.filters = CapabilitiesFilter = cinder.openstack.common.scheduler.filters.capabilities_filter:CapabilitiesFilter CapacityFilter = cinder.scheduler.filters.capacity_filter:CapacityFilter JsonFilter = cinder.openstack.common.scheduler.filters.json_filter:JsonFilter - RetryFilter = cinder.scheduler.filters.retry_filter:RetryFilter + RetryFilter = cinder.openstack.common.scheduler.filters.ignore_attempted_hosts_filter:IgnoreAttemptedHostsFilter cinder.scheduler.weights = AllocatedCapacityWeigher = cinder.scheduler.weights.capacity:AllocatedCapacityWeigher CapacityWeigher = cinder.scheduler.weights.capacity:CapacityWeigher