The scheduler and scheduler.weights modules have not been
updated since early in the Icehouse release cycle. This
patch brings the version in Cinder up to date with what is in
oslo-incubator.
Current HEAD in OSLO:
---------------------
commit
20d7dc57819a70abdff967299542068946d75ac1
Date: Wed Jul 29 03:49:46 2015 +0000
Updated from global requirements
Change-Id: Iba0e4f9545fc9ba82f080a0fec672761dcfeaeec
Patches included with sync by file:
---------------------
base_filter.py
-
2fbf5065 - Remove oslo.log code and clean up versionutils API
-
5d40e143 - Remove code that moved to oslo.i18n
-
2af88ece - Use _LI instead of _ for info message translation
-
4a47188e - Allow filters to only run once per request if their data is static
-
fcf517d7 - Update oslo log messages with translation domains
-
cae33101 - Stop looping all filters if no objects return
base_handler.py
-
6fa29aee - Trivial: Make vertical white space after license header consistent
-
35660dac7 - Enable hacking H401 test
-
5dcc43b1 - Break out common functionality for filters and weights
base_weight.py
-
825cb870 - Upgrade to hacking 0.10
-
a2fa4878 - Fix common.scheduler.base_weight.BaseWeigher to be Python3 compat
-
e47bc70e - Normalize Scheduler Weights
Note: Key changes were made that impact cinder:
(1) The weigher base class changed the name of _weight_mulitipler() to
weight_multiplier() which impacted cinder/scheduler/weights/capacity.py
and cinder/scheduler/weights/volume_number.py.
(2) Application of the weight multiplier was moved from the weigher to the
handler and normalization of the weights prior to applying the weight
multiplier was added. This impacted cinder weigher test cases which
were checking for non-normalized weights.
(3) The normalization does not support the use of infinite weights which
are used by the cinder capacity weigher. When an infinite value is
used, it is not known what the other weights will be, so this adds an
override to CapacityWeigher for the weigh_objects() method that wraps
the parent method to use the returned complete set of weights to
replace any infinite weights with values that are much larger than
the largest non-infinite weight.
Change-Id: Iad9bb431f0876e4957598c0ac0d3c8d7f67199fc
"""
Filter support
"""
+import logging
+from cinder.openstack.common._i18n import _LI
from cinder.openstack.common.scheduler import base_handler
+LOG = logging.getLogger(__name__)
+
class BaseFilter(object):
"""Base class for all filter classes."""
if self._filter_one(obj, filter_properties):
yield obj
+ # Set to true in a subclass if a filter only needs to be run once
+ # for each request rather than for each instance
+ run_filter_once_per_request = False
+
+ def run_filter_for_index(self, index):
+ """Return True if the filter needs to be run for the "index-th"
+ instance in a request. Only need to override this if a filter
+ needs anything other than "first only" or "all" behaviour.
+ """
+ return not (self.run_filter_once_per_request and index > 0)
+
class BaseFilterHandler(base_handler.BaseHandler):
"""Base class to handle loading filter classes.
"""
def get_filtered_objects(self, filter_classes, objs,
- filter_properties):
+ filter_properties, index=0):
+ """Get objects after filter
+
+ :param filter_classes: filters that will be used to filter the
+ objects
+ :param objs: objects that will be filtered
+ :param filter_properties: client filter properties
+ :param index: This value needs to be increased in the caller
+ function of get_filtered_objects when handling
+ each resource.
+ """
+ list_objs = list(objs)
+ LOG.debug("Starting with %d host(s)", len(list_objs))
for filter_cls in filter_classes:
- objs = filter_cls().filter_all(objs, filter_properties)
- return list(objs)
+ cls_name = filter_cls.__name__
+ filter_class = filter_cls()
+
+ if filter_class.run_filter_for_index(index):
+ objs = filter_class.filter_all(list_objs, filter_properties)
+ if objs is None:
+ LOG.debug("Filter %(cls_name)s says to stop filtering",
+ {'cls_name': cls_name})
+ return
+ list_objs = list(objs)
+ msg = (_LI("Filter %(cls_name)s returned %(obj_len)d host(s)")
+ % {'cls_name': cls_name, 'obj_len': len(list_objs)})
+ if not list_objs:
+ LOG.info(msg)
+ break
+ LOG.debug(msg)
+ return list_objs
# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
# License for the specific language governing permissions and limitations
# under the License.
+
"""
A common base for handling extension classes.
Pluggable Weighing support
"""
+import abc
+
+import six
+
from cinder.openstack.common.scheduler import base_handler
+def normalize(weight_list, minval=None, maxval=None):
+ """Normalize the values in a list between 0 and 1.0.
+
+ The normalization is made regarding the lower and upper values present in
+ weight_list. If the minval and/or maxval parameters are set, these values
+ will be used instead of the minimum and maximum from the list.
+
+ If all the values are equal, they are normalized to 0.
+ """
+
+ if not weight_list:
+ return ()
+
+ if maxval is None:
+ maxval = max(weight_list)
+
+ if minval is None:
+ minval = min(weight_list)
+
+ maxval = float(maxval)
+ minval = float(minval)
+
+ if minval == maxval:
+ return [0] * len(weight_list)
+
+ range_ = maxval - minval
+ return ((i - minval) / range_ for i in weight_list)
+
+
class WeighedObject(object):
"""Object with weight information."""
def __init__(self, obj, weight):
return "<WeighedObject '%s': %s>" % (self.obj, self.weight)
+@six.add_metaclass(abc.ABCMeta)
class BaseWeigher(object):
- """Base class for pluggable weighers."""
- def _weight_multiplier(self):
- """How weighted this weigher should be. Normally this would
- be overridden in a subclass based on a config value.
+ """Base class for pluggable weighers.
+
+ The attributes maxval and minval can be specified to set up the maximum
+ and minimum values for the weighed objects. These values will then be
+ taken into account in the normalization step, instead of taking the values
+ from the calculated weights.
+ """
+
+ minval = None
+ maxval = None
+
+ def weight_multiplier(self):
+ """How weighted this weigher should be.
+
+ Override this method in a subclass, so that the returned value is
+ read from a configuration option to permit operators specify a
+ multiplier for the weigher.
"""
return 1.0
+ @abc.abstractmethod
def _weigh_object(self, obj, weight_properties):
"""Override in a subclass to specify a weight for a specific
object.
"""
- return 0.0
def weigh_objects(self, weighed_obj_list, weight_properties):
- """Weigh multiple objects. Override in a subclass if you need
- need access to all objects in order to manipulate weights.
+ """Weigh multiple objects.
+
+ Override in a subclass if you need access to all objects in order
+ to calculate weights. Do not modify the weight of an object here,
+ just return a list of weights.
"""
- constant = self._weight_multiplier()
+ # Calculate the weights
+ weights = []
for obj in weighed_obj_list:
- obj.weight += (constant *
- self._weigh_object(obj.obj, weight_properties))
+ weight = self._weigh_object(obj.obj, weight_properties)
+
+ # Record the min and max values if they are None. If they anything
+ # but none we assume that the weigher has set them
+ if self.minval is None:
+ self.minval = weight
+ if self.maxval is None:
+ self.maxval = weight
+
+ if weight < self.minval:
+ self.minval = weight
+ elif weight > self.maxval:
+ self.maxval = weight
+
+ weights.append(weight)
+
+ return weights
class BaseWeightHandler(base_handler.BaseHandler):
def get_weighed_objects(self, weigher_classes, obj_list,
weighing_properties):
- """Return a sorted (highest score first) list of WeighedObjects."""
+ """Return a sorted (descending), normalized list of WeighedObjects."""
if not obj_list:
return []
weighed_objs = [self.object_class(obj, 0.0) for obj in obj_list]
for weigher_cls in weigher_classes:
weigher = weigher_cls()
- weigher.weigh_objects(weighed_objs, weighing_properties)
+ weights = weigher.weigh_objects(weighed_objs, weighing_properties)
+
+ # Normalize the weights
+ weights = normalize(weights,
+ minval=weigher.minval,
+ maxval=weigher.maxval)
+
+ for i, weight in enumerate(weights):
+ obj = weighed_objs[i]
+ obj.weight += weigher.weight_multiplier() * weight
return sorted(weighed_objs, key=lambda x: x.weight, reverse=True)
CONF = cfg.CONF
CONF.register_opts(capacity_weight_opts)
+OFFSET_MIN = 10000
+OFFSET_MULT = 100
+
class CapacityWeigher(weights.BaseHostWeigher):
- def _weight_multiplier(self):
+ def weight_multiplier(self):
"""Override the weight multiplier."""
return CONF.capacity_weight_multiplier
+ def weigh_objects(self, weighed_obj_list, weight_properties):
+ """Override the weigh objects.
+
+
+ This override calls the parent to do the weigh objects and then
+ replaces any infinite weights with a value that is a multiple of the
+ delta between the min and max values.
+
+ NOTE(jecarey): the infinite weight value is only used when the
+ smallest value is being favored (negative multiplier). When the
+ largest weight value is being used a weight of -1 is used instead.
+ See _weigh_object method.
+ """
+ tmp_weights = super(weights.BaseHostWeigher, self).weigh_objects(
+ weighed_obj_list, weight_properties)
+
+ if math.isinf(self.maxval):
+ # NOTE(jecarey): if all weights were infinite then parent
+ # method returns 0 for all of the weights. Thus self.minval
+ # cannot be infinite at this point
+ copy_weights = [w for w in tmp_weights if not math.isinf(w)]
+ self.maxval = max(copy_weights)
+ offset = (self.maxval - self.minval) * OFFSET_MULT
+ self.maxval += OFFSET_MIN if offset == 0.0 else offset
+ tmp_weights = [self.maxval if math.isinf(w) else w
+ for w in tmp_weights]
+
+ return tmp_weights
+
def _weigh_object(self, host_state, weight_properties):
"""Higher weights win. We want spreading to be the default."""
reserved = float(host_state.reserved_percentage) / 100
class AllocatedCapacityWeigher(weights.BaseHostWeigher):
- def _weight_multiplier(self):
+ def weight_multiplier(self):
"""Override the weight multiplier."""
return CONF.allocated_capacity_weight_multiplier
class VolumeNumberWeigher(weights.BaseHostWeigher):
- def _weight_multiplier(self):
+ def weight_multiplier(self):
"""Override the weight multiplier."""
return CONF.volume_number_multiplier
def test_default_of_spreading_first(self):
hostinfo_list = self._get_all_hosts()
- # host1: allocated_capacity_gb=0, weight=0
+ # host1: allocated_capacity_gb=0, weight=0 Norm=0.0
# host2: allocated_capacity_gb=1748, weight=-1748
# host3: allocated_capacity_gb=256, weight=-256
- # host4: allocated_capacity_gb=1848, weight=-1848
+ # host4: allocated_capacity_gb=1848, weight=-1848 Norm=-1.0
+ # host5: allocated_capacity_gb=1548, weight=-1540
# so, host1 should win:
weighed_host = self._get_weighed_host(hostinfo_list)
- self.assertEqual(0, weighed_host.weight)
+ self.assertEqual(0.0, weighed_host.weight)
self.assertEqual(
'host1', utils.extract_host(weighed_host.obj.host))
self.flags(allocated_capacity_weight_multiplier=1.0)
hostinfo_list = self._get_all_hosts()
- # host1: allocated_capacity_gb=0, weight=0
+ # host1: allocated_capacity_gb=0, weight=0 Norm=0.0
# host2: allocated_capacity_gb=1748, weight=1748
# host3: allocated_capacity_gb=256, weight=256
- # host4: allocated_capacity_gb=1848, weight=1848
+ # host4: allocated_capacity_gb=1848, weight=1848 Norm=1.0
+ # host5: allocated_capacity_gb=1548, weight=1540
# so, host4 should win:
weighed_host = self._get_weighed_host(hostinfo_list)
- self.assertEqual(1848.0, weighed_host.weight)
+ self.assertEqual(1.0, weighed_host.weight)
self.assertEqual(
'host4', utils.extract_host(weighed_host.obj.host))
self.flags(allocated_capacity_weight_multiplier=-2.0)
hostinfo_list = self._get_all_hosts()
- # host1: allocated_capacity_gb=0, weight=0
+ # host1: allocated_capacity_gb=0, weight=0 Norm=0.0
# host2: allocated_capacity_gb=1748, weight=-3496
# host3: allocated_capacity_gb=256, weight=-512
- # host4: allocated_capacity_gb=1848, weight=-3696
+ # host4: allocated_capacity_gb=1848, weight=-3696 Norm=-2.0
+ # host5: allocated_capacity_gb=1548, weight=-3080
# so, host1 should win:
weighed_host = self._get_weighed_host(hostinfo_list)
- self.assertEqual(weighed_host.weight, 0)
+ self.assertEqual(0.0, weighed_host.weight)
self.assertEqual(
'host1', utils.extract_host(weighed_host.obj.host))
# host1: thin_provisioning_support = False
# free_capacity_gb=1024,
# free=1024-math.floor(1024*0.1)=922
+ # Norm=0.837837837838
# host2: thin_provisioning_support = True
# free_capacity_gb=300,
# free=2048*1.5-1748-math.floor(2048*0.1)=1120
+ # Norm=1.0
# host3: thin_provisioning_support = False
# free_capacity_gb=512, free=256-512*0=256
+ # Norm=0.292383292383
# host4: thin_provisioning_support = True
# free_capacity_gb=200,
# free=2048*1.0-2047-math.floor(2048*0.05)=-101
+ # Norm=0.0
# host5: free_capacity_gb=unknown free=-1
+ # Norm=0.0819000819001
# so, host2 should win:
weighed_host = self._get_weighed_host(hostinfo_list)
- self.assertEqual(weighed_host.weight, 1120.0)
+ self.assertEqual(weighed_host.weight, 1.0)
self.assertEqual(
utils.extract_host(weighed_host.obj.host), 'host2')
# host1: thin_provisioning_support = False
# free_capacity_gb=1024,
# free=-(1024-math.floor(1024*0.1))=-922
+ # Norm=-0.00829542413701
# host2: thin_provisioning_support = True
# free_capacity_gb=300,
# free=-(2048*1.5-1748-math.floor(2048*0.1))=-1120
+ # Norm=-0.00990099009901
# host3: thin_provisioning_support = False
# free_capacity_gb=512, free=-(256-512*0)=-256
+ # Norm=--0.002894884083
# host4: thin_provisioning_support = True
# free_capacity_gb=200,
# free=-(2048*1.0-2047-math.floor(2048*0.05))=101
+ # Norm=0.0
# host5: free_capacity_gb=unknown free=-float('inf')
+ # Norm=-1.0
# so, host4 should win:
weighed_host = self._get_weighed_host(hostinfo_list)
- self.assertEqual(weighed_host.weight, 101.0)
+ self.assertEqual(weighed_host.weight, 0.0)
self.assertEqual(
utils.extract_host(weighed_host.obj.host), 'host4')
# host1: thin_provisioning_support = False
# free_capacity_gb=1024,
# free=(1024-math.floor(1024*0.1))*2=1844
+ # Norm=1.67567567568
# host2: thin_provisioning_support = True
# free_capacity_gb=300,
# free=(2048*1.5-1748-math.floor(2048*0.1))*2=2240
+ # Norm=2.0
# host3: thin_provisioning_support = False
# free_capacity_gb=512, free=(256-512*0)*2=512
+ # Norm=0.584766584767
# host4: thin_provisioning_support = True
# free_capacity_gb=200,
# free=(2048*1.0-2047-math.floor(2048*0.05))*2=-202
+ # Norm=0.0
# host5: free_capacity_gb=unknown free=-2
+ # Norm=0.1638001638
# so, host2 should win:
weighed_host = self._get_weighed_host(hostinfo_list)
- self.assertEqual(weighed_host.weight, 1120.0 * 2)
+ self.assertEqual(weighed_host.weight, 1.0 * 2)
self.assertEqual(
utils.extract_host(weighed_host.obj.host), 'host2')
self.flags(volume_number_multiplier=-1.0)
hostinfo_list = self._get_all_hosts()
- # host1: 1 volume
+ # host1: 1 volume Norm=0.0
# host2: 2 volumes
# host3: 3 volumes
# host4: 4 volumes
- # host5: 5 volumes
+ # host5: 5 volumes Norm=-1.0
# so, host1 should win:
with mock.patch.object(api, 'volume_data_get_for_host',
fake_volume_data_get_for_host):
weighed_host = self._get_weighed_host(hostinfo_list)
- self.assertEqual(weighed_host.weight, -1.0)
+ self.assertEqual(weighed_host.weight, 0.0)
self.assertEqual(utils.extract_host(weighed_host.obj.host),
'host1')
self.flags(volume_number_multiplier=1.0)
hostinfo_list = self._get_all_hosts()
- # host1: 1 volume
+ # host1: 1 volume Norm=0
# host2: 2 volumes
# host3: 3 volumes
# host4: 4 volumes
- # host5: 5 volumes
+ # host5: 5 volumes Norm=1
# so, host5 should win:
with mock.patch.object(api, 'volume_data_get_for_host',
fake_volume_data_get_for_host):
weighed_host = self._get_weighed_host(hostinfo_list)
- self.assertEqual(weighed_host.weight, 5.0)
+ self.assertEqual(weighed_host.weight, 1.0)
self.assertEqual(utils.extract_host(weighed_host.obj.host),
'host5')
Code Review / openstack-build / cinder-build.git / commitdiff