eventlet/tests/tpool_test.py

   1 # Copyright (c) 2007, Linden Research, Inc.
   2 # Copyright (c) 2007, IBM Corp.
   3 #
   4 # Licensed under the Apache License, Version 2.0 (the "License");
   5 # you may not use this file except in compliance with the License.
   6 # You may obtain a copy of the License at
   7 #
   8 #   http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10 # Unless required by applicable law or agreed to in writing, software
  11 # distributed under the License is distributed on an "AS IS" BASIS,
  12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 # See the License for the specific language governing permissions and
  14 # limitations under the License.
  15 from __future__ import print_function
  16
  17 import gc
  18 import random
  19 import re
  20 import time
  21
  22 import eventlet
  23 from eventlet import tpool
  24 from eventlet.support import six
  25 from tests import LimitedTestCase, skipped, skip_with_pyevent, main
  26
  27
  28 one = 1
  29 two = 2
  30 three = 3
  31 none = None
  32
  33
  34 def noop():
  35     pass
  36
  37
  38 def raise_exception():
  39     raise RuntimeError("hi")
  40
  41
  42 class TestTpool(LimitedTestCase):
  43     def setUp(self):
  44         super(TestTpool, self).setUp()
  45
  46     def tearDown(self):
  47         tpool.killall()
  48         super(TestTpool, self).tearDown()
  49
  50     @skip_with_pyevent
  51     def test_wrap_tuple(self):
  52         my_tuple = (1, 2)
  53         prox = tpool.Proxy(my_tuple)
  54         self.assertEqual(prox[0], 1)
  55         self.assertEqual(prox[1], 2)
  56         self.assertEqual(len(my_tuple), 2)
  57
  58     @skip_with_pyevent
  59     def test_wrap_string(self):
  60         my_object = "whatever"
  61         prox = tpool.Proxy(my_object)
  62         self.assertEqual(str(my_object), str(prox))
  63         self.assertEqual(len(my_object), len(prox))
  64         self.assertEqual(my_object.join(['a', 'b']), prox.join(['a', 'b']))
  65
  66     @skip_with_pyevent
  67     def test_wrap_uniterable(self):
  68         prox = tpool.Proxy([])
  69
  70         def index():
  71             prox[0]
  72
  73         def key():
  74             prox['a']
  75
  76         self.assertRaises(IndexError, index)
  77         self.assertRaises(TypeError, key)
  78
  79     @skip_with_pyevent
  80     def test_wrap_dict(self):
  81         my_object = {'a': 1}
  82         prox = tpool.Proxy(my_object)
  83         self.assertEqual('a', list(prox.keys())[0])
  84         self.assertEqual(1, prox['a'])
  85         self.assertEqual(str(my_object), str(prox))
  86         self.assertEqual(repr(my_object), repr(prox))
  87
  88     @skip_with_pyevent
  89     def test_wrap_module_class(self):
  90         prox = tpool.Proxy(re)
  91         self.assertEqual(tpool.Proxy, type(prox))
  92         exp = prox.compile('(.)(.)(.)')
  93         self.assertEqual(exp.groups, 3)
  94         assert repr(prox.compile)
  95
  96     @skip_with_pyevent
  97     def test_wrap_eq(self):
  98         prox = tpool.Proxy(re)
  99         exp1 = prox.compile('.')
 100         exp2 = prox.compile(exp1.pattern)
 101         self.assertEqual(exp1, exp2)
 102         exp3 = prox.compile('/')
 103         assert exp1 != exp3
 104
 105     @skip_with_pyevent
 106     def test_wrap_ints(self):
 107         p = tpool.Proxy(4)
 108         assert p == 4
 109
 110     @skip_with_pyevent
 111     def test_wrap_hash(self):
 112         prox1 = tpool.Proxy('' + 'A')
 113         prox2 = tpool.Proxy('A' + '')
 114         assert prox1 == 'A'
 115         assert 'A' == prox2
 116         # assert prox1 == prox2 FIXME - could __eq__ unwrap rhs if it is other proxy?
 117         self.assertEqual(hash(prox1), hash(prox2))
 118         proxList = tpool.Proxy([])
 119         self.assertRaises(TypeError, hash, proxList)
 120
 121     @skip_with_pyevent
 122     def test_wrap_nonzero(self):
 123         prox = tpool.Proxy(re)
 124         exp1 = prox.compile('.')
 125         assert bool(exp1)
 126         prox2 = tpool.Proxy([1, 2, 3])
 127         assert bool(prox2)
 128
 129     @skip_with_pyevent
 130     def test_multiple_wraps(self):
 131         prox1 = tpool.Proxy(re)
 132         prox2 = tpool.Proxy(re)
 133         prox1.compile('.')
 134         x2 = prox1.compile('.')
 135         del x2
 136         prox2.compile('.')
 137
 138     @skip_with_pyevent
 139     def test_wrap_getitem(self):
 140         prox = tpool.Proxy([0, 1, 2])
 141         self.assertEqual(prox[0], 0)
 142
 143     @skip_with_pyevent
 144     def test_wrap_setitem(self):
 145         prox = tpool.Proxy([0, 1, 2])
 146         prox[1] = 2
 147         self.assertEqual(prox[1], 2)
 148
 149     @skip_with_pyevent
 150     def test_wrap_iterator(self):
 151         self.reset_timeout(2)
 152         prox = tpool.Proxy(range(10))
 153         result = []
 154         for i in prox:
 155             result.append(i)
 156         self.assertEqual(list(range(10)), result)
 157
 158     @skip_with_pyevent
 159     def test_wrap_iterator2(self):
 160         self.reset_timeout(5)  # might take a while due to imprecise sleeping
 161
 162         def foo():
 163             import time
 164             for x in range(2):
 165                 yield x
 166                 time.sleep(0.001)
 167
 168         counter = [0]
 169
 170         def tick():
 171             for i in six.moves.range(20000):
 172                 counter[0] += 1
 173                 if counter[0] % 20 == 0:
 174                     eventlet.sleep(0.0001)
 175                 else:
 176                     eventlet.sleep()
 177
 178         gt = eventlet.spawn(tick)
 179         previtem = 0
 180         for item in tpool.Proxy(foo()):
 181             assert item >= previtem
 182         # make sure the tick happened at least a few times so that we know
 183         # that our iterations in foo() were actually tpooled
 184         assert counter[0] > 10, counter[0]
 185         gt.kill()
 186
 187     @skip_with_pyevent
 188     def test_raising_exceptions(self):
 189         prox = tpool.Proxy(re)
 190
 191         def nofunc():
 192             prox.never_name_a_function_like_this()
 193         self.assertRaises(AttributeError, nofunc)
 194
 195         from tests import tpool_test
 196         prox = tpool.Proxy(tpool_test)
 197         self.assertRaises(RuntimeError, prox.raise_exception)
 198
 199     @skip_with_pyevent
 200     def test_variable_and_keyword_arguments_with_function_calls(self):
 201         import optparse
 202         parser = tpool.Proxy(optparse.OptionParser())
 203         parser.add_option('-n', action='store', type='string', dest='n')
 204         opts, args = parser.parse_args(["-nfoo"])
 205         self.assertEqual(opts.n, 'foo')
 206
 207     @skip_with_pyevent
 208     def test_contention(self):
 209         from tests import tpool_test
 210         prox = tpool.Proxy(tpool_test)
 211
 212         pile = eventlet.GreenPile(4)
 213         pile.spawn(lambda: self.assertEqual(prox.one, 1))
 214         pile.spawn(lambda: self.assertEqual(prox.two, 2))
 215         pile.spawn(lambda: self.assertEqual(prox.three, 3))
 216         results = list(pile)
 217         self.assertEqual(len(results), 3)
 218
 219     @skip_with_pyevent
 220     def test_timeout(self):
 221         import time
 222         eventlet.Timeout(0.1, eventlet.TimeoutError())
 223         self.assertRaises(eventlet.TimeoutError,
 224                           tpool.execute, time.sleep, 0.3)
 225
 226     @skip_with_pyevent
 227     def test_killall(self):
 228         tpool.killall()
 229         tpool.setup()
 230
 231     @skip_with_pyevent
 232     def test_autowrap(self):
 233         x = tpool.Proxy({'a': 1, 'b': 2}, autowrap=(int,))
 234         assert isinstance(x.get('a'), tpool.Proxy)
 235         assert not isinstance(x.items(), tpool.Proxy)
 236         # attributes as well as callables
 237         from tests import tpool_test
 238         x = tpool.Proxy(tpool_test, autowrap=(int,))
 239         assert isinstance(x.one, tpool.Proxy)
 240         assert not isinstance(x.none, tpool.Proxy)
 241
 242     @skip_with_pyevent
 243     def test_autowrap_names(self):
 244         x = tpool.Proxy({'a': 1, 'b': 2}, autowrap_names=('get',))
 245         assert isinstance(x.get('a'), tpool.Proxy)
 246         assert not isinstance(x.items(), tpool.Proxy)
 247         from tests import tpool_test
 248         x = tpool.Proxy(tpool_test, autowrap_names=('one',))
 249         assert isinstance(x.one, tpool.Proxy)
 250         assert not isinstance(x.two, tpool.Proxy)
 251
 252     @skip_with_pyevent
 253     def test_autowrap_both(self):
 254         from tests import tpool_test
 255         x = tpool.Proxy(tpool_test, autowrap=(int,), autowrap_names=('one',))
 256         assert isinstance(x.one, tpool.Proxy)
 257         # violating the abstraction to check that we didn't double-wrap
 258         assert not isinstance(x._obj, tpool.Proxy)
 259
 260     @skip_with_pyevent
 261     def test_callable(self):
 262         def wrapped(arg):
 263             return arg
 264         x = tpool.Proxy(wrapped)
 265         self.assertEqual(4, x(4))
 266         # verify that it wraps return values if specified
 267         x = tpool.Proxy(wrapped, autowrap_names=('__call__',))
 268         assert isinstance(x(4), tpool.Proxy)
 269         self.assertEqual("4", str(x(4)))
 270
 271     @skip_with_pyevent
 272     def test_callable_iterator(self):
 273         def wrapped(arg):
 274             yield arg
 275             yield arg
 276             yield arg
 277
 278         x = tpool.Proxy(wrapped, autowrap_names=('__call__',))
 279         for r in x(3):
 280             self.assertEqual(3, r)
 281
 282     @skip_with_pyevent
 283     def test_eventlet_timeout(self):
 284         def raise_timeout():
 285             raise eventlet.Timeout()
 286         self.assertRaises(eventlet.Timeout, tpool.execute, raise_timeout)
 287
 288     @skip_with_pyevent
 289     def test_tpool_set_num_threads(self):
 290         tpool.set_num_threads(5)
 291         self.assertEqual(5, tpool._nthreads)
 292
 293
 294 class TpoolLongTests(LimitedTestCase):
 295     TEST_TIMEOUT = 60
 296
 297     @skip_with_pyevent
 298     def test_a_buncha_stuff(self):
 299         assert_ = self.assert_
 300
 301         class Dummy(object):
 302             def foo(self, when, token=None):
 303                 assert_(token is not None)
 304                 time.sleep(random.random() / 200.0)
 305                 return token
 306
 307         def sender_loop(loopnum):
 308             obj = tpool.Proxy(Dummy())
 309             count = 100
 310             for n in six.moves.range(count):
 311                 eventlet.sleep(random.random() / 200.0)
 312                 now = time.time()
 313                 token = loopnum * count + n
 314                 rv = obj.foo(now, token=token)
 315                 self.assertEqual(token, rv)
 316                 eventlet.sleep(random.random() / 200.0)
 317
 318         cnt = 10
 319         pile = eventlet.GreenPile(cnt)
 320         for i in six.moves.range(cnt):
 321             pile.spawn(sender_loop, i)
 322         results = list(pile)
 323         self.assertEqual(len(results), cnt)
 324         tpool.killall()
 325
 326     @skipped
 327     def test_benchmark(self):
 328         """ Benchmark computing the amount of overhead tpool adds to function calls."""
 329         iterations = 10000
 330         import timeit
 331         imports = """
 332 from tests.tpool_test import noop
 333 from eventlet.tpool import execute
 334         """
 335         t = timeit.Timer("noop()", imports)
 336         results = t.repeat(repeat=3, number=iterations)
 337         best_normal = min(results)
 338
 339         t = timeit.Timer("execute(noop)", imports)
 340         results = t.repeat(repeat=3, number=iterations)
 341         best_tpool = min(results)
 342
 343         tpool_overhead = (best_tpool - best_normal) / iterations
 344         print("%s iterations\nTpool overhead is %s seconds per call.  Normal: %s; Tpool: %s" % (
 345             iterations, tpool_overhead, best_normal, best_tpool))
 346         tpool.killall()
 347
 348     @skip_with_pyevent
 349     def test_leakage_from_tracebacks(self):
 350         tpool.execute(noop)  # get it started
 351         gc.collect()
 352         initial_objs = len(gc.get_objects())
 353         for i in range(10):
 354             self.assertRaises(RuntimeError, tpool.execute, raise_exception)
 355         gc.collect()
 356         middle_objs = len(gc.get_objects())
 357         # some objects will inevitably be created by the previous loop
 358         # now we test to ensure that running the loop an order of
 359         # magnitude more doesn't generate additional objects
 360         for i in six.moves.range(100):
 361             self.assertRaises(RuntimeError, tpool.execute, raise_exception)
 362         first_created = middle_objs - initial_objs
 363         gc.collect()
 364         second_created = len(gc.get_objects()) - middle_objs
 365         self.assert_(second_created - first_created < 10,
 366                      "first loop: %s, second loop: %s" % (first_created,
 367                                                           second_created))
 368         tpool.killall()
 369
 370
 371 if __name__ == '__main__':
 372     main()