python-eventlet/tests/tpool_test.py

   1 # Copyright (c) 2007, Linden Research, Inc.
   2 # Copyright (c) 2007, IBM Corp.
   3 #
   4 # Licensed under the Apache License, Version 2.0 (the "License");
   5 # you may not use this file except in compliance with the License.
   6 # You may obtain a copy of the License at
   7 #
   8 #   http://www.apache.org/licenses/LICENSE-2.0
   9 #
  10 # Unless required by applicable law or agreed to in writing, software
  11 # distributed under the License is distributed on an "AS IS" BASIS,
  12 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  13 # See the License for the specific language governing permissions and
  14 # limitations under the License.
  15 from __future__ import print_function
  16
  17 import gc
  18 import random
  19 import re
  20 import time
  21
  22 import eventlet
  23 from eventlet import tpool, debug, event
  24 from eventlet.support import six
  25 from tests import LimitedTestCase, skipped, skip_with_pyevent, main
  26
  27
  28 one = 1
  29 two = 2
  30 three = 3
  31 none = None
  32
  33
  34 def noop():
  35     pass
  36
  37
  38 def raise_exception():
  39     raise RuntimeError("hi")
  40
  41
  42 class TestTpool(LimitedTestCase):
  43     def setUp(self):
  44         super(TestTpool, self).setUp()
  45
  46     def tearDown(self):
  47         tpool.killall()
  48         super(TestTpool, self).tearDown()
  49
  50     @skip_with_pyevent
  51     def test_wrap_tuple(self):
  52         my_tuple = (1, 2)
  53         prox = tpool.Proxy(my_tuple)
  54         self.assertEqual(prox[0], 1)
  55         self.assertEqual(prox[1], 2)
  56         self.assertEqual(len(my_tuple), 2)
  57
  58     @skip_with_pyevent
  59     def test_wrap_string(self):
  60         my_object = "whatever"
  61         prox = tpool.Proxy(my_object)
  62         self.assertEqual(str(my_object), str(prox))
  63         self.assertEqual(len(my_object), len(prox))
  64         self.assertEqual(my_object.join(['a', 'b']), prox.join(['a', 'b']))
  65
  66     @skip_with_pyevent
  67     def test_wrap_uniterable(self):
  68         prox = tpool.Proxy([])
  69
  70         def index():
  71             prox[0]
  72
  73         def key():
  74             prox['a']
  75
  76         self.assertRaises(IndexError, index)
  77         self.assertRaises(TypeError, key)
  78
  79     @skip_with_pyevent
  80     def test_wrap_dict(self):
  81         my_object = {'a': 1}
  82         prox = tpool.Proxy(my_object)
  83         self.assertEqual('a', list(prox.keys())[0])
  84         self.assertEqual(1, prox['a'])
  85         self.assertEqual(str(my_object), str(prox))
  86         self.assertEqual(repr(my_object), repr(prox))
  87
  88     @skip_with_pyevent
  89     def test_wrap_module_class(self):
  90         prox = tpool.Proxy(re)
  91         self.assertEqual(tpool.Proxy, type(prox))
  92         exp = prox.compile('(.)(.)(.)')
  93         self.assertEqual(exp.groups, 3)
  94         assert repr(prox.compile)
  95
  96     @skip_with_pyevent
  97     def test_wrap_eq(self):
  98         prox = tpool.Proxy(re)
  99         exp1 = prox.compile('.')
 100         exp2 = prox.compile(exp1.pattern)
 101         self.assertEqual(exp1, exp2)
 102         exp3 = prox.compile('/')
 103         assert exp1 != exp3
 104
 105     @skip_with_pyevent
 106     def test_wrap_ints(self):
 107         p = tpool.Proxy(4)
 108         assert p == 4
 109
 110     @skip_with_pyevent
 111     def test_wrap_hash(self):
 112         prox1 = tpool.Proxy('' + 'A')
 113         prox2 = tpool.Proxy('A' + '')
 114         assert prox1 == 'A'
 115         assert 'A' == prox2
 116         # assert prox1 == prox2 FIXME - could __eq__ unwrap rhs if it is other proxy?
 117         self.assertEqual(hash(prox1), hash(prox2))
 118         proxList = tpool.Proxy([])
 119         self.assertRaises(TypeError, hash, proxList)
 120
 121     @skip_with_pyevent
 122     def test_wrap_nonzero(self):
 123         prox = tpool.Proxy(re)
 124         exp1 = prox.compile('.')
 125         assert bool(exp1)
 126         prox2 = tpool.Proxy([1, 2, 3])
 127         assert bool(prox2)
 128
 129     @skip_with_pyevent
 130     def test_multiple_wraps(self):
 131         prox1 = tpool.Proxy(re)
 132         prox2 = tpool.Proxy(re)
 133         prox1.compile('.')
 134         x2 = prox1.compile('.')
 135         del x2
 136         prox2.compile('.')
 137
 138     @skip_with_pyevent
 139     def test_wrap_getitem(self):
 140         prox = tpool.Proxy([0, 1, 2])
 141         self.assertEqual(prox[0], 0)
 142
 143     @skip_with_pyevent
 144     def test_wrap_setitem(self):
 145         prox = tpool.Proxy([0, 1, 2])
 146         prox[1] = 2
 147         self.assertEqual(prox[1], 2)
 148
 149     @skip_with_pyevent
 150     def test_wrap_iterator(self):
 151         self.reset_timeout(2)
 152         prox = tpool.Proxy(range(10))
 153         result = []
 154         for i in prox:
 155             result.append(i)
 156         self.assertEqual(list(range(10)), result)
 157
 158     @skip_with_pyevent
 159     def test_wrap_iterator2(self):
 160         self.reset_timeout(5)  # might take a while due to imprecise sleeping
 161
 162         def foo():
 163             import time
 164             for x in range(2):
 165                 yield x
 166                 time.sleep(0.001)
 167
 168         counter = [0]
 169
 170         def tick():
 171             for i in six.moves.range(20000):
 172                 counter[0] += 1
 173                 if counter[0] % 20 == 0:
 174                     eventlet.sleep(0.0001)
 175                 else:
 176                     eventlet.sleep()
 177
 178         gt = eventlet.spawn(tick)
 179         previtem = 0
 180         for item in tpool.Proxy(foo()):
 181             assert item >= previtem
 182         # make sure the tick happened at least a few times so that we know
 183         # that our iterations in foo() were actually tpooled
 184         assert counter[0] > 10, counter[0]
 185         gt.kill()
 186
 187     @skip_with_pyevent
 188     def test_raising_exceptions(self):
 189         prox = tpool.Proxy(re)
 190
 191         def nofunc():
 192             prox.never_name_a_function_like_this()
 193         self.assertRaises(AttributeError, nofunc)
 194
 195         from tests import tpool_test
 196         prox = tpool.Proxy(tpool_test)
 197         self.assertRaises(RuntimeError, prox.raise_exception)
 198
 199     @skip_with_pyevent
 200     def test_variable_and_keyword_arguments_with_function_calls(self):
 201         import optparse
 202         parser = tpool.Proxy(optparse.OptionParser())
 203         parser.add_option('-n', action='store', type='string', dest='n')
 204         opts, args = parser.parse_args(["-nfoo"])
 205         self.assertEqual(opts.n, 'foo')
 206
 207     @skip_with_pyevent
 208     def test_contention(self):
 209         from tests import tpool_test
 210         prox = tpool.Proxy(tpool_test)
 211
 212         pile = eventlet.GreenPile(4)
 213         pile.spawn(lambda: self.assertEqual(prox.one, 1))
 214         pile.spawn(lambda: self.assertEqual(prox.two, 2))
 215         pile.spawn(lambda: self.assertEqual(prox.three, 3))
 216         results = list(pile)
 217         self.assertEqual(len(results), 3)
 218
 219     @skip_with_pyevent
 220     def test_timeout(self):
 221         import time
 222         eventlet.Timeout(0.1, eventlet.TimeoutError())
 223         self.assertRaises(eventlet.TimeoutError,
 224                           tpool.execute, time.sleep, 0.3)
 225
 226     @skip_with_pyevent
 227     def test_killall(self):
 228         tpool.killall()
 229         tpool.setup()
 230
 231     @skip_with_pyevent
 232     def test_killall_remaining_results(self):
 233         semaphore = event.Event()
 234
 235         def native_fun():
 236             time.sleep(.5)
 237
 238         def gt_fun():
 239             semaphore.send(None)
 240             tpool.execute(native_fun)
 241
 242         gt = eventlet.spawn(gt_fun)
 243         semaphore.wait()
 244         tpool.killall()
 245         gt.wait()
 246
 247     @skip_with_pyevent
 248     def test_autowrap(self):
 249         x = tpool.Proxy({'a': 1, 'b': 2}, autowrap=(int,))
 250         assert isinstance(x.get('a'), tpool.Proxy)
 251         assert not isinstance(x.items(), tpool.Proxy)
 252         # attributes as well as callables
 253         from tests import tpool_test
 254         x = tpool.Proxy(tpool_test, autowrap=(int,))
 255         assert isinstance(x.one, tpool.Proxy)
 256         assert not isinstance(x.none, tpool.Proxy)
 257
 258     @skip_with_pyevent
 259     def test_autowrap_names(self):
 260         x = tpool.Proxy({'a': 1, 'b': 2}, autowrap_names=('get',))
 261         assert isinstance(x.get('a'), tpool.Proxy)
 262         assert not isinstance(x.items(), tpool.Proxy)
 263         from tests import tpool_test
 264         x = tpool.Proxy(tpool_test, autowrap_names=('one',))
 265         assert isinstance(x.one, tpool.Proxy)
 266         assert not isinstance(x.two, tpool.Proxy)
 267
 268     @skip_with_pyevent
 269     def test_autowrap_both(self):
 270         from tests import tpool_test
 271         x = tpool.Proxy(tpool_test, autowrap=(int,), autowrap_names=('one',))
 272         assert isinstance(x.one, tpool.Proxy)
 273         # violating the abstraction to check that we didn't double-wrap
 274         assert not isinstance(x._obj, tpool.Proxy)
 275
 276     @skip_with_pyevent
 277     def test_callable(self):
 278         def wrapped(arg):
 279             return arg
 280         x = tpool.Proxy(wrapped)
 281         self.assertEqual(4, x(4))
 282         # verify that it wraps return values if specified
 283         x = tpool.Proxy(wrapped, autowrap_names=('__call__',))
 284         assert isinstance(x(4), tpool.Proxy)
 285         self.assertEqual("4", str(x(4)))
 286
 287     @skip_with_pyevent
 288     def test_callable_iterator(self):
 289         def wrapped(arg):
 290             yield arg
 291             yield arg
 292             yield arg
 293
 294         x = tpool.Proxy(wrapped, autowrap_names=('__call__',))
 295         for r in x(3):
 296             self.assertEqual(3, r)
 297
 298     @skip_with_pyevent
 299     def test_eventlet_timeout(self):
 300         def raise_timeout():
 301             raise eventlet.Timeout()
 302         self.assertRaises(eventlet.Timeout, tpool.execute, raise_timeout)
 303
 304     @skip_with_pyevent
 305     def test_tpool_set_num_threads(self):
 306         tpool.set_num_threads(5)
 307         self.assertEqual(5, tpool._nthreads)
 308
 309
 310 class TpoolLongTests(LimitedTestCase):
 311     TEST_TIMEOUT = 60
 312
 313     @skip_with_pyevent
 314     def test_a_buncha_stuff(self):
 315         assert_ = self.assert_
 316
 317         class Dummy(object):
 318             def foo(self, when, token=None):
 319                 assert_(token is not None)
 320                 time.sleep(random.random() / 200.0)
 321                 return token
 322
 323         def sender_loop(loopnum):
 324             obj = tpool.Proxy(Dummy())
 325             count = 100
 326             for n in six.moves.range(count):
 327                 eventlet.sleep(random.random() / 200.0)
 328                 now = time.time()
 329                 token = loopnum * count + n
 330                 rv = obj.foo(now, token=token)
 331                 self.assertEqual(token, rv)
 332                 eventlet.sleep(random.random() / 200.0)
 333
 334         cnt = 10
 335         pile = eventlet.GreenPile(cnt)
 336         for i in six.moves.range(cnt):
 337             pile.spawn(sender_loop, i)
 338         results = list(pile)
 339         self.assertEqual(len(results), cnt)
 340         tpool.killall()
 341
 342     @skipped
 343     def test_benchmark(self):
 344         """ Benchmark computing the amount of overhead tpool adds to function calls."""
 345         iterations = 10000
 346         import timeit
 347         imports = """
 348 from tests.tpool_test import noop
 349 from eventlet.tpool import execute
 350         """
 351         t = timeit.Timer("noop()", imports)
 352         results = t.repeat(repeat=3, number=iterations)
 353         best_normal = min(results)
 354
 355         t = timeit.Timer("execute(noop)", imports)
 356         results = t.repeat(repeat=3, number=iterations)
 357         best_tpool = min(results)
 358
 359         tpool_overhead = (best_tpool - best_normal) / iterations
 360         print("%s iterations\nTpool overhead is %s seconds per call.  Normal: %s; Tpool: %s" % (
 361             iterations, tpool_overhead, best_normal, best_tpool))
 362         tpool.killall()
 363
 364     @skip_with_pyevent
 365     def test_leakage_from_tracebacks(self):
 366         tpool.execute(noop)  # get it started
 367         gc.collect()
 368         initial_objs = len(gc.get_objects())
 369         for i in range(10):
 370             self.assertRaises(RuntimeError, tpool.execute, raise_exception)
 371         gc.collect()
 372         middle_objs = len(gc.get_objects())
 373         # some objects will inevitably be created by the previous loop
 374         # now we test to ensure that running the loop an order of
 375         # magnitude more doesn't generate additional objects
 376         for i in six.moves.range(100):
 377             self.assertRaises(RuntimeError, tpool.execute, raise_exception)
 378         first_created = middle_objs - initial_objs
 379         gc.collect()
 380         second_created = len(gc.get_objects()) - middle_objs
 381         self.assert_(second_created - first_created < 10,
 382                      "first loop: %s, second loop: %s" % (first_created,
 383                                                           second_created))
 384         tpool.killall()
 385
 386
 387 if __name__ == '__main__':
 388     main()