From: Maru Newby Date: Sat, 26 Oct 2013 11:42:09 +0000 (+0000) Subject: Add post-mortem debug option for tests X-Git-Url: https://review.fuel-infra.org/gitweb?a=commitdiff_plain;h=f3461c76feb0940d1a56d64b1f0a5508151fe95f;p=openstack-build%2Fneutron-build.git Add post-mortem debug option for tests Post-mortem debugging, the ability to drop into a debugger with the execution state that triggered the exception, is very useful in diagnosing failure conditions. Our previous test runner, nose, provided the ability to enable post-mortem debugging on test failures (via --pdb-failure) and errors (via --pdb). testr lacks these options at present, so this change adds support for enabling post-mortem debugging via an environment variable. All test-triggered exceptions will result in a post-mortem debugger being invoked if OS_POST_MORTEM_DEBUG is set to "1" or "True". Implements: blueprint neutron-pm-debug-on-test-failure Change-Id: Iddbe1335b059d062c0286df2ad27aef7728461b7 --- diff --git a/TESTING b/TESTING index cd5ae4dab..735f3d1c8 100644 --- a/TESTING +++ b/TESTING @@ -62,3 +62,39 @@ Development process fixed! In addition, before proposing for merge, all of the current tests should be passing. +Debugging + + By default, calls to pdb.set_trace() will be ignored when tests + are run. For pdb statements to work, invoke run_tests as follows: + + $ ./run_tests.sh -d [test module path] + + It's possible to debug tests in a tox environment: + + $ tox -e venv -- python -m testtools.run [test module path] + + Tox-created virtual environments (venv's) can also be activated + after a tox run and reused for debugging: + + $ tox -e venv + $ . .tox/venv/bin/activate + $ python -m testtools.run [test module path] + + Tox packages and installs the neutron source tree in a given venv + on every invocation, but if modifications need to be made between + invocation (e.g. adding more pdb statements), it is recommended + that the source tree be installed in the venv in editable mode: + + # run this only after activating the venv + $ pip install --editable . + + Editable mode ensures that changes made to the source tree are + automatically reflected in the venv, and that such changes are not + overwritten during the next tox run. + +Post-mortem debugging + + Setting OS_POST_MORTEM_DEBUG=1 in the shell environment will ensure + that pdb.post_mortem() will be invoked on test failure: + + $ OS_POST_MORTEM_DEBUG=1 ./run_tests.sh -d [test module path] diff --git a/neutron/tests/base.py b/neutron/tests/base.py index 8426d1af8..82739ecf2 100644 --- a/neutron/tests/base.py +++ b/neutron/tests/base.py @@ -26,6 +26,8 @@ import fixtures from oslo.config import cfg import testtools +from neutron.tests import post_mortem_debug + CONF = cfg.CONF TRUE_STRING = ['True', '1'] @@ -41,6 +43,10 @@ class BaseTestCase(testtools.TestCase): def setUp(self): super(BaseTestCase, self).setUp() + # Configure this first to ensure pm debugging support for setUp() + if os.environ.get('OS_POST_MORTEM_DEBUG') in TRUE_STRING: + self.addOnException(post_mortem_debug.exception_handler) + if os.environ.get('OS_DEBUG') in TRUE_STRING: _level = logging.DEBUG else: diff --git a/neutron/tests/post_mortem_debug.py b/neutron/tests/post_mortem_debug.py new file mode 100644 index 000000000..1208505c3 --- /dev/null +++ b/neutron/tests/post_mortem_debug.py @@ -0,0 +1,106 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2013 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import pdb +import traceback + + +def exception_handler(exc_info): + """Exception handler enabling post-mortem debugging. + + A class extending testtools.TestCase can add this handler in setUp(): + + self.addOnException(post_mortem_debug.exception_handler) + + When an exception occurs, the user will be dropped into a pdb + session in the execution environment of the failure. + + Frames associated with the testing framework are excluded so that + the post-mortem session for an assertion failure will start at the + assertion call (e.g. self.assertTrue) rather than the framework code + that raises the failure exception (e.g. the assertTrue method). + """ + tb = exc_info[2] + ignored_traceback = get_ignored_traceback(tb) + if ignored_traceback: + tb = FilteredTraceback(tb, ignored_traceback) + traceback.print_exception(exc_info[0], exc_info[1], tb) + pdb.post_mortem(tb) + + +def get_ignored_traceback(tb): + """Retrieve the first traceback of an ignored trailing chain. + + Given an initial traceback, find the first traceback of a trailing + chain of tracebacks that should be ignored. The criteria for + whether a traceback should be ignored is whether its frame's + globals include the __unittest marker variable. This criteria is + culled from: + + unittest.TestResult._is_relevant_tb_level + + For example: + + tb.tb_next => tb0.tb_next => tb1.tb_next + + - If no tracebacks were to be ignored, None would be returned. + - If only tb1 was to be ignored, tb1 would be returned. + - If tb0 and tb1 were to be ignored, tb0 would be returned. + - If either of only tb or only tb0 was to be ignored, None would + be returned because neither tb or tb0 would be part of a + trailing chain of ignored tracebacks. + """ + # Turn the traceback chain into a list + tb_list = [] + while tb: + tb_list.append(tb) + tb = tb.tb_next + + # Find all members of an ignored trailing chain + ignored_tracebacks = [] + for tb in reversed(tb_list): + if '__unittest' in tb.tb_frame.f_globals: + ignored_tracebacks.append(tb) + else: + break + + # Return the first member of the ignored trailing chain + if ignored_tracebacks: + return ignored_tracebacks[-1] + + +class FilteredTraceback(object): + """Wraps a traceback to filter unwanted frames.""" + + def __init__(self, tb, filtered_traceback): + """Constructor. + + :param tb: The start of the traceback chain to filter. + :param filtered_traceback: The first traceback of a trailing + chain that is to be filtered. + """ + self._tb = tb + self.tb_lasti = self._tb.tb_lasti + self.tb_lineno = self._tb.tb_lineno + self.tb_frame = self._tb.tb_frame + self._filtered_traceback = filtered_traceback + + @property + def tb_next(self): + tb_next = self._tb.tb_next + if tb_next and tb_next != self._filtered_traceback: + return FilteredTraceback(tb_next, self._filtered_traceback) diff --git a/neutron/tests/unit/test_post_mortem_debug.py b/neutron/tests/unit/test_post_mortem_debug.py new file mode 100644 index 000000000..458170924 --- /dev/null +++ b/neutron/tests/unit/test_post_mortem_debug.py @@ -0,0 +1,98 @@ +# vim: tabstop=4 shiftwidth=4 softtabstop=4 + +# Copyright 2013 Red Hat, Inc. +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import sys + +import mock + +from neutron.tests import base +from neutron.tests import post_mortem_debug + + +class TestTesttoolsExceptionHandler(base.BaseTestCase): + + def test_exception_handler(self): + try: + self.assertTrue(False) + except Exception: + exc_info = sys.exc_info() + with mock.patch('traceback.print_exception') as mock_print_exception: + with mock.patch('pdb.post_mortem') as mock_post_mortem: + with mock.patch.object(post_mortem_debug, + 'get_ignored_traceback', + return_value=mock.Mock()): + post_mortem_debug.exception_handler(exc_info) + + mock_print_exception.called_once_with(*exc_info) + mock_post_mortem.called_once() + + +class TestFilteredTraceback(base.BaseTestCase): + + def test_filter_traceback(self): + tb1 = mock.Mock() + tb2 = mock.Mock() + tb1.tb_next = tb2 + tb2.tb_next = None + ftb1 = post_mortem_debug.FilteredTraceback(tb1, tb2) + for attr in ['lasti', 'lineno', 'frame']: + attr_name = 'tb_%s' % attr + self.assertEqual(getattr(tb1, attr_name, None), + getattr(ftb1, attr_name, None)) + self.assertIsNone(ftb1.tb_next) + + +class TestGetIgnoredTraceback(base.BaseTestCase): + + def _test_get_ignored_traceback(self, ignored_bit_array, expected): + root_tb = mock.Mock() + + tb = root_tb + tracebacks = [tb] + for x in xrange(len(ignored_bit_array) - 1): + tb.tb_next = mock.Mock() + tb = tb.tb_next + tracebacks.append(tb) + tb.tb_next = None + + tb = root_tb + for ignored in ignored_bit_array: + if ignored: + tb.tb_frame.f_globals = ['__unittest'] + else: + tb.tb_frame.f_globals = [] + tb = tb.tb_next + + actual = post_mortem_debug.get_ignored_traceback(root_tb) + if expected is not None: + expected = tracebacks[expected] + self.assertEqual(actual, expected) + + def test_no_ignored_tracebacks(self): + self._test_get_ignored_traceback([0, 0, 0], None) + + def test_single_member_trailing_chain(self): + self._test_get_ignored_traceback([0, 0, 1], 2) + + def test_two_member_trailing_chain(self): + self._test_get_ignored_traceback([0, 1, 1], 1) + + def test_first_traceback_ignored(self): + self._test_get_ignored_traceback([1, 0, 0], None) + + def test_middle_traceback_ignored(self): + self._test_get_ignored_traceback([0, 1, 0], None)