From: Elena Ezhova Date: Fri, 25 Sep 2015 13:07:51 +0000 (+0300) Subject: Consume sslutils and wsgi modules from oslo.service X-Git-Url: https://review.fuel-infra.org/gitweb?a=commitdiff_plain;h=216d2d0b7594ad05916bcee6ed8f80f1e0b385d0;p=openstack-build%2Fneutron-build.git Consume sslutils and wsgi modules from oslo.service sslutils and basic WSGI functionality have been moved to oslo.service and now Neutron can reuse them. Marked ssl options that were renamed in oslo.service as deprecated. Added a note about possible implications for out-of-tree plugins to neutron_api.rst Bumped oslo.service version to 0.9.0. Related-Bug: #1482633 Depends-On: I0424a6c261fae447dbc25b3abf00258c860a88f5 Change-Id: Ibfdf07e665fcfcd093a0e31274e1a6116706aec2 --- diff --git a/doc/source/devref/neutron_api.rst b/doc/source/devref/neutron_api.rst index 40b5d8a91..8a6592c31 100644 --- a/doc/source/devref/neutron_api.rst +++ b/doc/source/devref/neutron_api.rst @@ -50,6 +50,13 @@ Neutron API is not very stable, and there are cases when a desired change in neutron tree is expected to trigger breakage for one or more external repositories under the neutron tent. Below you can find a list of known incompatible changes that could or are known to trigger those breakages. +The changes are listed in reverse chronological order (newer at the top). + +* change: Consume sslutils and wsgi modules from oslo.service. + + - commit: Ibfdf07e665fcfcd093a0e31274e1a6116706aec2 + - solution: switch using oslo_service.wsgi.Router; stop using neutron.wsgi.Router. + - severity: Low (some out-of-tree plugins might be affected). * change: oslo.service adopted. diff --git a/etc/neutron.conf b/etc/neutron.conf index dc8894860..e0d35c430 100644 --- a/etc/neutron.conf +++ b/etc/neutron.conf @@ -343,15 +343,21 @@ # use_ssl = False # Certificate file to use when starting API server securely +# This option is deprecated for removal in the N release, please +# use cert_file option from [ssl] section instead. # ssl_cert_file = /path/to/certfile # Private key file to use when starting API server securely +# This option is deprecated for removal in the N release, please +# use key_file option from [ssl] section instead. # ssl_key_file = /path/to/keyfile # CA certificate file to use when starting API server securely to # verify connecting clients. This is an optional parameter only required if # API clients need to authenticate to the API server using SSL certificates # signed by a trusted CA +# This option is deprecated for removal in the N release, please +# use ca_file option from [ssl] section instead. # ssl_ca_file = /path/to/cafile # ======== end of WSGI parameters related to the API server ========== @@ -1037,3 +1043,21 @@ lock_path = $state_path/lock [qos] # Drivers list to use to send the update notification # notification_drivers = message_queue + +[ssl] + +# +# From oslo.service.sslutils +# + +# CA certificate file to use to verify connecting clients. (string +# value) +#ca_file = + +# Certificate file to use when starting the server securely. (string +# value) +#cert_file = + +# Private key file to use when starting the server securely. (string +# value) +#key_file = diff --git a/neutron/agent/metadata/namespace_proxy.py b/neutron/agent/metadata/namespace_proxy.py index 8f6ee2e76..3aa62aff0 100644 --- a/neutron/agent/metadata/namespace_proxy.py +++ b/neutron/agent/metadata/namespace_proxy.py @@ -15,6 +15,7 @@ import httplib2 from oslo_config import cfg from oslo_log import log as logging +from oslo_service import wsgi as base_wsgi from oslo_utils import encodeutils import six import six.moves.urllib.parse as urlparse @@ -45,7 +46,7 @@ class NetworkMetadataProxyHandler(object): if network_id is None and router_id is None: raise exceptions.NetworkIdOrRouterIdRequiredError() - @webob.dec.wsgify(RequestClass=webob.Request) + @webob.dec.wsgify(RequestClass=base_wsgi.Request) def __call__(self, req): LOG.debug("Request: %s", req) try: diff --git a/neutron/api/v2/router.py b/neutron/api/v2/router.py index bd59d854b..c23679dfa 100644 --- a/neutron/api/v2/router.py +++ b/neutron/api/v2/router.py @@ -15,6 +15,7 @@ from oslo_config import cfg from oslo_log import log as logging +from oslo_service import wsgi as base_wsgi import routes as routes_mapper import six import six.moves.urllib.parse as urlparse @@ -66,7 +67,7 @@ class Index(wsgi.Application): return webob.Response(body=body, content_type=content_type) -class APIRouter(wsgi.Router): +class APIRouter(base_wsgi.Router): @classmethod def factory(cls, global_config, **local_config): diff --git a/neutron/common/config.py b/neutron/common/config.py index b6ba1a716..0c744b7cb 100644 --- a/neutron/common/config.py +++ b/neutron/common/config.py @@ -17,7 +17,6 @@ Routines for configuring Neutron """ -import os import sys from keystoneclient import auth @@ -26,7 +25,8 @@ from oslo_config import cfg from oslo_db import options as db_options from oslo_log import log as logging import oslo_messaging -from paste import deploy +from oslo_service import _options +from oslo_service import wsgi from neutron.api.v2 import attributes from neutron.common import utils @@ -42,8 +42,6 @@ core_opts = [ help=_("The host IP to bind to")), cfg.IntOpt('bind_port', default=9696, help=_("The port to bind to")), - cfg.StrOpt('api_paste_config', default="api-paste.ini", - help=_("The API paste config file to use")), cfg.StrOpt('api_extensions_path', default="", help=_("The path for API extensions")), cfg.StrOpt('auth_strategy', default='keystone', @@ -152,6 +150,9 @@ core_cli_opts = [ # Register the configuration options cfg.CONF.register_opts(core_opts) cfg.CONF.register_cli_opts(core_cli_opts) +# TODO(eezhova): Replace it with wsgi.register_opts(CONF) when oslo.service +# 0.10.0 releases. +cfg.CONF.register_opts(_options.wsgi_opts) # Ensure that the control exchange is set correctly oslo_messaging.set_transport_defaults(control_exchange='neutron') @@ -231,24 +232,7 @@ def load_paste_app(app_name): """Builds and returns a WSGI app from a paste config file. :param app_name: Name of the application to load - :raises ConfigFilesNotFoundError when config file cannot be located - :raises RuntimeError when application cannot be loaded from config file """ - - config_path = cfg.CONF.find_file(cfg.CONF.api_paste_config) - if not config_path: - raise cfg.ConfigFilesNotFoundError( - config_files=[cfg.CONF.api_paste_config]) - config_path = os.path.abspath(config_path) - LOG.info(_LI("Config paste file: %s"), config_path) - - try: - app = deploy.loadapp("config:%s" % config_path, name=app_name) - except (LookupError, ImportError): - msg = (_("Unable to load %(app_name)s from " - "configuration file %(config_path)s.") % - {'app_name': app_name, - 'config_path': config_path}) - LOG.exception(msg) - raise RuntimeError(msg) + loader = wsgi.Loader(cfg.CONF) + app = loader.load_app(app_name) return app diff --git a/neutron/tests/unit/api/test_extensions.py b/neutron/tests/unit/api/test_extensions.py index 69093f2bf..c8fea09c9 100644 --- a/neutron/tests/unit/api/test_extensions.py +++ b/neutron/tests/unit/api/test_extensions.py @@ -19,6 +19,7 @@ import mock from oslo_config import cfg from oslo_log import log as logging from oslo_serialization import jsonutils +from oslo_service import wsgi as base_wsgi import routes import six import webob @@ -48,7 +49,7 @@ _get_path = test_base._get_path extensions_path = ':'.join(neutron.tests.unit.extensions.__path__) -class ExtensionsTestApp(wsgi.Router): +class ExtensionsTestApp(base_wsgi.Router): def __init__(self, options={}): mapper = routes.Mapper() diff --git a/neutron/tests/unit/common/test_config.py b/neutron/tests/unit/common/test_config.py deleted file mode 100644 index 52521ac9d..000000000 --- a/neutron/tests/unit/common/test_config.py +++ /dev/null @@ -1,31 +0,0 @@ -# Copyright (c) 2012 OpenStack Foundation. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -# implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import mock -from oslo_config import cfg - -from neutron.common import config -from neutron.tests import base - - -class ConfigurationTest(base.BaseTestCase): - - def test_load_paste_app_not_found(self): - self.config(api_paste_config='no_such_file.conf') - with mock.patch.object(cfg.CONF, 'find_file', return_value=None) as ff: - e = self.assertRaises(cfg.ConfigFilesNotFoundError, - config.load_paste_app, 'app') - ff.assert_called_once_with('no_such_file.conf') - self.assertEqual(['no_such_file.conf'], e.config_files) diff --git a/neutron/tests/unit/test_wsgi.py b/neutron/tests/unit/test_wsgi.py index c7a403e2d..884a4e736 100644 --- a/neutron/tests/unit/test_wsgi.py +++ b/neutron/tests/unit/test_wsgi.py @@ -715,139 +715,3 @@ class FaultTest(base.BaseTestCase): "/", method='POST', headers={'Content-Type': "unknow"}) response = my_fault(request) self.assertEqual(415, response.status_int) - - -class TestWSGIServerWithSSL(base.BaseTestCase): - """WSGI server tests.""" - - def setUp(self): - super(TestWSGIServerWithSSL, self).setUp() - if six.PY3: - self.skip("bug/1482633") - - @mock.patch("exceptions.RuntimeError") - @mock.patch("os.path.exists") - def test__check_ssl_settings(self, exists_mock, runtime_error_mock): - exists_mock.return_value = True - CONF.set_default('use_ssl', True) - CONF.set_default("ssl_cert_file", 'certificate.crt') - CONF.set_default("ssl_key_file", 'privatekey.key') - CONF.set_default("ssl_ca_file", 'cacert.pem') - wsgi.Server("test_app") - self.assertFalse(runtime_error_mock.called) - - @mock.patch("os.path.exists") - def test__check_ssl_settings_no_ssl_cert_file_fails(self, exists_mock): - exists_mock.side_effect = [False] - CONF.set_default('use_ssl', True) - CONF.set_default("ssl_cert_file", "/no/such/file") - self.assertRaises(RuntimeError, wsgi.Server, "test_app") - - @mock.patch("os.path.exists") - def test__check_ssl_settings_no_ssl_key_file_fails(self, exists_mock): - exists_mock.side_effect = [True, False] - CONF.set_default('use_ssl', True) - CONF.set_default("ssl_cert_file", 'certificate.crt') - CONF.set_default("ssl_key_file", "/no/such/file") - self.assertRaises(RuntimeError, wsgi.Server, "test_app") - - @mock.patch("os.path.exists") - def test__check_ssl_settings_no_ssl_ca_file_fails(self, exists_mock): - exists_mock.side_effect = [True, True, False] - CONF.set_default('use_ssl', True) - CONF.set_default("ssl_cert_file", 'certificate.crt') - CONF.set_default("ssl_key_file", 'privatekey.key') - CONF.set_default("ssl_ca_file", "/no/such/file") - self.assertRaises(RuntimeError, wsgi.Server, "test_app") - - @mock.patch("ssl.wrap_socket") - @mock.patch("os.path.exists") - def _test_wrap_ssl(self, exists_mock, wrap_socket_mock, **kwargs): - exists_mock.return_value = True - sock = mock.Mock() - CONF.set_default("ssl_cert_file", 'certificate.crt') - CONF.set_default("ssl_key_file", 'privatekey.key') - ssl_kwargs = {'server_side': True, - 'certfile': CONF.ssl_cert_file, - 'keyfile': CONF.ssl_key_file, - 'cert_reqs': ssl.CERT_NONE, - } - if kwargs: - ssl_kwargs.update(**kwargs) - server = wsgi.Server("test_app") - server.wrap_ssl(sock) - wrap_socket_mock.assert_called_once_with(sock, **ssl_kwargs) - - def test_wrap_ssl(self): - self._test_wrap_ssl() - - def test_wrap_ssl_ca_file(self): - CONF.set_default("ssl_ca_file", 'cacert.pem') - ssl_kwargs = {'ca_certs': CONF.ssl_ca_file, - 'cert_reqs': ssl.CERT_REQUIRED - } - self._test_wrap_ssl(**ssl_kwargs) - - def test_app_using_ssl(self): - CONF.set_default('use_ssl', True) - CONF.set_default("ssl_cert_file", - os.path.join(TEST_VAR_DIR, 'certificate.crt')) - CONF.set_default("ssl_key_file", - os.path.join(TEST_VAR_DIR, 'privatekey.key')) - - greetings = 'Hello, World!!!' - - @webob.dec.wsgify - def hello_world(req): - return greetings - - server = wsgi.Server("test_app") - server.start(hello_world, 0, host="127.0.0.1") - - response = open_no_proxy('https://127.0.0.1:%d/' % server.port) - - self.assertEqual(greetings, response.read()) - - server.stop() - - def test_app_using_ssl_combined_cert_and_key(self): - CONF.set_default('use_ssl', True) - CONF.set_default("ssl_cert_file", - os.path.join(TEST_VAR_DIR, 'certandkey.pem')) - - greetings = 'Hello, World!!!' - - @webob.dec.wsgify - def hello_world(req): - return greetings - - server = wsgi.Server("test_app") - server.start(hello_world, 0, host="127.0.0.1") - - response = open_no_proxy('https://127.0.0.1:%d/' % server.port) - - self.assertEqual(greetings, response.read()) - - server.stop() - - def test_app_using_ipv6_and_ssl(self): - CONF.set_default('use_ssl', True) - CONF.set_default("ssl_cert_file", - os.path.join(TEST_VAR_DIR, 'certificate.crt')) - CONF.set_default("ssl_key_file", - os.path.join(TEST_VAR_DIR, 'privatekey.key')) - - greetings = 'Hello, World!!!' - - @webob.dec.wsgify - def hello_world(req): - return greetings - - server = wsgi.Server("test_app") - server.start(hello_world, 0, host="::1") - - response = open_no_proxy('https://[::1]:%d/' % server.port) - - self.assertEqual(greetings, response.read()) - - server.stop() diff --git a/neutron/wsgi.py b/neutron/wsgi.py index dacbadf8e..db46ac669 100644 --- a/neutron/wsgi.py +++ b/neutron/wsgi.py @@ -19,9 +19,7 @@ Utility methods for working with WSGI servers from __future__ import print_function import errno -import os import socket -import ssl import sys import time @@ -31,10 +29,12 @@ import oslo_i18n from oslo_log import log as logging from oslo_log import loggers from oslo_serialization import jsonutils +from oslo_service import _options from oslo_service import service as common_service +from oslo_service import sslutils from oslo_service import systemd +from oslo_service import wsgi from oslo_utils import excutils -import routes.middleware import six import webob.dec import webob.exc @@ -51,44 +51,19 @@ socket_opts = [ default=4096, help=_("Number of backlog requests to configure " "the socket with")), - cfg.IntOpt('tcp_keepidle', - default=600, - help=_("Sets the value of TCP_KEEPIDLE in seconds for each " - "server socket. Not supported on OS X.")), cfg.IntOpt('retry_until_window', default=30, help=_("Number of seconds to keep retrying to listen")), - cfg.IntOpt('max_header_line', - default=16384, - help=_("Max header line to accommodate large tokens")), cfg.BoolOpt('use_ssl', default=False, help=_('Enable SSL on the API server')), - cfg.StrOpt('ssl_ca_file', - help=_("CA certificate file to use to verify " - "connecting clients")), - cfg.StrOpt('ssl_cert_file', - help=_("Certificate file to use when starting " - "the server securely")), - cfg.StrOpt('ssl_key_file', - help=_("Private key file to use when starting " - "the server securely")), - cfg.BoolOpt('wsgi_keep_alive', - default=True, - help=_("Determines if connections are allowed to be held " - "open by clients after a request is fulfilled. A value " - "of False will ensure that the socket connection will " - "be explicitly closed once a response has been sent to " - "the client.")), - cfg.IntOpt('client_socket_timeout', default=900, - help=_("Timeout for client connections socket operations. " - "If an incoming connection is idle for this number of " - "seconds it will be closed. A value of '0' means " - "wait forever.")), ] CONF = cfg.CONF CONF.register_opts(socket_opts) +# TODO(eezhova): Replace it with wsgi.register_opts(CONF) when oslo.service +# 0.10.0 releases. +CONF.register_opts(_options.wsgi_opts) LOG = logging.getLogger(__name__) @@ -119,7 +94,7 @@ class WorkerService(worker.NeutronWorker): # Duplicate a socket object to keep a file descriptor usable. dup_sock = self._service._socket.dup() if CONF.use_ssl: - dup_sock = self._service.wrap_ssl(dup_sock) + dup_sock = sslutils.wrap(CONF, dup_sock) self._server = self._service.pool.spawn(self._service._run, self._application, dup_sock) @@ -153,7 +128,7 @@ class Server(object): # wsgi server to wait forever. self.client_socket_timeout = CONF.client_socket_timeout or None if CONF.use_ssl: - self._check_ssl_settings() + sslutils.is_enabled(CONF) def _get_socket(self, host, port, backlog): bind_addr = (host, port) @@ -202,37 +177,6 @@ class Server(object): return sock - @staticmethod - def _check_ssl_settings(): - if not os.path.exists(CONF.ssl_cert_file): - raise RuntimeError(_("Unable to find ssl_cert_file " - ": %s") % CONF.ssl_cert_file) - - # ssl_key_file is optional because the key may be embedded in the - # certificate file - if CONF.ssl_key_file and not os.path.exists(CONF.ssl_key_file): - raise RuntimeError(_("Unable to find " - "ssl_key_file : %s") % CONF.ssl_key_file) - - # ssl_ca_file is optional - if CONF.ssl_ca_file and not os.path.exists(CONF.ssl_ca_file): - raise RuntimeError(_("Unable to find ssl_ca_file " - ": %s") % CONF.ssl_ca_file) - - @staticmethod - def wrap_ssl(sock): - ssl_kwargs = {'server_side': True, - 'certfile': CONF.ssl_cert_file, - 'keyfile': CONF.ssl_key_file, - 'cert_reqs': ssl.CERT_NONE, - } - - if CONF.ssl_ca_file: - ssl_kwargs['ca_certs'] = CONF.ssl_ca_file - ssl_kwargs['cert_reqs'] = ssl.CERT_REQUIRED - - return ssl.wrap_socket(sock, **ssl_kwargs) - def start(self, application, port, host='0.0.0.0', workers=0): """Run a WSGI server with the given application.""" self._host = host @@ -354,7 +298,7 @@ class Middleware(object): return self.process_response(response) -class Request(webob.Request): +class Request(wsgi.Request): def best_match_content_type(self): """Determine the most acceptable content-type. @@ -711,63 +655,6 @@ class Debug(Middleware): print() -class Router(object): - """WSGI middleware that maps incoming requests to WSGI apps.""" - - def __init__(self, mapper): - """Create a router for the given routes.Mapper. - - Each route in `mapper` must specify a 'controller', which is a - WSGI app to call. You'll probably want to specify an 'action' as - well and have your controller be a wsgi.Controller, who will route - the request to the action method. - - Examples: - mapper = routes.Mapper() - sc = ServerController() - - # Explicit mapping of one route to a controller+action - mapper.connect(None, "/svrlist", controller=sc, action="list") - - # Actions are all implicitly defined - mapper.resource("network", "networks", controller=nc) - - # Pointing to an arbitrary WSGI app. You can specify the - # {path_info:.*} parameter so the target app can be handed just that - # section of the URL. - mapper.connect(None, "/v1.0/{path_info:.*}", controller=BlogApp()) - """ - self.map = mapper - self._router = routes.middleware.RoutesMiddleware(self._dispatch, - self.map) - - @webob.dec.wsgify - def __call__(self, req): - """Route the incoming request to a controller based on self.map. - - If no match, return a 404. - """ - return self._router - - @staticmethod - @webob.dec.wsgify(RequestClass=Request) - def _dispatch(req): - """Dispatch a Request. - - Called by self._router after matching the incoming request to a route - and putting the information into req.environ. Either returns 404 - or the routed WSGI app's response. - """ - match = req.environ['wsgiorg.routing_args'][1] - if not match: - language = req.best_match_language() - msg = _('The resource could not be found.') - msg = oslo_i18n.translate(msg, language) - return webob.exc.HTTPNotFound(explanation=msg) - app = match['controller'] - return app - - class Resource(Application): """WSGI app that handles (de)serialization and controller dispatch. diff --git a/requirements.txt b/requirements.txt index 3f6823d7b..0676de1f5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -36,7 +36,7 @@ oslo.middleware>=2.8.0 # Apache-2.0 oslo.policy>=0.5.0 # Apache-2.0 oslo.rootwrap>=2.0.0 # Apache-2.0 oslo.serialization>=1.4.0 # Apache-2.0 -oslo.service>=0.7.0 # Apache-2.0 +oslo.service>=0.9.0 # Apache-2.0 oslo.utils>=2.0.0 # Apache-2.0 oslo.versionedobjects>=0.9.0