From: Ihar Hrachyshka Date: Fri, 5 Dec 2014 17:17:15 +0000 (+0100) Subject: Migrate to oslo.concurrency X-Git-Url: https://review.fuel-infra.org/gitweb?a=commitdiff_plain;h=35d838ff600729f4b7b2ac8be92d3a3be4d96cb8;p=openstack-build%2Fneutron-build.git Migrate to oslo.concurrency Namespaces are not used anymore, so imports are for oslo_concurrency, not oslo.concurrency. lockutils direct call is not supported anymore, so using shipped lockutils-wrapper instead. Use ExternalLockFixture to set lock_path for unit testing. Updated cache and cache/_backends module from oslo-incubator to use oslo.concurrency and not incubator version of lockutils module. Dropped lockutils incubator module that is now moved to oslo.concurrency. Not dropping fixture module that includes lockutils fixture since other fixtures are still used in the tree. Closes-Bug: #1387092 Change-Id: I2ba2295f8f5d55f272a9d23555940586b25b5a1c --- diff --git a/neutron/agent/linux/external_process.py b/neutron/agent/linux/external_process.py index 35053d41a..a061f77b9 100644 --- a/neutron/agent/linux/external_process.py +++ b/neutron/agent/linux/external_process.py @@ -16,11 +16,11 @@ import collections import eventlet from oslo.config import cfg +from oslo_concurrency import lockutils from neutron.agent.linux import ip_lib from neutron.agent.linux import utils from neutron.i18n import _LE -from neutron.openstack.common import lockutils from neutron.openstack.common import log as logging LOG = logging.getLogger(__name__) diff --git a/neutron/agent/linux/iptables_manager.py b/neutron/agent/linux/iptables_manager.py index ce8dae365..bfd49f074 100644 --- a/neutron/agent/linux/iptables_manager.py +++ b/neutron/agent/linux/iptables_manager.py @@ -24,13 +24,13 @@ import sys from oslo.config import cfg from oslo.utils import excutils +from oslo_concurrency import lockutils from neutron.agent.common import config from neutron.agent.linux import iptables_comments as ic from neutron.agent.linux import utils as linux_utils from neutron.common import utils from neutron.i18n import _LE, _LW -from neutron.openstack.common import lockutils from neutron.openstack.common import log as logging LOG = logging.getLogger(__name__) diff --git a/neutron/common/utils.py b/neutron/common/utils.py index b5256a047..33ebcfaf7 100644 --- a/neutron/common/utils.py +++ b/neutron/common/utils.py @@ -32,9 +32,9 @@ import uuid from eventlet.green import subprocess from oslo.config import cfg from oslo.utils import excutils +from oslo_concurrency import lockutils from neutron.common import constants as q_const -from neutron.openstack.common import lockutils from neutron.openstack.common import log as logging diff --git a/neutron/openstack/common/cache/_backends/memory.py b/neutron/openstack/common/cache/_backends/memory.py index 5c02cfc40..7c9503862 100644 --- a/neutron/openstack/common/cache/_backends/memory.py +++ b/neutron/openstack/common/cache/_backends/memory.py @@ -15,9 +15,9 @@ import collections from oslo.utils import timeutils +from oslo_concurrency import lockutils from neutron.openstack.common.cache import backends -from neutron.openstack.common import lockutils class MemoryBackend(backends.BaseCache): diff --git a/neutron/openstack/common/cache/cache.py b/neutron/openstack/common/cache/cache.py index 70c4545de..a0cd2cd65 100644 --- a/neutron/openstack/common/cache/cache.py +++ b/neutron/openstack/common/cache/cache.py @@ -20,6 +20,20 @@ Supported configuration options: `key_namespace`: Namespace under which keys will be created. """ +######################################################################## +# +# THIS MODULE IS DEPRECATED +# +# Please refer to +# https://etherpad.openstack.org/p/kilo-neutron-library-proposals for +# the discussion leading to this deprecation. +# +# We recommend helping with the new neutron.cache library being created +# as a wrapper for dogpile. +# +######################################################################## + + from six.moves.urllib import parse from stevedore import driver diff --git a/neutron/openstack/common/lockutils.py b/neutron/openstack/common/lockutils.py deleted file mode 100644 index 996a810bb..000000000 --- a/neutron/openstack/common/lockutils.py +++ /dev/null @@ -1,326 +0,0 @@ -# Copyright 2011 OpenStack Foundation. -# All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may -# not use this file except in compliance with the License. You may obtain -# a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations -# under the License. - -import contextlib -import errno -import functools -import logging -import os -import shutil -import subprocess -import sys -import tempfile -import threading -import time -import weakref - -from oslo.config import cfg - -from neutron.openstack.common import fileutils -from neutron.openstack.common._i18n import _, _LE, _LI - - -LOG = logging.getLogger(__name__) - - -util_opts = [ - cfg.BoolOpt('disable_process_locking', default=False, - help='Enables or disables inter-process locks.'), - cfg.StrOpt('lock_path', - default=os.environ.get("NEUTRON_LOCK_PATH"), - help='Directory to use for lock files.') -] - - -CONF = cfg.CONF -CONF.register_opts(util_opts) - - -def set_defaults(lock_path): - cfg.set_defaults(util_opts, lock_path=lock_path) - - -class _FileLock(object): - """Lock implementation which allows multiple locks, working around - issues like bugs.debian.org/cgi-bin/bugreport.cgi?bug=632857 and does - not require any cleanup. Since the lock is always held on a file - descriptor rather than outside of the process, the lock gets dropped - automatically if the process crashes, even if __exit__ is not executed. - - There are no guarantees regarding usage by multiple green threads in a - single process here. This lock works only between processes. Exclusive - access between local threads should be achieved using the semaphores - in the @synchronized decorator. - - Note these locks are released when the descriptor is closed, so it's not - safe to close the file descriptor while another green thread holds the - lock. Just opening and closing the lock file can break synchronisation, - so lock files must be accessed only using this abstraction. - """ - - def __init__(self, name): - self.lockfile = None - self.fname = name - - def acquire(self): - basedir = os.path.dirname(self.fname) - - if not os.path.exists(basedir): - fileutils.ensure_tree(basedir) - LOG.info(_LI('Created lock path: %s'), basedir) - - self.lockfile = open(self.fname, 'w') - - while True: - try: - # Using non-blocking locks since green threads are not - # patched to deal with blocking locking calls. - # Also upon reading the MSDN docs for locking(), it seems - # to have a laughable 10 attempts "blocking" mechanism. - self.trylock() - LOG.debug('Got file lock "%s"', self.fname) - return True - except IOError as e: - if e.errno in (errno.EACCES, errno.EAGAIN): - # external locks synchronise things like iptables - # updates - give it some time to prevent busy spinning - time.sleep(0.01) - else: - raise threading.ThreadError(_("Unable to acquire lock on" - " `%(filename)s` due to" - " %(exception)s") % - {'filename': self.fname, - 'exception': e}) - - def __enter__(self): - self.acquire() - return self - - def release(self): - try: - self.unlock() - self.lockfile.close() - LOG.debug('Released file lock "%s"', self.fname) - except IOError: - LOG.exception(_LE("Could not release the acquired lock `%s`"), - self.fname) - - def __exit__(self, exc_type, exc_val, exc_tb): - self.release() - - def exists(self): - return os.path.exists(self.fname) - - def trylock(self): - raise NotImplementedError() - - def unlock(self): - raise NotImplementedError() - - -class _WindowsLock(_FileLock): - def trylock(self): - msvcrt.locking(self.lockfile.fileno(), msvcrt.LK_NBLCK, 1) - - def unlock(self): - msvcrt.locking(self.lockfile.fileno(), msvcrt.LK_UNLCK, 1) - - -class _FcntlLock(_FileLock): - def trylock(self): - fcntl.lockf(self.lockfile, fcntl.LOCK_EX | fcntl.LOCK_NB) - - def unlock(self): - fcntl.lockf(self.lockfile, fcntl.LOCK_UN) - - -if os.name == 'nt': - import msvcrt - InterProcessLock = _WindowsLock -else: - import fcntl - InterProcessLock = _FcntlLock - -_semaphores = weakref.WeakValueDictionary() -_semaphores_lock = threading.Lock() - - -def _get_lock_path(name, lock_file_prefix, lock_path=None): - # NOTE(mikal): the lock name cannot contain directory - # separators - name = name.replace(os.sep, '_') - if lock_file_prefix: - sep = '' if lock_file_prefix.endswith('-') else '-' - name = '%s%s%s' % (lock_file_prefix, sep, name) - - local_lock_path = lock_path or CONF.lock_path - - if not local_lock_path: - raise cfg.RequiredOptError('lock_path') - - return os.path.join(local_lock_path, name) - - -def external_lock(name, lock_file_prefix=None, lock_path=None): - LOG.debug('Attempting to grab external lock "%(lock)s"', - {'lock': name}) - - lock_file_path = _get_lock_path(name, lock_file_prefix, lock_path) - - return InterProcessLock(lock_file_path) - - -def remove_external_lock_file(name, lock_file_prefix=None): - """Remove an external lock file when it's not used anymore - This will be helpful when we have a lot of lock files - """ - with internal_lock(name): - lock_file_path = _get_lock_path(name, lock_file_prefix) - try: - os.remove(lock_file_path) - except OSError: - LOG.info(_LI('Failed to remove file %(file)s'), - {'file': lock_file_path}) - - -def internal_lock(name): - with _semaphores_lock: - try: - sem = _semaphores[name] - LOG.debug('Using existing semaphore "%s"', name) - except KeyError: - sem = threading.Semaphore() - _semaphores[name] = sem - LOG.debug('Created new semaphore "%s"', name) - - return sem - - -@contextlib.contextmanager -def lock(name, lock_file_prefix=None, external=False, lock_path=None): - """Context based lock - - This function yields a `threading.Semaphore` instance (if we don't use - eventlet.monkey_patch(), else `semaphore.Semaphore`) unless external is - True, in which case, it'll yield an InterProcessLock instance. - - :param lock_file_prefix: The lock_file_prefix argument is used to provide - lock files on disk with a meaningful prefix. - - :param external: The external keyword argument denotes whether this lock - should work across multiple processes. This means that if two different - workers both run a method decorated with @synchronized('mylock', - external=True), only one of them will execute at a time. - """ - int_lock = internal_lock(name) - with int_lock: - LOG.debug('Acquired semaphore "%(lock)s"', {'lock': name}) - try: - if external and not CONF.disable_process_locking: - ext_lock = external_lock(name, lock_file_prefix, lock_path) - with ext_lock: - yield ext_lock - else: - yield int_lock - finally: - LOG.debug('Releasing semaphore "%(lock)s"', {'lock': name}) - - -def synchronized(name, lock_file_prefix=None, external=False, lock_path=None): - """Synchronization decorator. - - Decorating a method like so:: - - @synchronized('mylock') - def foo(self, *args): - ... - - ensures that only one thread will execute the foo method at a time. - - Different methods can share the same lock:: - - @synchronized('mylock') - def foo(self, *args): - ... - - @synchronized('mylock') - def bar(self, *args): - ... - - This way only one of either foo or bar can be executing at a time. - """ - - def wrap(f): - @functools.wraps(f) - def inner(*args, **kwargs): - try: - with lock(name, lock_file_prefix, external, lock_path): - LOG.debug('Got semaphore / lock "%(function)s"', - {'function': f.__name__}) - return f(*args, **kwargs) - finally: - LOG.debug('Semaphore / lock released "%(function)s"', - {'function': f.__name__}) - return inner - return wrap - - -def synchronized_with_prefix(lock_file_prefix): - """Partial object generator for the synchronization decorator. - - Redefine @synchronized in each project like so:: - - (in nova/utils.py) - from nova.openstack.common import lockutils - - synchronized = lockutils.synchronized_with_prefix('nova-') - - - (in nova/foo.py) - from nova import utils - - @utils.synchronized('mylock') - def bar(self, *args): - ... - - The lock_file_prefix argument is used to provide lock files on disk with a - meaningful prefix. - """ - - return functools.partial(synchronized, lock_file_prefix=lock_file_prefix) - - -def main(argv): - """Create a dir for locks and pass it to command from arguments - - If you run this: - python -m openstack.common.lockutils python setup.py testr - - a temporary directory will be created for all your locks and passed to all - your tests in an environment variable. The temporary dir will be deleted - afterwards and the return value will be preserved. - """ - - lock_dir = tempfile.mkdtemp() - os.environ["NEUTRON_LOCK_PATH"] = lock_dir - try: - ret_val = subprocess.call(argv[1:]) - finally: - shutil.rmtree(lock_dir, ignore_errors=True) - return ret_val - - -if __name__ == '__main__': - sys.exit(main(sys.argv)) diff --git a/neutron/plugins/cisco/cfg_agent/cfg_agent.py b/neutron/plugins/cisco/cfg_agent/cfg_agent.py index 636fb5c81..3123bed61 100644 --- a/neutron/plugins/cisco/cfg_agent/cfg_agent.py +++ b/neutron/plugins/cisco/cfg_agent/cfg_agent.py @@ -22,6 +22,7 @@ from oslo.config import cfg from oslo import messaging from oslo.utils import importutils from oslo.utils import timeutils +from oslo_concurrency import lockutils from neutron.agent.common import config from neutron.agent.linux import external_process @@ -33,7 +34,6 @@ from neutron.common import topics from neutron import context as n_context from neutron.i18n import _LE, _LI, _LW from neutron import manager -from neutron.openstack.common import lockutils from neutron.openstack.common import log as logging from neutron.openstack.common import loopingcall from neutron.openstack.common import periodic_task diff --git a/neutron/plugins/cisco/db/l3/l3_router_appliance_db.py b/neutron/plugins/cisco/db/l3/l3_router_appliance_db.py index 7d0c619ce..147e1eba5 100644 --- a/neutron/plugins/cisco/db/l3/l3_router_appliance_db.py +++ b/neutron/plugins/cisco/db/l3/l3_router_appliance_db.py @@ -15,6 +15,7 @@ import copy from oslo.config import cfg +from oslo_concurrency import lockutils from sqlalchemy.orm import exc from sqlalchemy.orm import joinedload from sqlalchemy.sql import expression as expr @@ -31,7 +32,6 @@ from neutron.db import models_v2 from neutron.db import portbindings_db as p_binding from neutron.extensions import providernet as pr_net from neutron.i18n import _LE, _LI -from neutron.openstack.common import lockutils from neutron.openstack.common import log as logging from neutron.openstack.common import loopingcall from neutron.plugins.cisco.common import cisco_constants as c_const diff --git a/neutron/plugins/ml2/drivers/cisco/apic/apic_topology.py b/neutron/plugins/ml2/drivers/cisco/apic/apic_topology.py index 29b03eae9..78e1be790 100644 --- a/neutron/plugins/ml2/drivers/cisco/apic/apic_topology.py +++ b/neutron/plugins/ml2/drivers/cisco/apic/apic_topology.py @@ -22,6 +22,7 @@ eventlet.monkey_patch() from oslo.config import cfg from oslo import messaging +from oslo_concurrency import lockutils from neutron.agent.common import config from neutron.agent.linux import ip_lib @@ -32,7 +33,6 @@ from neutron.common import utils as neutron_utils from neutron.db import agents_db from neutron.i18n import _LE, _LI from neutron import manager -from neutron.openstack.common import lockutils from neutron.openstack.common import log as logging from neutron.openstack.common import periodic_task from neutron.openstack.common import service as svc diff --git a/neutron/plugins/ml2/drivers/cisco/apic/mechanism_apic.py b/neutron/plugins/ml2/drivers/cisco/apic/mechanism_apic.py index f57bbf205..6d62dc35a 100644 --- a/neutron/plugins/ml2/drivers/cisco/apic/mechanism_apic.py +++ b/neutron/plugins/ml2/drivers/cisco/apic/mechanism_apic.py @@ -17,9 +17,9 @@ from apicapi import apic_manager from keystoneclient.v2_0 import client as keyclient import netaddr from oslo.config import cfg +from oslo_concurrency import lockutils from neutron.common import constants as n_constants -from neutron.openstack.common import lockutils from neutron.openstack.common import log from neutron.plugins.common import constants from neutron.plugins.ml2 import driver_api as api diff --git a/neutron/plugins/ml2/plugin.py b/neutron/plugins/ml2/plugin.py index 3a68f2047..d58c8074a 100644 --- a/neutron/plugins/ml2/plugin.py +++ b/neutron/plugins/ml2/plugin.py @@ -21,6 +21,7 @@ from oslo.db import exception as os_db_exception from oslo.serialization import jsonutils from oslo.utils import excutils from oslo.utils import importutils +from oslo_concurrency import lockutils from sqlalchemy import exc as sql_exc from sqlalchemy.orm import exc as sa_exc @@ -55,7 +56,6 @@ from neutron.extensions import portbindings from neutron.extensions import providernet as provider from neutron.i18n import _LE, _LI, _LW from neutron import manager -from neutron.openstack.common import lockutils from neutron.openstack.common import log from neutron.openstack.common import uuidutils from neutron.plugins.common import constants as service_constants diff --git a/neutron/plugins/nuage/plugin.py b/neutron/plugins/nuage/plugin.py index c6640b4a4..224578565 100644 --- a/neutron/plugins/nuage/plugin.py +++ b/neutron/plugins/nuage/plugin.py @@ -19,6 +19,7 @@ import netaddr from oslo.config import cfg from oslo.utils import excutils from oslo.utils import importutils +from oslo_concurrency import lockutils from sqlalchemy.orm import exc from neutron.api import extensions as neutron_extensions @@ -38,7 +39,6 @@ from neutron.extensions import l3 from neutron.extensions import portbindings from neutron.extensions import providernet as pnet from neutron.extensions import securitygroup as ext_sg -from neutron.openstack.common import lockutils from neutron.openstack.common import log as logging from neutron.openstack.common import loopingcall from neutron.plugins.nuage.common import config diff --git a/neutron/plugins/nuage/syncmanager.py b/neutron/plugins/nuage/syncmanager.py index fee9319de..37e51bffb 100644 --- a/neutron/plugins/nuage/syncmanager.py +++ b/neutron/plugins/nuage/syncmanager.py @@ -15,6 +15,7 @@ from oslo.config import cfg from oslo.utils import importutils +from oslo_concurrency import lockutils import sqlalchemy.orm.exc as db_exc from neutron import context as ncontext @@ -22,7 +23,6 @@ from neutron.db import db_base_plugin_v2 from neutron.db import extraroute_db from neutron.db import securitygroups_db from neutron.i18n import _LE, _LI, _LW -from neutron.openstack.common import lockutils from neutron.openstack.common import log from neutron.plugins.nuage.common import config from neutron.plugins.nuage import nuagedb diff --git a/neutron/plugins/vmware/plugins/base.py b/neutron/plugins/vmware/plugins/base.py index 4ca8ff118..0ab860e4c 100644 --- a/neutron/plugins/vmware/plugins/base.py +++ b/neutron/plugins/vmware/plugins/base.py @@ -18,6 +18,7 @@ import uuid from oslo.config import cfg from oslo.db import exception as db_exc from oslo.utils import excutils +from oslo_concurrency import lockutils from sqlalchemy import exc as sql_exc from sqlalchemy.orm import exc as sa_exc import webob.exc @@ -52,7 +53,6 @@ from neutron.extensions import portsecurity as psec from neutron.extensions import providernet as pnet from neutron.extensions import securitygroup as ext_sg from neutron.i18n import _LE, _LI, _LW -from neutron.openstack.common import lockutils from neutron.openstack.common import log as logging from neutron.plugins.common import constants as plugin_const from neutron.plugins import vmware diff --git a/neutron/services/advanced_service.py b/neutron/services/advanced_service.py index 359a70f4a..d7fcff1f9 100644 --- a/neutron/services/advanced_service.py +++ b/neutron/services/advanced_service.py @@ -13,7 +13,8 @@ # License for the specific language governing permissions and limitations # under the License. -from neutron.openstack.common import lockutils +from oslo_concurrency import lockutils + from neutron.openstack.common import log as logging LOG = logging.getLogger(__name__) diff --git a/neutron/tests/base.py b/neutron/tests/base.py index a39397c28..eb1104c51 100644 --- a/neutron/tests/base.py +++ b/neutron/tests/base.py @@ -29,6 +29,7 @@ import os.path import fixtures from oslo.config import cfg from oslo.messaging import conffixture as messaging_conffixture +from oslo_concurrency.fixture import lockutils from neutron.common import config from neutron.common import rpc as n_rpc @@ -85,6 +86,8 @@ class BaseTestCase(sub_base.SubBaseTestCase): nuke_handlers=capture_logs, )) + self.useFixture(lockutils.ExternalLockFixture()) + self.temp_dir = self.useFixture(fixtures.TempDir()).path cfg.CONF.set_override('state_path', self.temp_dir) diff --git a/neutron/tests/unit/test_security_groups_rpc.py b/neutron/tests/unit/test_security_groups_rpc.py index b17ecbead..d25d15812 100644 --- a/neutron/tests/unit/test_security_groups_rpc.py +++ b/neutron/tests/unit/test_security_groups_rpc.py @@ -2515,9 +2515,6 @@ class TestSecurityGroupAgentWithIptables(base.BaseTestCase): super(TestSecurityGroupAgentWithIptables, self).setUp() config.register_root_helper(cfg.CONF) config.register_iptables_opts(cfg.CONF) - cfg.CONF.set_override( - 'lock_path', - '$state_path/lock') set_firewall_driver(self.FIREWALL_DRIVER) cfg.CONF.set_override('enable_ipset', False, group='SECURITYGROUP') cfg.CONF.set_override('comment_iptables_rules', False, group='AGENT') diff --git a/openstack-common.conf b/openstack-common.conf index f3195375d..d6cd8711e 100644 --- a/openstack-common.conf +++ b/openstack-common.conf @@ -5,7 +5,6 @@ module=eventlet_backdoor module=fileutils module=install_venv_common module=local -module=lockutils module=log module=loopingcall module=middleware diff --git a/requirements.txt b/requirements.txt index a3ce8191d..5ec390669 100644 --- a/requirements.txt +++ b/requirements.txt @@ -22,6 +22,7 @@ python-keystoneclient>=0.11.1 alembic>=0.7.1 six>=1.7.0 stevedore>=1.1.0 # Apache-2.0 +oslo.concurrency>=0.3.0,!=0.4.0 oslo.config>=1.6.0 # Apache-2.0 oslo.context>=0.1.0 # Apache-2.0 oslo.db>=1.3.0 # Apache-2.0 diff --git a/run_tests.sh b/run_tests.sh index 51a1046cb..6fe001f53 100755 --- a/run_tests.sh +++ b/run_tests.sh @@ -196,7 +196,7 @@ function run_pep8_changed { } -TESTRTESTS="python -m neutron.openstack.common.lockutils python setup.py testr" +TESTRTESTS="python setup.py testr" if [ $never_venv -eq 0 ] then diff --git a/tools/pretty_tox.sh b/tools/pretty_tox.sh index f27ba30f5..0fc360530 100755 --- a/tools/pretty_tox.sh +++ b/tools/pretty_tox.sh @@ -3,4 +3,4 @@ TESTRARGS=$1 exec 3>&1 -status=$(exec 4>&1 >&3; ( python -m neutron.openstack.common.lockutils python setup.py testr --slowest --testr-args="--subunit $TESTRARGS"; echo $? >&4 ) | subunit-trace -f) && exit $status +status=$(exec 4>&1 >&3; ( python setup.py testr --slowest --testr-args="--subunit $TESTRARGS"; echo $? >&4 ) | subunit-trace -f) && exit $status diff --git a/tox.ini b/tox.ini index 81dc56c1a..8a36217e2 100644 --- a/tox.ini +++ b/tox.ini @@ -61,7 +61,7 @@ commands = python ./tools/check_i18n.py ./neutron ./tools/i18n_cfg.py [testenv:cover] commands = - python -m neutron.openstack.common.lockutils python setup.py testr --coverage --testr-args='{posargs}' + python setup.py testr --coverage --testr-args='{posargs}' [testenv:venv] commands = {posargs}