NSX sync cache: add a flag to skip item deletion

This new flag will prevent updates on single resources, which
can happen asynchronously to the main synchronization thread,
from processing the 'changed' flag for items in the cache and
delete items not present anymore in NSX.

This will avoid failures seen in CI runs where the sync
thread failed because of a KeyError because some items where
concurrently removed from the NSX cache.

Change-Id: I6a11b8dabb1e9481dc807b5fbba17c9027dad5f2
Closes-Bug: #1329650

diff --git a/neutron/plugins/vmware/common/sync.py b/neutron/plugins/vmware/common/sync.py
index 76c6a27f4164c5210276aabea16169b11d97ec7f..a35ae07a4c9403df992fc184bc9f9b239f65c64f 100644 (file)
--- a/neutron/plugins/vmware/common/sync.py
+++ b/neutron/plugins/vmware/common/sync.py
@@ -71,7 +71,7 @@ class NsxCache(object):
         resources = self._uuid_dict_mappings[key]
         return resources[key]
 
-    def _update_resources(self, resources, new_resources):
+    def _clear_changed_flag_and_remove_from_cache(self, resources):
         # Clear the 'changed' attribute for all items
         for uuid, item in resources.items():
             if item.pop('changed', None) and not item.get('data'):
@@ -80,6 +80,10 @@ class NsxCache(object):
                 del self._uuid_dict_mappings[uuid]
                 LOG.debug("Removed item %s from NSX object cache", uuid)
 
+    def _update_resources(self, resources, new_resources, clear_changed=True):
+        if clear_changed:
+            self._clear_changed_flag_and_remove_from_cache(resources)
+
         def do_hash(item):
             return hash(jsonutils.dumps(item))
 
@@ -132,13 +136,14 @@ class NsxCache(object):
         return self._get_resource_ids(self._lswitchports, changed_only)
 
     def update_lswitch(self, lswitch):
-        self._update_resources(self._lswitches, [lswitch])
+        self._update_resources(self._lswitches, [lswitch], clear_changed=False)
 
     def update_lrouter(self, lrouter):
-        self._update_resources(self._lrouters, [lrouter])
+        self._update_resources(self._lrouters, [lrouter], clear_changed=False)
 
     def update_lswitchport(self, lswitchport):
-        self._update_resources(self._lswitchports, [lswitchport])
+        self._update_resources(self._lswitchports, [lswitchport],
+                               clear_changed=False)
 
     def process_updates(self, lswitches=None,
                         lrouters=None, lswitchports=None):

diff --git a/neutron/tests/unit/vmware/test_nsx_sync.py b/neutron/tests/unit/vmware/test_nsx_sync.py
index 67c2fdd440b58907bc727e234c7a6069249eec35..0c5e782ee363c85bf7d0bf651950318225cd0e07 100644 (file)
--- a/neutron/tests/unit/vmware/test_nsx_sync.py
+++ b/neutron/tests/unit/vmware/test_nsx_sync.py
@@ -199,6 +199,12 @@ class CacheTestCase(base.BaseTestCase):
         self.nsx_cache.process_updates(lswitches, LROUTERS, LSWITCHPORTS)
         self.assertNotIn(deleted_lswitch['uuid'], self.nsx_cache._lswitches)
 
+    def test_update_resource_does_not_cleanup_deleted_resources(self):
+        deleted_lswitch, lswitches = self._test_process_updates_with_removals()
+        self.nsx_cache.process_deletes()
+        self.nsx_cache.update_lswitch(deleted_lswitch)
+        self.assertIn(deleted_lswitch['uuid'], self.nsx_cache._lswitches)
+
     def _verify_delete(self, resource, deleted=True, hit=True):
         cached_resource = self.nsx_cache[resource['uuid']]
         data_field = 'data_bk' if deleted else 'data'