]> review.fuel-infra Code Review - openstack-build/neutron-build.git/commitdiff
Improve error handling when nvp and quantum are out of sync
authorAaron Rosen <arosen@nicira.com>
Sun, 17 Feb 2013 00:26:14 +0000 (16:26 -0800)
committerAaron Rosen <arosen@nicira.com>
Mon, 18 Feb 2013 02:41:33 +0000 (18:41 -0800)
Previouly when nvp and quantum were out of sync an exception would
be raised notifying the user of this. This patch changes the behavior so that
now when  elements are found in the quantum db but not in NVP they are
put into error state. In addition there error state elements are now able
to be deleted if not found in nvp.

This patch also removes outdated doc strings that seem to keep being adding.

Fixes bug 1127643

Change-Id: I3e79ab7a284e61623cd63fefc3755746b8e4dfd7

quantum/plugins/nicira/nicira_nvp_plugin/QuantumPlugin.py
quantum/plugins/nicira/nicira_nvp_plugin/common/exceptions.py
quantum/plugins/nicira/nicira_nvp_plugin/nvplib.py
quantum/tests/unit/nicira/fake_nvpapiclient.py
quantum/tests/unit/nicira/test_nicira_plugin.py

index a8e2de004c283ee7f15d9e4e30405835d575265c..93d26149340fc262da8fec747fccc011ff64c793 100644 (file)
@@ -397,13 +397,17 @@ class NvpPluginV2(db_base_plugin_v2.QuantumDbPluginV2,
         # TODO(bgh): if this is a bridged network and the lswitch we just got
         # back will have zero ports after the delete we should garbage collect
         # the lswitch.
-        nvplib.delete_port(self.default_cluster,
-                           port_data['network_id'],
-                           port)
-        LOG.debug(_("_nvp_delete_port completed for port %(port_id)s "
-                    "on network %(net_id)s"),
-                  {'port_id': port_data['id'],
-                   'net_id': port_data['network_id']})
+        try:
+            nvplib.delete_port(self.default_cluster,
+                               port_data['network_id'],
+                               port)
+            LOG.debug(_("_nvp_delete_port completed for port %(port_id)s "
+                        "on network %(net_id)s"),
+                      {'port_id': port_data['id'],
+                       'net_id': port_data['network_id']})
+
+        except q_exc.NotFound:
+            LOG.warning(_("port %s not found in NVP"), port_data['id'])
 
     def _nvp_delete_router_port(self, context, port_data):
         # Delete logical router port
@@ -768,14 +772,6 @@ class NvpPluginV2(db_base_plugin_v2.QuantumDbPluginV2,
         return new_net
 
     def delete_network(self, context, id):
-        """
-        Deletes the network with the specified network identifier
-        belonging to the specified tenant.
-
-        :returns: None
-        :raises: exception.NetworkInUse
-        :raises: exception.NetworkNotFound
-        """
         external = self._network_is_external(context, id)
         # Before deleting ports, ensure the peer of a NVP logical
         # port with a patch attachment is removed too
@@ -812,14 +808,17 @@ class NvpPluginV2(db_base_plugin_v2.QuantumDbPluginV2,
 
         # Do not go to NVP for external networks
         if not external:
-            # FIXME(salvatore-orlando): Failures here might lead NVP
-            # and quantum state to diverge
-            pairs = self._get_lswitch_cluster_pairs(id, context.tenant_id)
-            for (cluster, switches) in pairs:
-                nvplib.delete_networks(cluster, id, switches)
+            try:
+                # FIXME(salvatore-orlando): Failures here might lead NVP
+                # and quantum state to diverge
+                pairs = self._get_lswitch_cluster_pairs(id, context.tenant_id)
+                for (cluster, switches) in pairs:
+                    nvplib.delete_networks(cluster, id, switches)
 
-        LOG.debug(_("delete_network completed for tenant: %s"),
-                  context.tenant_id)
+                LOG.debug(_("delete_network completed for tenant: %s"),
+                          context.tenant_id)
+            except q_exc.NotFound:
+                LOG.warning(_("Did not found lswitch %s in NVP"), id)
 
     def _get_lswitch_cluster_pairs(self, netw_id, tenant_id):
         """Figure out the set of lswitches on each cluster that maps to this
@@ -872,6 +871,8 @@ class NvpPluginV2(db_base_plugin_v2.QuantumDbPluginV2,
                     if nvp_net_status != network.status:
                         # update the network status
                         network.status = nvp_net_status
+                except q_exc.NotFound:
+                    network.status = constants.NET_STATUS_ERROR
                 except Exception:
                     err_msg = _("Unable to get logical switches")
                     LOG.exception(err_msg)
@@ -933,14 +934,17 @@ class NvpPluginV2(db_base_plugin_v2.QuantumDbPluginV2,
             if quantum_lswitch[l3.EXTERNAL]:
                 continue
             elif quantum_lswitch['id'] not in nvp_lswitches:
-                raise nvp_exc.NvpOutOfSyncException()
-            # TODO(salvatore-orlando): be careful about "extended"
-            # logical switches
-            ls = nvp_lswitches.pop(quantum_lswitch['id'])
-            if (ls["_relations"]["LogicalSwitchStatus"]["fabric_status"]):
-                quantum_lswitch["status"] = constants.NET_STATUS_ACTIVE
+                LOG.warning(_("Logical Switch %s found in quantum database "
+                              "but not in NVP."), quantum_lswitch["id"])
+                quantum_lswitch["status"] = constants.NET_STATUS_ERROR
             else:
-                quantum_lswitch["status"] = constants.NET_STATUS_DOWN
+                # TODO(salvatore-orlando): be careful about "extended"
+                # logical switches
+                ls = nvp_lswitches.pop(quantum_lswitch['id'])
+                if (ls["_relations"]["LogicalSwitchStatus"]["fabric_status"]):
+                    quantum_lswitch["status"] = constants.NET_STATUS_ACTIVE
+                else:
+                    quantum_lswitch["status"] = constants.NET_STATUS_DOWN
 
         # do not make the case in which switches are found in NVP
         # but not in Quantum catastrophic.
@@ -1030,7 +1034,12 @@ class NvpPluginV2(db_base_plugin_v2.QuantumDbPluginV2,
                     "&relations=LogicalPortStatus" %
                     (lswitch, lport_fields_str, vm_filter, tenant_filter))
 
-                ports = nvplib.get_all_query_pages(lport_query_path, c)
+                try:
+                    ports = nvplib.get_all_query_pages(lport_query_path, c)
+                except q_exc.NotFound:
+                    LOG.warn(_("Lswitch %s not found in NVP"), lswitch)
+                    ports = None
+
                 if ports:
                     for port in ports:
                         for tag in port["tags"]:
@@ -1063,12 +1072,12 @@ class NvpPluginV2(db_base_plugin_v2.QuantumDbPluginV2,
                     quantum_lport["status"] = constants.PORT_STATUS_DOWN
 
                 del nvp_lports[quantum_lport["id"]]
-                lports.append(quantum_lport)
             except KeyError:
-
+                quantum_lport["status"] = constants.PORT_STATUS_ERROR
                 LOG.debug(_("Quantum logical port %s was not found on NVP"),
                           quantum_lport['id'])
 
+            lports.append(quantum_lport)
         # do not make the case in which ports are found in NVP
         # but not in Quantum catastrophic.
         if len(nvp_lports):
@@ -1282,13 +1291,18 @@ class NvpPluginV2(db_base_plugin_v2.QuantumDbPluginV2,
 
             nvp_id = nicira_db.get_nvp_port_id(context.session, id)
             #TODO: pass the appropriate cluster here
-            port = nvplib.get_logical_port_status(
-                self.default_cluster, quantum_db_port['network_id'], nvp_id)
-            quantum_db_port["admin_state_up"] = port["admin_status_enabled"]
-            if port["fabric_status_up"]:
-                quantum_db_port["status"] = constants.PORT_STATUS_ACTIVE
-            else:
-                quantum_db_port["status"] = constants.PORT_STATUS_DOWN
+            try:
+                port = nvplib.get_logical_port_status(
+                    self.default_cluster, quantum_db_port['network_id'],
+                    nvp_id)
+                quantum_db_port["admin_state_up"] = (
+                    port["admin_status_enabled"])
+                if port["fabric_status_up"]:
+                    quantum_db_port["status"] = constants.PORT_STATUS_ACTIVE
+                else:
+                    quantum_db_port["status"] = constants.PORT_STATUS_DOWN
+            except q_exc.NotFound:
+                quantum_db_port["status"] = constants.PORT_STATUS_ERROR
         return quantum_db_port
 
     def create_router(self, context, router):
@@ -1393,13 +1407,12 @@ class NvpPluginV2(db_base_plugin_v2.QuantumDbPluginV2,
             # together with the resource
             try:
                 nvplib.delete_lrouter(self.default_cluster, id)
-            except NvpApiClient.ResourceNotFound:
-                raise nvp_exc.NvpPluginException(
-                    err_msg=(_("Logical router '%s' not found "
-                               "on NVP Platform") % id))
+            except q_exc.NotFound:
+                LOG.warning(_("Logical router '%s' not found "
+                              "on NVP Platform") % id)
             except NvpApiClient.NvpApiException:
                 raise nvp_exc.NvpPluginException(
-                    err_msg=(_("Unable to update logical router"
+                    err_msg=(_("Unable to delete logical router"
                                "on NVP Platform")))
 
     def get_router(self, context, id, fields=None):
@@ -1408,23 +1421,26 @@ class NvpPluginV2(db_base_plugin_v2.QuantumDbPluginV2,
             # FIXME(salvatore-orlando): We need to
             # find the appropriate cluster!
             cluster = self.default_cluster
-            lrouter = nvplib.get_lrouter(cluster, id)
-            router_op_status = constants.NET_STATUS_DOWN
+            try:
+                lrouter = nvplib.get_lrouter(cluster, id)
+            except q_exc.NotFound:
+                lrouter = {}
+                router_op_status = constants.NET_STATUS_ERROR
             relations = lrouter.get('_relations')
             if relations:
                 lrouter_status = relations.get('LogicalRouterStatus')
-            # FIXME(salvatore-orlando): Being unable to fetch the
-            # logical router status should be an exception.
-            if lrouter_status:
-                router_op_status = (lrouter_status.get('fabric_status')
-                                    and constants.NET_STATUS_ACTIVE or
-                                    constants.NET_STATUS_DOWN)
-            LOG.debug(_("Current router status:%(router_status)s;"
-                        "Status in Quantum DB:%(db_router_status)s"),
-                      {'router_status': router_op_status,
-                       'db_router_status': router.status})
+                # FIXME(salvatore-orlando): Being unable to fetch the
+                # logical router status should be an exception.
+                if lrouter_status:
+                    router_op_status = (lrouter_status.get('fabric_status')
+                                        and constants.NET_STATUS_ACTIVE or
+                                        constants.NET_STATUS_DOWN)
             if router_op_status != router.status:
-                # update the network status
+                LOG.debug(_("Current router status:%(router_status)s;"
+                            "Status in Quantum DB:%(db_router_status)s"),
+                          {'router_status': router_op_status,
+                           'db_router_status': router.status})
+                 # update the router status
                 with context.session.begin(subtransactions=True):
                     router.status = router_op_status
         except NvpApiClient.NvpApiException:
@@ -1467,6 +1483,8 @@ class NvpPluginV2(db_base_plugin_v2.QuantumDbPluginV2,
                 else:
                     router.status = constants.NET_STATUS_DOWN
                 nvp_lrouters.remove(nvp_lrouter)
+            else:
+                router.status = constants.NET_STATUS_ERROR
 
         # do not make the case in which routers are found in NVP
         # but not in Quantum catastrophic.
@@ -1655,7 +1673,6 @@ class NvpPluginV2(db_base_plugin_v2.QuantumDbPluginV2,
 
         Overrides method from base class.
         The method is augmented for creating NAT rules in the process.
-
         """
         if (('fixed_ip_address' in fip and fip['fixed_ip_address']) and
             not ('port_id' in fip and fip['port_id'])):
index caad357eb5130dd667f5b11a773796822a5e2673..64d365fa4c6f69fab423fd6a1adebed69a26ad98 100644 (file)
@@ -38,10 +38,6 @@ class NvpNoMorePortsException(NvpPluginException):
                 "Maximum number of ports reached")
 
 
-class NvpOutOfSyncException(NvpPluginException):
-    message = _("Quantum state has diverged from the networking backend!")
-
-
 class NvpNatRuleMismatch(NvpPluginException):
     message = _("While retrieving NAT rules, %(actual_rules)s were found "
                 "whereas rules in the (%(min_rules)s,%(max_rules)s) interval "
index 2732d6b0bcd38360149f1c739442ce49b0e09d9c..7198929d9e37ba92e683aa6c86116b8e05cb645d 100644 (file)
@@ -187,7 +187,11 @@ def do_single_request(*args, **kwargs):
     """Issue a request to a specified cluster if specified via kwargs
        (cluster=<cluster>)."""
     cluster = kwargs["cluster"]
-    return cluster.api_client.request(*args)
+    try:
+        req = cluster.api_client.request(*args)
+    except NvpApiClient.ResourceNotFound:
+        raise exception.NotFound()
+    return req
 
 
 def do_multi_request(*args, **kwargs):
@@ -254,6 +258,8 @@ def get_lswitches(cluster, quantum_net_id):
                                                      cluster)
                 results.extend(extra_switches)
         return results
+    except NvpApiClient.ResourceNotFound:
+        raise exception.NetworkNotFound(net_id=quantum_net_id)
     except NvpApiClient.NvpApiException:
         # TODO(salvatore-olrando): Do a better exception handling
         # and re-raising
index bbbae9f15e1966f7b4f650e2511d221360b96a47..86e5b107982cae81258706b3b13fa44f107831ab 100644 (file)
@@ -19,6 +19,7 @@ import urlparse
 
 from quantum.openstack.common import log as logging
 from quantum.openstack.common import uuidutils
+from quantum.plugins.nicira.nicira_nvp_plugin import NvpApiClient
 
 
 LOG = logging.getLogger(__name__)
@@ -374,8 +375,7 @@ class FakeClient:
                      for res_uuid in res_dict if res_uuid == target_uuid]
             if items:
                 return json.dumps(items[0])
-            raise Exception("show: resource %s:%s not found" %
-                            (resource_type, target_uuid))
+            raise NvpApiClient.ResourceNotFound()
 
     def handle_get(self, url):
         #TODO(salvatore-orlando): handle field selection
@@ -483,7 +483,10 @@ class FakeClient:
         if not response_file:
             raise Exception("resource not found")
         res_dict = getattr(self, '_fake_%s_dict' % res_type)
-        del res_dict[uuids[-1]]
+        try:
+            del res_dict[uuids[-1]]
+        except KeyError:
+            raise NvpApiClient.ResourceNotFound()
         return ""
 
     def fake_request(self, *args, **kwargs):
index 261120fc42d745c375b76ccc9e0756d0799ddc59..f96b4bfe1a433911dcd7ae4de00c9871f824fc51 100644 (file)
@@ -21,6 +21,7 @@ import mock
 from oslo.config import cfg
 import webob.exc
 
+from quantum.common import constants
 import quantum.common.test_lib as test_lib
 from quantum import context
 from quantum.extensions import providernet as pnet
@@ -490,3 +491,105 @@ class TestNiciraQoSQueue(NiciraPluginV2TestCase):
             res = req.get_response(self.ext_api)
             queue = self.deserialize('json', res)
             self.assertEqual(queue['qos_queue']['max'], 20)
+
+
+class NiciraQuantumNVPOutOfSync(test_l3_plugin.L3NatTestCaseBase,
+                                NiciraPluginV2TestCase):
+
+    def test_delete_network_not_in_nvp(self):
+        res = self._create_network('json', 'net1', True)
+        net1 = self.deserialize('json', res)
+        self.fc._fake_lswitch_dict.clear()
+        req = self.new_delete_request('networks', net1['network']['id'])
+        res = req.get_response(self.api)
+        self.assertEqual(res.status_int, 204)
+
+    def test_list_networks_not_in_nvp(self):
+        res = self._create_network('json', 'net1', True)
+        self.deserialize('json', res)
+        self.fc._fake_lswitch_dict.clear()
+        req = self.new_list_request('networks')
+        nets = self.deserialize('json', req.get_response(self.api))
+        self.assertEquals(nets['networks'][0]['status'],
+                          constants.NET_STATUS_ERROR)
+
+    def test_show_network_not_in_nvp(self):
+        res = self._create_network('json', 'net1', True)
+        net = self.deserialize('json', res)
+        self.fc._fake_lswitch_dict.clear()
+        req = self.new_show_request('networks', net['network']['id'])
+        net = self.deserialize('json', req.get_response(self.api))
+        self.assertEquals(net['network']['status'],
+                          constants.NET_STATUS_ERROR)
+
+    def test_delete_port_not_in_nvp(self):
+        res = self._create_network('json', 'net1', True)
+        net1 = self.deserialize('json', res)
+        res = self._create_port('json', net1['network']['id'])
+        port = self.deserialize('json', res)
+        self.fc._fake_lswitch_lport_dict.clear()
+        req = self.new_delete_request('ports', port['port']['id'])
+        res = req.get_response(self.api)
+        self.assertEqual(res.status_int, 204)
+
+    def test_list_port_not_in_nvp(self):
+        res = self._create_network('json', 'net1', True)
+        net1 = self.deserialize('json', res)
+        res = self._create_port('json', net1['network']['id'])
+        self.deserialize('json', res)
+        self.fc._fake_lswitch_lport_dict.clear()
+        req = self.new_list_request('ports')
+        nets = self.deserialize('json', req.get_response(self.api))
+        self.assertEquals(nets['ports'][0]['status'],
+                          constants.PORT_STATUS_ERROR)
+
+    def test_show_port_not_in_nvp(self):
+        res = self._create_network('json', 'net1', True)
+        net1 = self.deserialize('json', res)
+        res = self._create_port('json', net1['network']['id'])
+        port = self.deserialize('json', res)
+        self.fc._fake_lswitch_lport_dict.clear()
+        req = self.new_show_request('ports', port['port']['id'])
+        net = self.deserialize('json', req.get_response(self.api))
+        self.assertEquals(net['port']['status'],
+                          constants.PORT_STATUS_ERROR)
+
+    def test_delete_port_and_network_not_in_nvp(self):
+        res = self._create_network('json', 'net1', True)
+        net1 = self.deserialize('json', res)
+        res = self._create_port('json', net1['network']['id'])
+        port = self.deserialize('json', res)
+        self.fc._fake_lswitch_dict.clear()
+        self.fc._fake_lswitch_lport_dict.clear()
+        req = self.new_delete_request('ports', port['port']['id'])
+        res = req.get_response(self.api)
+        self.assertEqual(res.status_int, 204)
+        req = self.new_delete_request('networks', net1['network']['id'])
+        res = req.get_response(self.api)
+        self.assertEqual(res.status_int, 204)
+
+    def test_delete_router_not_in_nvp(self):
+        res = self._create_router('json', 'tenant')
+        router = self.deserialize('json', res)
+        self.fc._fake_lrouter_dict.clear()
+        req = self.new_delete_request('routers', router['router']['id'])
+        res = req.get_response(self.ext_api)
+        self.assertEqual(res.status_int, 204)
+
+    def test_list_routers_not_in_nvp(self):
+        res = self._create_router('json', 'tenant')
+        self.deserialize('json', res)
+        self.fc._fake_lrouter_dict.clear()
+        req = self.new_list_request('routers')
+        routers = self.deserialize('json', req.get_response(self.ext_api))
+        self.assertEquals(routers['routers'][0]['status'],
+                          constants.NET_STATUS_ERROR)
+
+    def test_show_router_not_in_nvp(self):
+        res = self._create_router('json', 'tenant')
+        router = self.deserialize('json', res)
+        self.fc._fake_lrouter_dict.clear()
+        req = self.new_show_request('routers', router['router']['id'])
+        router = self.deserialize('json', req.get_response(self.ext_api))
+        self.assertEquals(router['router']['status'],
+                          constants.NET_STATUS_ERROR)