cinder volume service keeps retrying even code exception

Make sure that ProcessLauncher does not try to restart the failed
service. And make sure the ProcessLauncher exits, when the number
of the failed services equal the number of total services.

Fixes: bug 1167841
Change-Id: I818c754534c18a62d292e21f0c98086571fe0565
(cherry picked from commit 2cee8fc3a4899097801ad93b5d6e6a4f373ad8fe)

diff --git a/cinder/service.py b/cinder/service.py
index 4056f44950eff312c12954a8c8d3dbe14eaa8942..4e7c549770d177de36f1254d214ab16df45bc182 100644 (file)
--- a/cinder/service.py
+++ b/cinder/service.py
@@ -142,12 +142,15 @@ class ServerWrapper(object):
         self.workers = workers
         self.children = set()
         self.forktimes = []
+        self.failed = False
 
 
 class ProcessLauncher(object):
     def __init__(self):
         self.children = {}
         self.sigcaught = None
+        self.totalwrap = 0
+        self.failedwrap = 0
         self.running = True
         rfd, self.writepipe = os.pipe()
         self.readpipe = eventlet.greenio.GreenPipe(rfd, 'r')
@@ -246,9 +249,10 @@ class ProcessLauncher(object):
 
     def launch_server(self, server, workers=1):
         wrap = ServerWrapper(server, workers)
-
+        self.totalwrap = self.totalwrap + 1
         LOG.info(_('Starting %d workers'), wrap.workers)
-        while self.running and len(wrap.children) < wrap.workers:
+        while (self.running and len(wrap.children) < wrap.workers
+               and not wrap.failed):
             self._start_child(wrap)
 
     def _wait_child(self):
@@ -262,6 +266,7 @@ class ProcessLauncher(object):
                 raise
             return None
 
+        code = 0
         if os.WIFSIGNALED(status):
             sig = os.WTERMSIG(status)
             LOG.info(_('Child %(pid)d killed by signal %(sig)d'), locals())
@@ -275,6 +280,12 @@ class ProcessLauncher(object):
 
         wrap = self.children.pop(pid)
         wrap.children.remove(pid)
+        if 2 == code:
+            wrap.failed = True
+            self.failedwrap = self.failedwrap + 1
+            LOG.info(_('_wait_child %d'), self.failedwrap)
+            if self.failedwrap == self.totalwrap:
+                self.running = False
         return wrap
 
     def wait(self):
@@ -288,7 +299,9 @@ class ProcessLauncher(object):
                 eventlet.greenthread.sleep(.01)
                 continue
 
-            while self.running and len(wrap.children) < wrap.workers:
+            LOG.info(_('wait wrap.failed %s'), wrap.failed)
+            while (self.running and len(wrap.children) < wrap.workers
+                   and not wrap.failed):
                 self._start_child(wrap)
 
         if self.sigcaught: