Backport infinite loop detection

Upstream fix https://github.com/rabbitmq/rabbitmq-common/pull/26 (patch
modified to reflect current state of upstream code).

Sudden death of cluster node could result in a stuck queue process -
this will result in redeclare attempts to hang. With this patch such
condition will be detected - AMQP channel will be closed and error will
be logged. And probably it could help us to discover underlying bug, by
localizing it in time.

And for referenced partial bugs it'll allow us to confirm or reject
hypothesis that it's related.

Change-Id: I09df5c5f2333cc462475798260cdfa9f4f5de654
Closes-Bug: #1515223
(cherry picked from commit 6b45958024465ff64f5466815cf04169da0c9cf5)

diff --git a/debian/changelog b/debian/changelog
index 24eb81b1cea43767ceda32d1bbec5e4a1b2fdf39..74bb11165ad109276fc5773e0dccbe13879ff9ec 100644 (file)
--- a/debian/changelog
+++ b/debian/changelog
@@ -1,3 +1,9 @@
+rabbitmq-server (3.5.4-1~u14.04+mos4) mos7.0; urgency=medium
+
+  * Backport https://github.com/rabbitmq/rabbitmq-common/pull/26
+
+ -- Alexey Lebedeff <alebedev@mirantis.com>  Thu, 24 Dec 2015 14:28:56 +0300
+
 rabbitmq-server (3.5.4-1~u14.04+mos3) mos7.0; urgency=medium
 
   * Backport upstream fix for management plugin

diff --git a/debian/patches/detect-stuck-queue-on-declare.diff b/debian/patches/detect-stuck-queue-on-declare.diff
new file mode 100644 (file)
index 0000000..00c43f3
--- /dev/null
+++ b/debian/patches/detect-stuck-queue-on-declare.diff
@@ -0,0 +1,35 @@
+Description: Detect infinite loop in AMQP channel code
+    Sudden death of cluster node could result in a stuck queue process - this will result in
+    redeclare attempts to hang. With this patch such condition will be detected - AMQP channel will
+    be closed and error will be logged. And probably it could help us to discover underlying bug, by
+    localizing the event in time.
+Author: Alexey Lebedeff <alebedev@mirantis.com>
+Origin: upstream, https://github.com/rabbitmq/rabbitmq-common/pull/26
+Bug:https://github.com/rabbitmq/rabbitmq-server/issues/349
+---
+This patch header follows DEP-3: http://dep.debian.net/deps/dep3/
+--- a/src/rabbit_amqqueue.erl
++++ b/src/rabbit_amqqueue.erl
+@@ -385,6 +385,11 @@ not_found_or_absent_dirty(Name) ->
+     end.
+ 
+ with(Name, F, E) ->
++    with(Name, F, E, 2000).
++
++with(Name, _F, E, 0) ->
++    E(not_found_or_absent_dirty(Name));
++with(Name, F, E, RetriesLeft) ->
+     case lookup(Name) of
+         {ok, Q = #amqqueue{state = crashed}} ->
+             E({absent, Q, crashed});
+@@ -397,8 +402,8 @@ with(Name, F, E) ->
+             %% the retry loop.
+             rabbit_misc:with_exit_handler(
+               fun () -> false = rabbit_mnesia:is_process_alive(QPid),
+-                        timer:sleep(25),
+-                        with(Name, F, E)
++                        timer:sleep(30),
++                        with(Name, F, E, RetriesLeft - 1)
+               end, fun () -> F(Q) end);
+         {error, not_found} ->
+             E(not_found_or_absent_dirty(Name))

diff --git a/debian/patches/series b/debian/patches/series
index 970b278fdb3c6b0bb052996d5a3e0e60190586c2..a053945a07633c60bcb95868266f4c4d294882d3 100644 (file)
--- a/debian/patches/series
+++ b/debian/patches/series
@@ -1 +1,2 @@
 fix-management-startup-after-split.diff
+detect-stuck-queue-on-declare.diff