Implement retry on tests which pull key from a key server which sometimes times out...

diff --git a/spec/acceptance/apt_key_provider_spec.rb b/spec/acceptance/apt_key_provider_spec.rb
index 3e056ee31ca968fc53047e036cbd8e0c2a551577..b4347e19c5f9b1952e9015fb28aff1b00e48ccb7 100644 (file)
--- a/spec/acceptance/apt_key_provider_spec.rb
+++ b/spec/acceptance/apt_key_provider_spec.rb
@@ -17,6 +17,10 @@ KEY_CHECK_COMMAND              = "apt-key adv --list-keys --with-colons --finger
 PUPPETLABS_KEY_CHECK_COMMAND   = "#{KEY_CHECK_COMMAND} #{PUPPETLABS_GPG_KEY_FINGERPRINT}"
 CENTOS_KEY_CHECK_COMMAND       = "#{KEY_CHECK_COMMAND} #{CENTOS_GPG_KEY_FINGERPRINT}"
 
+MAX_TIMEOUT_RETRY              = 3
+TIMEOUT_RETRY_WAIT             = 5
+TIMEOUT_ERROR_MATCHER    = /no valid OpenPGP data found/
+
 describe 'apt_key' do
   before(:each) do
     # Delete twice to make sure everything is cleaned
@@ -85,9 +89,11 @@ describe 'apt_key' do
         }
         EOS
 
-        # Install the key first
-        shell("apt-key adv --keyserver hkps.pool.sks-keyservers.net \
+        # Install the key first (retry because key pool may timeout)
+        retry_on_error_matching(MAX_TIMEOUT_RETRY, TIMEOUT_RETRY_WAIT, TIMEOUT_ERROR_MATCHER) do
+          shell("apt-key adv --keyserver hkps.pool.sks-keyservers.net \
               --recv-keys #{CENTOS_GPG_KEY_FINGERPRINT}")
+        end
         shell(CENTOS_KEY_CHECK_COMMAND)
 
         # Time to remove it using Puppet
@@ -97,8 +103,11 @@ describe 'apt_key' do
         shell(CENTOS_KEY_CHECK_COMMAND,
               :acceptable_exit_codes => [1])
 
-        shell("apt-key adv --keyserver hkps.pool.sks-keyservers.net \
-              --recv-keys #{CENTOS_GPG_KEY_FINGERPRINT}")
+        # Re-Install the key (retry because key pool may timeout)
+        retry_on_error_matching(MAX_TIMEOUT_RETRY, TIMEOUT_RETRY_WAIT, TIMEOUT_ERROR_MATCHER) do
+          shell("apt-key adv --keyserver hkps.pool.sks-keyservers.net \
+                --recv-keys #{CENTOS_GPG_KEY_FINGERPRINT}")
+        end
       end
     end
 
@@ -111,9 +120,12 @@ describe 'apt_key' do
         }
         EOS
 
-        # Install the key first
-        shell("apt-key adv --keyserver hkps.pool.sks-keyservers.net \
-              --recv-keys #{PUPPETLABS_GPG_KEY_LONG_ID}")
+        # Install the key first (retry because key pool may timeout)
+        retry_on_error_matching(MAX_TIMEOUT_RETRY, TIMEOUT_RETRY_WAIT, TIMEOUT_ERROR_MATCHER) do
+          shell("apt-key adv --keyserver hkps.pool.sks-keyservers.net \
+                --recv-keys #{PUPPETLABS_GPG_KEY_LONG_ID}")
+        end
+
         shell(PUPPETLABS_KEY_CHECK_COMMAND)
 
         # Time to remove it using Puppet
@@ -188,12 +200,17 @@ zGioYMWgVePywFGaTV51/0uF9ymHHC7BDIcLgUWHdg/1B67jR5YQfzPJUqLhnylt
           }
         EOS
 
-        apply_manifest(pp, :catch_failures => true)
+        #Apply the manifest (Retry if timeout error is received from key pool)
+        retry_on_error_matching(MAX_TIMEOUT_RETRY, TIMEOUT_RETRY_WAIT, TIMEOUT_ERROR_MATCHER) do
+          apply_manifest(pp, :catch_failures => true)
+        end
+
         apply_manifest(pp, :catch_changes => true)
         shell(PUPPETLABS_KEY_CHECK_COMMAND)
       end
     end
 
+
     context 'multiple keys' do
       it 'runs without errors' do
         pp = <<-EOS
@@ -448,6 +465,7 @@ FPfZDNCu/TXoqyJk7434jJrcHgPryzrHFBLfEmc=
 -----END PGP PUBLIC KEY BLOCK----- ",
           }
         EOS
+
         apply_manifest(pp, :catch_failures => true)
         apply_manifest(pp, :catch_changes => true)
         shell(PUPPETLABS_KEY_CHECK_COMMAND)
@@ -482,7 +500,11 @@ FPfZDNCu/TXoqyJk7434jJrcHgPryzrHFBLfEmc=
         }
         EOS
 
-        apply_manifest(pp, :catch_failures => true)
+        #Apply the manifest (Retry if timeout error is received from key pool)
+        retry_on_error_matching(MAX_TIMEOUT_RETRY, TIMEOUT_RETRY_WAIT, TIMEOUT_ERROR_MATCHER) do
+          apply_manifest(pp, :catch_failures => true)
+        end
+
         apply_manifest(pp, :catch_changes => true)
         shell(PUPPETLABS_KEY_CHECK_COMMAND)
       end
@@ -498,7 +520,10 @@ FPfZDNCu/TXoqyJk7434jJrcHgPryzrHFBLfEmc=
         }
         EOS
 
-        apply_manifest(pp, :catch_failures => true)
+        retry_on_error_matching(MAX_TIMEOUT_RETRY, TIMEOUT_RETRY_WAIT, TIMEOUT_ERROR_MATCHER) do
+          apply_manifest(pp, :catch_failures => true)
+        end
+
         apply_manifest(pp, :catch_changes => true)
         shell(PUPPETLABS_KEY_CHECK_COMMAND)
       end

diff --git a/spec/acceptance/apt_spec.rb b/spec/acceptance/apt_spec.rb
index 86c93ffa87913540069653d707974f67282c011a..52415f5afaecb4e0cfe28fbe2af689c3ffad3f77 100644 (file)
--- a/spec/acceptance/apt_spec.rb
+++ b/spec/acceptance/apt_spec.rb
@@ -1,5 +1,9 @@
 require 'spec_helper_acceptance'
 
+MAX_TIMEOUT_RETRY              = 3
+TIMEOUT_RETRY_WAIT             = 5
+TIMEOUT_ERROR_MATCHER    = /no valid OpenPGP data found/
+
 describe 'apt class' do
 
   context 'reset' do
@@ -42,8 +46,12 @@ describe 'apt class' do
       }
       EOS
 
+      #Apply the manifest (Retry if timeout error is received from key pool)
+      retry_on_error_matching(MAX_TIMEOUT_RETRY, TIMEOUT_RETRY_WAIT, TIMEOUT_ERROR_MATCHER) do
+        apply_manifest(pp, :catch_failures => true)
+      end
+
       apply_manifest(pp, :catch_failures => true)
-      apply_manifest(pp, :catch_changes => true)
     end
     it 'should still work' do
       shell('apt-get update')

diff --git a/spec/spec_helper_acceptance.rb b/spec/spec_helper_acceptance.rb
index 409ce68b299d2ff4b302ab2e46726d26643017d0..a5ce06fc45815156e2d32c61377f97d04de21701 100644 (file)
--- a/spec/spec_helper_acceptance.rb
+++ b/spec/spec_helper_acceptance.rb
@@ -5,6 +5,32 @@ run_puppet_install_helper
 
 UNSUPPORTED_PLATFORMS = ['RedHat','Suse','windows','AIX','Solaris']
 
+# This method allows a block to be passed in and if an exception is raised
+# that matches the 'error_matcher' matcher, the block will wait a set number
+# of seconds before retrying.
+# Params:
+# - max_retry_count - Max number of retries
+# - retry_wait_interval_secs - Number of seconds to wait before retry
+# - error_matcher - Matcher which the exception raised must match to allow retry
+# Example Usage:
+# retry_on_error_matching(3, 5, /OpenGPG Error/) do
+#   apply_manifest(pp, :catch_failures => true)
+# end
+def retry_on_error_matching(max_retry_count = 3, retry_wait_interval_secs = 5, error_matcher = nil)
+  try = 0
+  begin
+    try += 1
+    yield
+  rescue Exception => e
+    if try < max_retry_count && (error_matcher == nil || e.message =~ error_matcher)
+      sleep retry_wait_interval_secs
+      retry
+    else
+      raise
+    end
+  end
+end
+
 RSpec.configure do |c|
   # Project root
   proj_root = File.expand_path(File.join(File.dirname(__FILE__), '..'))