diff --git a/lib/beanstalk-client/connection.rb b/lib/beanstalk-client/connection.rb
index 79af511..9fb917e 100644
--- a/lib/beanstalk-client/connection.rb
+++ b/lib/beanstalk-client/connection.rb
@@ -356,7 +356,7 @@ module Beanstalk
       self.last_conn = c
       c.send(*args)
     rescue EOFError, Errno::ECONNRESET, Errno::EPIPE, UnexpectedResponse => ex
-      self.remove(c)
+      self.remove(c) if ex.is_a?(TimedOut)
       raise ex
     end