From 0be5d5d4c4c26d28fd1f39496a7a88d208afff13 Mon Sep 17 00:00:00 2001 From: David Heinemeier Hansson Date: Mon, 1 Aug 2016 16:44:29 -0700 Subject: Add exponentially_longer and custom wait algorithms --- activejob/lib/active_job/exceptions.rb | 28 ++++++++++++++++++++--- activejob/test/cases/exceptions_test.rb | 39 +++++++++++++++++++++++++++++++++ activejob/test/jobs/retry_job.rb | 4 ++++ 3 files changed, 68 insertions(+), 3 deletions(-) diff --git a/activejob/lib/active_job/exceptions.rb b/activejob/lib/active_job/exceptions.rb index 6b43a8bcb5..d4c973d3c1 100644 --- a/activejob/lib/active_job/exceptions.rb +++ b/activejob/lib/active_job/exceptions.rb @@ -12,7 +12,10 @@ module ActiveJob # holding queue for inspection. # # ==== Options - # * :wait - Re-enqueues the job with the specified delay in seconds (default: 3 seconds) + # * :wait - Re-enqueues the job with a delay specified either in seconds (default: 3 seconds), + # as a computing proc that the number of executions so far as an argument, or as a symbol reference of + # :exponentially_longer<>, which applies the wait algorithm of (executions ** 4) + 2 + # (first wait 3s, then 18s, then 83s, etc) # * :attempts - Re-enqueues the job the specified number of times (default: 5 attempts) # * :queue - Re-enqueues the job on a different queue # * :priority - Re-enqueues the job with a different priority @@ -20,9 +23,14 @@ module ActiveJob # ==== Examples # # class RemoteServiceJob < ActiveJob::Base - # retry_on Net::OpenTimeout, wait: 30.seconds, attempts: 10 + # retry_on CustomAppException # defaults to 3s wait, 5 attempts + # retry_on AnotherCustomAppException, wait: ->(executions) { executions * 2 } + # retry_on ActiveRecord::StatementInvalid, wait: 5.seconds, attempts: 3 + # retry_on Net::OpenTimeout, wait: :exponentially_longer, attempts: 10 # # def perform(*args) + # # Might raise CustomAppException or AnotherCustomAppException for something domain specific + # # Might raise ActiveRecord::StatementInvalid when a local db deadlock is detected # # Might raise Net::OpenTimeout when the remote service is down # end # end @@ -30,7 +38,7 @@ module ActiveJob rescue_from exception do |error| if executions < attempts logger.error "Retrying #{self.class} in #{wait} seconds, due to a #{exception}. The original exception was #{error.cause.inspect}." - retry_job wait: wait, queue: queue, priority: priority + retry_job wait: determine_delay(wait), queue: queue, priority: priority else logger.error "Stopped retrying #{self.class} due to a #{exception}, which reoccurred on #{executions} attempts. The original exception was #{error.cause.inspect}." raise error @@ -81,5 +89,19 @@ module ActiveJob def retry_job(options = {}) enqueue options end + + private + def determine_delay(seconds_or_algorithm) + case seconds_or_algorithm + when :exponentially_longer + (executions ** 4) + 2 + when Integer + seconds = seconds_or_algorithm + seconds + when Proc + algorithm = seconds_or_algorithm + algorithm.call(executions) + end + end end end diff --git a/activejob/test/cases/exceptions_test.rb b/activejob/test/cases/exceptions_test.rb index f6ca528362..0bc1d82623 100644 --- a/activejob/test/cases/exceptions_test.rb +++ b/activejob/test/cases/exceptions_test.rb @@ -1,5 +1,6 @@ require 'helper' require 'jobs/retry_job' +require 'byebug' class ExceptionsTest < ActiveSupport::TestCase setup do @@ -45,3 +46,41 @@ class ExceptionsTest < ActiveSupport::TestCase assert_equal "Raised DiscardableError for the 1st time", JobBuffer.last_value end end + +class ExponentiallyBackoffExceptionsTest < ActiveJob::TestCase + setup do + JobBuffer.clear + end + + test "exponentially retrying job" do + travel_to Time.now + + perform_enqueued_jobs do + assert_performed_with at: (Time.now + 3.seconds).to_i do + assert_performed_with at: (Time.now + 18.seconds).to_i do + assert_performed_with at: (Time.now + 83.seconds).to_i do + assert_performed_with at: (Time.now + 258.seconds).to_i do + RetryJob.perform_later 'ExponentialWaitTenAttemptsError', 5 + end + end + end + end + end + end + + test "custom wait retrying job" do + travel_to Time.now + + perform_enqueued_jobs do + assert_performed_with at: (Time.now + 2.seconds).to_i do + assert_performed_with at: (Time.now + 4.seconds).to_i do + assert_performed_with at: (Time.now + 6.seconds).to_i do + assert_performed_with at: (Time.now + 8.seconds).to_i do + RetryJob.perform_later 'CustomWaitTenAttemptsError', 5 + end + end + end + end + end + end +end \ No newline at end of file diff --git a/activejob/test/jobs/retry_job.rb b/activejob/test/jobs/retry_job.rb index 91be37f106..35177c5be5 100644 --- a/activejob/test/jobs/retry_job.rb +++ b/activejob/test/jobs/retry_job.rb @@ -3,11 +3,15 @@ require 'active_support/core_ext/integer/inflections' class DefaultsError < StandardError; end class ShortWaitTenAttemptsError < StandardError; end +class ExponentialWaitTenAttemptsError < StandardError; end +class CustomWaitTenAttemptsError < StandardError; end class DiscardableError < StandardError; end class RetryJob < ActiveJob::Base retry_on DefaultsError retry_on ShortWaitTenAttemptsError, wait: 1.second, attempts: 10 + retry_on ExponentialWaitTenAttemptsError, wait: :exponentially_longer, attempts: 10 + retry_on CustomWaitTenAttemptsError, wait: ->(executions) { executions * 2 }, attempts: 10 discard_on DiscardableError def perform(raising, attempts) -- cgit v1.2.3