diff options
Diffstat (limited to 'activejob')
21 files changed, 237 insertions, 41 deletions
diff --git a/activejob/CHANGELOG.md b/activejob/CHANGELOG.md index 965b556fab..79235019fe 100644 --- a/activejob/CHANGELOG.md +++ b/activejob/CHANGELOG.md @@ -1,3 +1,16 @@ +* Fixed serializing `:at` option for `assert_enqueued_with` + and `assert_performed_with`. + + *Wojciech Wnętrzak* + +* Support passing array to `assert_enqueued_jobs` in `:only` option. + + *Wojciech Wnętrzak* + +* Add job priorities to Active Job. + + *wvengen* + * Implement a simple `AsyncJob` processor and associated `AsyncAdapter` that queue jobs to a `concurrent-ruby` thread pool. diff --git a/activejob/lib/active_job/async_job.rb b/activejob/lib/active_job/async_job.rb index 7fcffc4c24..6c1c070994 100644 --- a/activejob/lib/active_job/async_job.rb +++ b/activejob/lib/active_job/async_job.rb @@ -1,5 +1,4 @@ require 'concurrent' -require 'thread_safe' module ActiveJob # == Active Job Async Job @@ -31,7 +30,7 @@ module ActiveJob fallback_policy: :caller_runs # shouldn't matter -- 0 max queue }.freeze - QUEUES = ThreadSafe::Cache.new do |hash, queue_name| #:nodoc: + QUEUES = Concurrent::Map.new do |hash, queue_name| #:nodoc: hash.compute_if_absent(queue_name) { ActiveJob::AsyncJob.create_thread_pool } end diff --git a/activejob/lib/active_job/base.rb b/activejob/lib/active_job/base.rb index 5d7c4cfb91..e5f09f65fb 100644 --- a/activejob/lib/active_job/base.rb +++ b/activejob/lib/active_job/base.rb @@ -1,6 +1,7 @@ require 'active_job/core' require 'active_job/queue_adapter' require 'active_job/queue_name' +require 'active_job/queue_priority' require 'active_job/enqueuing' require 'active_job/execution' require 'active_job/callbacks' @@ -57,6 +58,7 @@ module ActiveJob #:nodoc: include Core include QueueAdapter include QueueName + include QueuePriority include Enqueuing include Execution include Callbacks diff --git a/activejob/lib/active_job/core.rb b/activejob/lib/active_job/core.rb index eac7279309..19b900a285 100644 --- a/activejob/lib/active_job/core.rb +++ b/activejob/lib/active_job/core.rb @@ -18,6 +18,9 @@ module ActiveJob # Queue in which the job will reside. attr_writer :queue_name + # Priority that the job will have (lower is more priority). + attr_writer :priority + # ID optionally provided by adapter attr_accessor :provider_job_id @@ -43,6 +46,7 @@ module ActiveJob # * <tt>:wait</tt> - Enqueues the job with the specified delay # * <tt>:wait_until</tt> - Enqueues the job at the time specified # * <tt>:queue</tt> - Enqueues the job on the specified queue + # * <tt>:priority</tt> - Enqueues the job with the specified priority # # ==== Examples # @@ -51,6 +55,7 @@ module ActiveJob # VideoJob.set(wait_until: Time.now.tomorrow).perform_later(Video.last) # VideoJob.set(queue: :some_queue, wait: 5.minutes).perform_later(Video.last) # VideoJob.set(queue: :some_queue, wait_until: Time.now.tomorrow).perform_later(Video.last) + # VideoJob.set(queue: :some_queue, wait: 5.minutes, priority: 10).perform_later(Video.last) def set(options={}) ConfiguredJob.new(self, options) end @@ -62,6 +67,7 @@ module ActiveJob @arguments = arguments @job_id = SecureRandom.uuid @queue_name = self.class.queue_name + @priority = self.class.priority end # Returns a hash with the job data that can safely be passed to the @@ -71,6 +77,7 @@ module ActiveJob 'job_class' => self.class.name, 'job_id' => job_id, 'queue_name' => queue_name, + 'priority' => priority, 'arguments' => serialize_arguments(arguments), 'locale' => I18n.locale } @@ -99,6 +106,7 @@ module ActiveJob def deserialize(job_data) self.job_id = job_data['job_id'] self.queue_name = job_data['queue_name'] + self.priority = job_data['priority'] self.serialized_arguments = job_data['arguments'] self.locale = job_data['locale'] || I18n.locale end diff --git a/activejob/lib/active_job/enqueuing.rb b/activejob/lib/active_job/enqueuing.rb index 98d92385dd..22154457fd 100644 --- a/activejob/lib/active_job/enqueuing.rb +++ b/activejob/lib/active_job/enqueuing.rb @@ -32,6 +32,7 @@ module ActiveJob # * <tt>:wait</tt> - Enqueues the job with the specified delay # * <tt>:wait_until</tt> - Enqueues the job at the time specified # * <tt>:queue</tt> - Enqueues the job on the specified queue + # * <tt>:priority</tt> - Enqueues the job with the specified priority # # ==== Examples # @@ -54,6 +55,7 @@ module ActiveJob # * <tt>:wait</tt> - Enqueues the job with the specified delay # * <tt>:wait_until</tt> - Enqueues the job at the time specified # * <tt>:queue</tt> - Enqueues the job on the specified queue + # * <tt>:priority</tt> - Enqueues the job with the specified priority # # ==== Examples # @@ -61,10 +63,12 @@ module ActiveJob # my_job_instance.enqueue wait: 5.minutes # my_job_instance.enqueue queue: :important # my_job_instance.enqueue wait_until: Date.tomorrow.midnight + # my_job_instance.enqueue priority: 10 def enqueue(options={}) self.scheduled_at = options[:wait].seconds.from_now.to_f if options[:wait] self.scheduled_at = options[:wait_until].to_f if options[:wait_until] self.queue_name = self.class.queue_name_from_part(options[:queue]) if options[:queue] + self.priority = options[:priority].to_i if options[:priority] run_callbacks :enqueue do if self.scheduled_at self.class.queue_adapter.enqueue_at self, self.scheduled_at diff --git a/activejob/lib/active_job/logging.rb b/activejob/lib/active_job/logging.rb index d72e1bdfce..605057d1e8 100644 --- a/activejob/lib/active_job/logging.rb +++ b/activejob/lib/active_job/logging.rb @@ -26,7 +26,7 @@ module ActiveJob end end - before_enqueue do |job| + after_enqueue do |job| if job.scheduled_at ActiveSupport::Notifications.instrument "enqueue_at.active_job", adapter: job.class.queue_adapter, job: job diff --git a/activejob/lib/active_job/queue_adapters/delayed_job_adapter.rb b/activejob/lib/active_job/queue_adapters/delayed_job_adapter.rb index ac83da2b9c..0a785fad3b 100644 --- a/activejob/lib/active_job/queue_adapters/delayed_job_adapter.rb +++ b/activejob/lib/active_job/queue_adapters/delayed_job_adapter.rb @@ -14,13 +14,13 @@ module ActiveJob # Rails.application.config.active_job.queue_adapter = :delayed_job class DelayedJobAdapter def enqueue(job) #:nodoc: - delayed_job = Delayed::Job.enqueue(JobWrapper.new(job.serialize), queue: job.queue_name) + delayed_job = Delayed::Job.enqueue(JobWrapper.new(job.serialize), queue: job.queue_name, priority: job.priority) job.provider_job_id = delayed_job.id delayed_job end def enqueue_at(job, timestamp) #:nodoc: - delayed_job = Delayed::Job.enqueue(JobWrapper.new(job.serialize), queue: job.queue_name, run_at: Time.at(timestamp)) + delayed_job = Delayed::Job.enqueue(JobWrapper.new(job.serialize), queue: job.queue_name, priority: job.priority, run_at: Time.at(timestamp)) job.provider_job_id = delayed_job.id delayed_job end diff --git a/activejob/lib/active_job/queue_adapters/que_adapter.rb b/activejob/lib/active_job/queue_adapters/que_adapter.rb index 90947aa98d..ab13689747 100644 --- a/activejob/lib/active_job/queue_adapters/que_adapter.rb +++ b/activejob/lib/active_job/queue_adapters/que_adapter.rb @@ -16,13 +16,13 @@ module ActiveJob # Rails.application.config.active_job.queue_adapter = :que class QueAdapter def enqueue(job) #:nodoc: - que_job = JobWrapper.enqueue job.serialize + que_job = JobWrapper.enqueue job.serialize, priority: job.priority job.provider_job_id = que_job.attrs["job_id"] que_job end def enqueue_at(job, timestamp) #:nodoc: - que_job = JobWrapper.enqueue job.serialize, run_at: Time.at(timestamp) + que_job = JobWrapper.enqueue job.serialize, priority: job.priority, run_at: Time.at(timestamp) job.provider_job_id = que_job.attrs["job_id"] que_job end diff --git a/activejob/lib/active_job/queue_adapters/sneakers_adapter.rb b/activejob/lib/active_job/queue_adapters/sneakers_adapter.rb index f102c6567e..d78bdecdcb 100644 --- a/activejob/lib/active_job/queue_adapters/sneakers_adapter.rb +++ b/activejob/lib/active_job/queue_adapters/sneakers_adapter.rb @@ -1,5 +1,5 @@ require 'sneakers' -require 'thread' +require 'monitor' module ActiveJob module QueueAdapters diff --git a/activejob/lib/active_job/queue_priority.rb b/activejob/lib/active_job/queue_priority.rb new file mode 100644 index 0000000000..01d84910ff --- /dev/null +++ b/activejob/lib/active_job/queue_priority.rb @@ -0,0 +1,44 @@ +module ActiveJob + module QueuePriority + extend ActiveSupport::Concern + + # Includes the ability to override the default queue priority. + module ClassMethods + mattr_accessor(:default_priority) + + # Specifies the priority of the queue to create the job with. + # + # class PublishToFeedJob < ActiveJob::Base + # queue_with_priority 50 + # + # def perform(post) + # post.to_feed! + # end + # end + # + # Specify either an argument or a block. + def queue_with_priority(priority=nil, &block) + if block_given? + self.priority = block + else + self.priority = priority + end + end + end + + included do + class_attribute :priority, instance_accessor: false + + self.priority = default_priority + end + + # Returns the priority that the job will be created with + def priority + if @priority.is_a?(Proc) + @priority = instance_exec(&@priority) + end + @priority + end + + end +end diff --git a/activejob/lib/active_job/test_helper.rb b/activejob/lib/active_job/test_helper.rb index 200d82838e..44ddfa5f69 100644 --- a/activejob/lib/active_job/test_helper.rb +++ b/activejob/lib/active_job/test_helper.rb @@ -226,20 +226,22 @@ module ActiveJob # assert_enqueued_with(job: MyJob, args: [1,2,3], queue: 'low') do # MyJob.perform_later(1,2,3) # end + # + # assert_enqueued_with(job: MyJob, at: Date.tomorrow.noon) do + # MyJob.set(wait_until: Date.tomorrow.noon).perform_later + # end # end - def assert_enqueued_with(args = {}, &_block) - original_enqueued_jobs = enqueued_jobs.dup - clear_enqueued_jobs + def assert_enqueued_with(args = {}) + original_enqueued_jobs_count = enqueued_jobs.count args.assert_valid_keys(:job, :args, :at, :queue) serialized_args = serialize_args_for_assertion(args) yield - matching_job = enqueued_jobs.find do |job| + in_block_jobs = enqueued_jobs.drop(original_enqueued_jobs_count) + matching_job = in_block_jobs.find do |job| serialized_args.all? { |key, value| value == job[key] } end assert matching_job, "No enqueued job found with #{args}" instantiate_job(matching_job) - ensure - queue_adapter.enqueued_jobs = original_enqueued_jobs + enqueued_jobs end # Asserts that the job passed in the block has been performed with the given arguments. @@ -248,20 +250,22 @@ module ActiveJob # assert_performed_with(job: MyJob, args: [1,2,3], queue: 'high') do # MyJob.perform_later(1,2,3) # end + # + # assert_performed_with(job: MyJob, at: Date.tomorrow.noon) do + # MyJob.set(wait_until: Date.tomorrow.noon).perform_later + # end # end - def assert_performed_with(args = {}, &_block) - original_performed_jobs = performed_jobs.dup - clear_performed_jobs + def assert_performed_with(args = {}) + original_performed_jobs_count = performed_jobs.count args.assert_valid_keys(:job, :args, :at, :queue) serialized_args = serialize_args_for_assertion(args) perform_enqueued_jobs { yield } - matching_job = performed_jobs.find do |job| + in_block_jobs = performed_jobs.drop(original_performed_jobs_count) + matching_job = in_block_jobs.find do |job| serialized_args.all? { |key, value| value == job[key] } end assert matching_job, "No performed job found with #{args}" instantiate_job(matching_job) - ensure - queue_adapter.performed_jobs = original_performed_jobs + performed_jobs end def perform_enqueued_jobs(only: nil) @@ -290,31 +294,30 @@ module ActiveJob to: :queue_adapter private - def clear_enqueued_jobs + def clear_enqueued_jobs # :nodoc: enqueued_jobs.clear end - def clear_performed_jobs + def clear_performed_jobs # :nodoc: performed_jobs.clear end - def enqueued_jobs_size(only: nil) + def enqueued_jobs_size(only: nil) # :nodoc: if only - enqueued_jobs.select { |job| job.fetch(:job) == only }.size + enqueued_jobs.count { |job| Array(only).include?(job.fetch(:job)) } else - enqueued_jobs.size + enqueued_jobs.count end end - def serialize_args_for_assertion(args) - serialized_args = args.dup - if job_args = serialized_args.delete(:args) - serialized_args[:args] = ActiveJob::Arguments.serialize(job_args) + def serialize_args_for_assertion(args) # :nodoc: + args.dup.tap do |serialized_args| + serialized_args[:args] = ActiveJob::Arguments.serialize(serialized_args[:args]) if serialized_args[:args] + serialized_args[:at] = serialized_args[:at].to_f if serialized_args[:at] end - serialized_args end - def instantiate_job(payload) + def instantiate_job(payload) # :nodoc: job = payload[:job].new(*payload[:args]) job.scheduled_at = Time.at(payload[:at]) if payload.key?(:at) job.queue_name = payload[:queue] diff --git a/activejob/test/cases/queue_priority_test.rb b/activejob/test/cases/queue_priority_test.rb new file mode 100644 index 0000000000..ca17b51dad --- /dev/null +++ b/activejob/test/cases/queue_priority_test.rb @@ -0,0 +1,47 @@ +require 'helper' +require 'jobs/hello_job' + +class QueuePriorityTest < ActiveSupport::TestCase + test 'priority unset by default' do + assert_equal nil, HelloJob.priority + end + + test 'uses given priority' do + original_priority = HelloJob.priority + + begin + HelloJob.queue_with_priority 90 + assert_equal 90, HelloJob.new.priority + ensure + HelloJob.priority = original_priority + end + end + + test 'evals block given to priority to determine priority' do + original_priority = HelloJob.priority + + begin + HelloJob.queue_with_priority { 25 } + assert_equal 25, HelloJob.new.priority + ensure + HelloJob.priority = original_priority + end + end + + test 'can use arguments to determine priority in priority block' do + original_priority = HelloJob.priority + + begin + HelloJob.queue_with_priority { self.arguments.first=='1' ? 99 : 11 } + assert_equal 99, HelloJob.new('1').priority + assert_equal 11, HelloJob.new('3').priority + ensure + HelloJob.priority = original_priority + end + end + + test 'uses priority passed to #set' do + job = HelloJob.set(priority: 123).perform_later + assert_equal 123, job.priority + end +end diff --git a/activejob/test/cases/test_case_test.rb b/activejob/test/cases/test_case_test.rb index ee816e1dd5..616454a4b6 100644 --- a/activejob/test/cases/test_case_test.rb +++ b/activejob/test/cases/test_case_test.rb @@ -9,7 +9,7 @@ class ActiveJobTestCaseTest < ActiveJob::TestCase # the `class_attribute` inheritance class TestClassAttributeInheritanceJob < ActiveJob::Base def self.queue_adapter=(*) - raise 'Attemping to break `class_attribute` inheritance, bad!' + raise 'Attempting to break `class_attribute` inheritance, bad!' end end diff --git a/activejob/test/cases/test_helper_test.rb b/activejob/test/cases/test_helper_test.rb index 8c60f037d2..f7ee763e8a 100644 --- a/activejob/test/cases/test_helper_test.rb +++ b/activejob/test/cases/test_helper_test.rb @@ -140,6 +140,16 @@ class EnqueuedJobsTest < ActiveJob::TestCase assert_match(/1 .* but 2/, error.message) end + def test_assert_enqueued_jobs_with_only_option_as_array + assert_nothing_raised do + assert_enqueued_jobs 2, only: [HelloJob, LoggingJob] do + HelloJob.perform_later('jeremy') + LoggingJob.perform_later('stewie') + RescueJob.perform_later('david') + end + end + end + def test_assert_no_enqueued_jobs_with_only_option assert_nothing_raised do assert_no_enqueued_jobs only: HelloJob do @@ -159,6 +169,14 @@ class EnqueuedJobsTest < ActiveJob::TestCase assert_match(/0 .* but 1/, error.message) end + def test_assert_no_enqueued_jobs_with_only_option_as_array + assert_nothing_raised do + assert_no_enqueued_jobs only: [HelloJob, RescueJob] do + LoggingJob.perform_later + end + end + end + def test_assert_enqueued_job assert_enqueued_with(job: LoggingJob, queue: 'default') do LoggingJob.set(wait_until: Date.tomorrow.noon).perform_later @@ -200,6 +218,12 @@ class EnqueuedJobsTest < ActiveJob::TestCase end end + def test_assert_enqueued_job_with_at_option + assert_enqueued_with(job: HelloJob, at: Date.tomorrow.noon) do + HelloJob.set(wait_until: Date.tomorrow.noon).perform_later + end + end + def test_assert_enqueued_job_with_global_id_args ricardo = Person.new(9) assert_enqueued_with(job: HelloJob, args: [ricardo]) do @@ -218,6 +242,15 @@ class EnqueuedJobsTest < ActiveJob::TestCase assert_equal "No enqueued job found with {:job=>HelloJob, :args=>[#{wilma.inspect}]}", error.message end + + def test_assert_enqueued_job_does_not_change_jobs_count + HelloJob.perform_later + assert_enqueued_with(job: HelloJob) do + HelloJob.perform_later + end + + assert_equal 2, ActiveJob::Base.queue_adapter.enqueued_jobs.count + end end class PerformedJobsTest < ActiveJob::TestCase @@ -421,14 +454,26 @@ class PerformedJobsTest < ActiveJob::TestCase def test_assert_performed_job_failure assert_raise ActiveSupport::TestCase::Assertion do - assert_performed_with(job: LoggingJob, at: Date.tomorrow.noon, queue: 'default') do - NestedJob.set(wait_until: Date.tomorrow.noon).perform_later + assert_performed_with(job: LoggingJob) do + HelloJob.perform_later end end assert_raise ActiveSupport::TestCase::Assertion do - assert_performed_with(job: NestedJob, at: Date.tomorrow.noon, queue: 'low') do - NestedJob.set(queue: 'low', wait_until: Date.tomorrow.noon).perform_later + assert_performed_with(job: HelloJob, queue: 'low') do + HelloJob.set(queue: 'important').perform_later + end + end + end + + def test_assert_performed_job_with_at_option + assert_performed_with(job: HelloJob, at: Date.tomorrow.noon) do + HelloJob.set(wait_until: Date.tomorrow.noon).perform_later + end + + assert_raise ActiveSupport::TestCase::Assertion do + assert_performed_with(job: HelloJob, at: Date.today.noon) do + HelloJob.set(wait_until: Date.tomorrow.noon).perform_later end end end @@ -451,4 +496,16 @@ class PerformedJobsTest < ActiveJob::TestCase assert_equal "No performed job found with {:job=>HelloJob, :args=>[#{wilma.inspect}]}", error.message end + + def test_assert_performed_job_does_not_change_jobs_count + assert_performed_with(job: HelloJob) do + HelloJob.perform_later + end + + assert_performed_with(job: HelloJob) do + HelloJob.perform_later + end + + assert_equal 2, ActiveJob::Base.queue_adapter.performed_jobs.count + end end diff --git a/activejob/test/helper.rb b/activejob/test/helper.rb index 55f690bda8..7e86415f48 100644 --- a/activejob/test/helper.rb +++ b/activejob/test/helper.rb @@ -11,6 +11,7 @@ GlobalID.app = 'aj' if ENV['AJ_INTEGRATION_TESTS'] require 'support/integration/helper' else + ActiveJob::Base.logger = Logger.new(nil) require "adapters/#{@adapter}" end diff --git a/activejob/test/integration/queuing_test.rb b/activejob/test/integration/queuing_test.rb index 125ba54302..e435ed4aa6 100644 --- a/activejob/test/integration/queuing_test.rb +++ b/activejob/test/integration/queuing_test.rb @@ -84,4 +84,16 @@ class QueuingTest < ActiveSupport::TestCase I18n.locale = :en end end + + test 'should run job with higher priority first' do + skip unless adapter_is?(:delayed_job, :que) + + wait_until = Time.now + 3.seconds + TestJob.set(wait_until: wait_until, priority: 20).perform_later "#{@id}.1" + TestJob.set(wait_until: wait_until, priority: 10).perform_later "#{@id}.2" + wait_for_jobs_to_finish_for(10.seconds) + assert job_executed "#{@id}.1" + assert job_executed "#{@id}.2" + assert job_executed_at("#{@id}.2") < job_executed_at("#{@id}.1") + end end diff --git a/activejob/test/support/integration/adapters/sidekiq.rb b/activejob/test/support/integration/adapters/sidekiq.rb index d9938600f2..9aa07bcb52 100644 --- a/activejob/test/support/integration/adapters/sidekiq.rb +++ b/activejob/test/support/integration/adapters/sidekiq.rb @@ -58,7 +58,7 @@ module SidekiqJobsManager timeout: 1, }) Sidekiq.average_scheduled_poll_interval = 0.5 - Sidekiq::Scheduled.const_set :INITIAL_WAIT, 1 + Sidekiq.options[:poll_interval_average] = 1 begin sidekiq.run continue_write.puts "started" diff --git a/activejob/test/support/integration/dummy_app_template.rb b/activejob/test/support/integration/dummy_app_template.rb index 4ffdb8cffa..0c062a025e 100644 --- a/activejob/test/support/integration/dummy_app_template.rb +++ b/activejob/test/support/integration/dummy_app_template.rb @@ -1,8 +1,9 @@ if ENV['AJ_ADAPTER'] == 'delayed_job' generate "delayed_job:active_record", "--quiet" - rake("db:migrate") end +rake("db:migrate") + initializer 'activejob.rb', <<-CODE require "#{File.expand_path("../jobs_manager.rb", __FILE__)}" JobsManager.current_manager.setup diff --git a/activejob/test/support/integration/helper.rb b/activejob/test/support/integration/helper.rb index 8c2e5a86c2..4a1b0bfbcb 100644 --- a/activejob/test/support/integration/helper.rb +++ b/activejob/test/support/integration/helper.rb @@ -1,4 +1,4 @@ -puts "*** rake aj:integration:#{ENV['AJ_ADAPTER']} ***\n" +puts "\n\n*** rake aj:integration:#{ENV['AJ_ADAPTER']} ***\n" ENV["RAILS_ENV"] = "test" ActiveJob::Base.queue_name_prefix = nil diff --git a/activejob/test/support/integration/test_case_helpers.rb b/activejob/test/support/integration/test_case_helpers.rb index 39aee6d407..8319d09520 100644 --- a/activejob/test/support/integration/test_case_helpers.rb +++ b/activejob/test/support/integration/test_case_helpers.rb @@ -42,8 +42,12 @@ module TestCaseHelpers end end - def job_executed - Dummy::Application.root.join("tmp/#{@id}").exist? + def job_executed(id=@id) + Dummy::Application.root.join("tmp/#{id}").exist? + end + + def job_executed_at(id=@id) + File.new(Dummy::Application.root.join("tmp/#{id}")).ctime end def job_output diff --git a/activejob/test/support/que/inline.rb b/activejob/test/support/que/inline.rb index 0232da1370..0950e52d28 100644 --- a/activejob/test/support/que/inline.rb +++ b/activejob/test/support/que/inline.rb @@ -6,6 +6,7 @@ Que::Job.class_eval do if args.last.is_a?(Hash) options = args.pop options.delete(:run_at) + options.delete(:priority) args << options unless options.empty? end self.run(*args) |