diff options
Diffstat (limited to 'activejob')
19 files changed, 163 insertions, 13 deletions
diff --git a/activejob/CHANGELOG.md b/activejob/CHANGELOG.md index 965b556fab..b5742191eb 100644 --- a/activejob/CHANGELOG.md +++ b/activejob/CHANGELOG.md @@ -1,3 +1,7 @@ +* Add job priorities to Active Job. + + *wvengen* + * Implement a simple `AsyncJob` processor and associated `AsyncAdapter` that queue jobs to a `concurrent-ruby` thread pool. diff --git a/activejob/lib/active_job/async_job.rb b/activejob/lib/active_job/async_job.rb index 7fcffc4c24..6c1c070994 100644 --- a/activejob/lib/active_job/async_job.rb +++ b/activejob/lib/active_job/async_job.rb @@ -1,5 +1,4 @@ require 'concurrent' -require 'thread_safe' module ActiveJob # == Active Job Async Job @@ -31,7 +30,7 @@ module ActiveJob fallback_policy: :caller_runs # shouldn't matter -- 0 max queue }.freeze - QUEUES = ThreadSafe::Cache.new do |hash, queue_name| #:nodoc: + QUEUES = Concurrent::Map.new do |hash, queue_name| #:nodoc: hash.compute_if_absent(queue_name) { ActiveJob::AsyncJob.create_thread_pool } end diff --git a/activejob/lib/active_job/base.rb b/activejob/lib/active_job/base.rb index 5d7c4cfb91..e5f09f65fb 100644 --- a/activejob/lib/active_job/base.rb +++ b/activejob/lib/active_job/base.rb @@ -1,6 +1,7 @@ require 'active_job/core' require 'active_job/queue_adapter' require 'active_job/queue_name' +require 'active_job/queue_priority' require 'active_job/enqueuing' require 'active_job/execution' require 'active_job/callbacks' @@ -57,6 +58,7 @@ module ActiveJob #:nodoc: include Core include QueueAdapter include QueueName + include QueuePriority include Enqueuing include Execution include Callbacks diff --git a/activejob/lib/active_job/core.rb b/activejob/lib/active_job/core.rb index eac7279309..19b900a285 100644 --- a/activejob/lib/active_job/core.rb +++ b/activejob/lib/active_job/core.rb @@ -18,6 +18,9 @@ module ActiveJob # Queue in which the job will reside. attr_writer :queue_name + # Priority that the job will have (lower is more priority). + attr_writer :priority + # ID optionally provided by adapter attr_accessor :provider_job_id @@ -43,6 +46,7 @@ module ActiveJob # * <tt>:wait</tt> - Enqueues the job with the specified delay # * <tt>:wait_until</tt> - Enqueues the job at the time specified # * <tt>:queue</tt> - Enqueues the job on the specified queue + # * <tt>:priority</tt> - Enqueues the job with the specified priority # # ==== Examples # @@ -51,6 +55,7 @@ module ActiveJob # VideoJob.set(wait_until: Time.now.tomorrow).perform_later(Video.last) # VideoJob.set(queue: :some_queue, wait: 5.minutes).perform_later(Video.last) # VideoJob.set(queue: :some_queue, wait_until: Time.now.tomorrow).perform_later(Video.last) + # VideoJob.set(queue: :some_queue, wait: 5.minutes, priority: 10).perform_later(Video.last) def set(options={}) ConfiguredJob.new(self, options) end @@ -62,6 +67,7 @@ module ActiveJob @arguments = arguments @job_id = SecureRandom.uuid @queue_name = self.class.queue_name + @priority = self.class.priority end # Returns a hash with the job data that can safely be passed to the @@ -71,6 +77,7 @@ module ActiveJob 'job_class' => self.class.name, 'job_id' => job_id, 'queue_name' => queue_name, + 'priority' => priority, 'arguments' => serialize_arguments(arguments), 'locale' => I18n.locale } @@ -99,6 +106,7 @@ module ActiveJob def deserialize(job_data) self.job_id = job_data['job_id'] self.queue_name = job_data['queue_name'] + self.priority = job_data['priority'] self.serialized_arguments = job_data['arguments'] self.locale = job_data['locale'] || I18n.locale end diff --git a/activejob/lib/active_job/enqueuing.rb b/activejob/lib/active_job/enqueuing.rb index 98d92385dd..22154457fd 100644 --- a/activejob/lib/active_job/enqueuing.rb +++ b/activejob/lib/active_job/enqueuing.rb @@ -32,6 +32,7 @@ module ActiveJob # * <tt>:wait</tt> - Enqueues the job with the specified delay # * <tt>:wait_until</tt> - Enqueues the job at the time specified # * <tt>:queue</tt> - Enqueues the job on the specified queue + # * <tt>:priority</tt> - Enqueues the job with the specified priority # # ==== Examples # @@ -54,6 +55,7 @@ module ActiveJob # * <tt>:wait</tt> - Enqueues the job with the specified delay # * <tt>:wait_until</tt> - Enqueues the job at the time specified # * <tt>:queue</tt> - Enqueues the job on the specified queue + # * <tt>:priority</tt> - Enqueues the job with the specified priority # # ==== Examples # @@ -61,10 +63,12 @@ module ActiveJob # my_job_instance.enqueue wait: 5.minutes # my_job_instance.enqueue queue: :important # my_job_instance.enqueue wait_until: Date.tomorrow.midnight + # my_job_instance.enqueue priority: 10 def enqueue(options={}) self.scheduled_at = options[:wait].seconds.from_now.to_f if options[:wait] self.scheduled_at = options[:wait_until].to_f if options[:wait_until] self.queue_name = self.class.queue_name_from_part(options[:queue]) if options[:queue] + self.priority = options[:priority].to_i if options[:priority] run_callbacks :enqueue do if self.scheduled_at self.class.queue_adapter.enqueue_at self, self.scheduled_at diff --git a/activejob/lib/active_job/logging.rb b/activejob/lib/active_job/logging.rb index 54774db601..d72e1bdfce 100644 --- a/activejob/lib/active_job/logging.rb +++ b/activejob/lib/active_job/logging.rb @@ -1,3 +1,4 @@ +require 'active_support/core_ext/hash/transform_values' require 'active_support/core_ext/string/filters' require 'active_support/tagged_logging' require 'active_support/logger' @@ -87,12 +88,25 @@ module ActiveJob def args_info(job) if job.arguments.any? ' with arguments: ' + - job.arguments.map { |arg| arg.try(:to_global_id).try(:to_s) || arg.inspect }.join(', ') + job.arguments.map { |arg| format(arg).inspect }.join(', ') else '' end end + def format(arg) + case arg + when Hash + arg.transform_values { |value| format(value) } + when Array + arg.map { |value| format(value) } + when GlobalID::Identification + arg.to_global_id rescue arg + else + arg + end + end + def scheduled_at(event) Time.at(event.payload[:job].scheduled_at).utc end diff --git a/activejob/lib/active_job/queue_adapters/delayed_job_adapter.rb b/activejob/lib/active_job/queue_adapters/delayed_job_adapter.rb index ac83da2b9c..0a785fad3b 100644 --- a/activejob/lib/active_job/queue_adapters/delayed_job_adapter.rb +++ b/activejob/lib/active_job/queue_adapters/delayed_job_adapter.rb @@ -14,13 +14,13 @@ module ActiveJob # Rails.application.config.active_job.queue_adapter = :delayed_job class DelayedJobAdapter def enqueue(job) #:nodoc: - delayed_job = Delayed::Job.enqueue(JobWrapper.new(job.serialize), queue: job.queue_name) + delayed_job = Delayed::Job.enqueue(JobWrapper.new(job.serialize), queue: job.queue_name, priority: job.priority) job.provider_job_id = delayed_job.id delayed_job end def enqueue_at(job, timestamp) #:nodoc: - delayed_job = Delayed::Job.enqueue(JobWrapper.new(job.serialize), queue: job.queue_name, run_at: Time.at(timestamp)) + delayed_job = Delayed::Job.enqueue(JobWrapper.new(job.serialize), queue: job.queue_name, priority: job.priority, run_at: Time.at(timestamp)) job.provider_job_id = delayed_job.id delayed_job end diff --git a/activejob/lib/active_job/queue_adapters/que_adapter.rb b/activejob/lib/active_job/queue_adapters/que_adapter.rb index 90947aa98d..ab13689747 100644 --- a/activejob/lib/active_job/queue_adapters/que_adapter.rb +++ b/activejob/lib/active_job/queue_adapters/que_adapter.rb @@ -16,13 +16,13 @@ module ActiveJob # Rails.application.config.active_job.queue_adapter = :que class QueAdapter def enqueue(job) #:nodoc: - que_job = JobWrapper.enqueue job.serialize + que_job = JobWrapper.enqueue job.serialize, priority: job.priority job.provider_job_id = que_job.attrs["job_id"] que_job end def enqueue_at(job, timestamp) #:nodoc: - que_job = JobWrapper.enqueue job.serialize, run_at: Time.at(timestamp) + que_job = JobWrapper.enqueue job.serialize, priority: job.priority, run_at: Time.at(timestamp) job.provider_job_id = que_job.attrs["job_id"] que_job end diff --git a/activejob/lib/active_job/queue_priority.rb b/activejob/lib/active_job/queue_priority.rb new file mode 100644 index 0000000000..01d84910ff --- /dev/null +++ b/activejob/lib/active_job/queue_priority.rb @@ -0,0 +1,44 @@ +module ActiveJob + module QueuePriority + extend ActiveSupport::Concern + + # Includes the ability to override the default queue priority. + module ClassMethods + mattr_accessor(:default_priority) + + # Specifies the priority of the queue to create the job with. + # + # class PublishToFeedJob < ActiveJob::Base + # queue_with_priority 50 + # + # def perform(post) + # post.to_feed! + # end + # end + # + # Specify either an argument or a block. + def queue_with_priority(priority=nil, &block) + if block_given? + self.priority = block + else + self.priority = priority + end + end + end + + included do + class_attribute :priority, instance_accessor: false + + self.priority = default_priority + end + + # Returns the priority that the job will be created with + def priority + if @priority.is_a?(Proc) + @priority = instance_exec(&@priority) + end + @priority + end + + end +end diff --git a/activejob/test/cases/logging_test.rb b/activejob/test/cases/logging_test.rb index b18be553ec..820e9112de 100644 --- a/activejob/test/cases/logging_test.rb +++ b/activejob/test/cases/logging_test.rb @@ -74,6 +74,14 @@ class LoggingTest < ActiveSupport::TestCase assert_match(%r{Performing.*gid://aj/Person/123}, @logger.messages) end + def test_globalid_nested_parameter_logging + person = Person.new(123) + LoggingJob.perform_later(person: person) + assert_match(%r{Enqueued.*gid://aj/Person/123}, @logger.messages) + assert_match(%r{Dummy, here is it: .*#<Person:.*>}, @logger.messages) + assert_match(%r{Performing.*gid://aj/Person/123}, @logger.messages) + end + def test_enqueue_job_logging HelloJob.perform_later "Cristian" assert_match(/Enqueued HelloJob \(Job ID: .*?\) to .*?:.*Cristian/, @logger.messages) diff --git a/activejob/test/cases/queue_priority_test.rb b/activejob/test/cases/queue_priority_test.rb new file mode 100644 index 0000000000..ca17b51dad --- /dev/null +++ b/activejob/test/cases/queue_priority_test.rb @@ -0,0 +1,47 @@ +require 'helper' +require 'jobs/hello_job' + +class QueuePriorityTest < ActiveSupport::TestCase + test 'priority unset by default' do + assert_equal nil, HelloJob.priority + end + + test 'uses given priority' do + original_priority = HelloJob.priority + + begin + HelloJob.queue_with_priority 90 + assert_equal 90, HelloJob.new.priority + ensure + HelloJob.priority = original_priority + end + end + + test 'evals block given to priority to determine priority' do + original_priority = HelloJob.priority + + begin + HelloJob.queue_with_priority { 25 } + assert_equal 25, HelloJob.new.priority + ensure + HelloJob.priority = original_priority + end + end + + test 'can use arguments to determine priority in priority block' do + original_priority = HelloJob.priority + + begin + HelloJob.queue_with_priority { self.arguments.first=='1' ? 99 : 11 } + assert_equal 99, HelloJob.new('1').priority + assert_equal 11, HelloJob.new('3').priority + ensure + HelloJob.priority = original_priority + end + end + + test 'uses priority passed to #set' do + job = HelloJob.set(priority: 123).perform_later + assert_equal 123, job.priority + end +end diff --git a/activejob/test/cases/test_case_test.rb b/activejob/test/cases/test_case_test.rb index ee816e1dd5..616454a4b6 100644 --- a/activejob/test/cases/test_case_test.rb +++ b/activejob/test/cases/test_case_test.rb @@ -9,7 +9,7 @@ class ActiveJobTestCaseTest < ActiveJob::TestCase # the `class_attribute` inheritance class TestClassAttributeInheritanceJob < ActiveJob::Base def self.queue_adapter=(*) - raise 'Attemping to break `class_attribute` inheritance, bad!' + raise 'Attempting to break `class_attribute` inheritance, bad!' end end diff --git a/activejob/test/helper.rb b/activejob/test/helper.rb index 57907042d9..7e86415f48 100644 --- a/activejob/test/helper.rb +++ b/activejob/test/helper.rb @@ -3,6 +3,7 @@ require File.expand_path('../../../load_paths', __FILE__) require 'active_job' require 'support/job_buffer' +ActiveSupport.halt_callback_chains_on_return_false = false GlobalID.app = 'aj' @adapter = ENV['AJ_ADAPTER'] || 'inline' @@ -10,6 +11,7 @@ GlobalID.app = 'aj' if ENV['AJ_INTEGRATION_TESTS'] require 'support/integration/helper' else + ActiveJob::Base.logger = Logger.new(nil) require "adapters/#{@adapter}" end diff --git a/activejob/test/integration/queuing_test.rb b/activejob/test/integration/queuing_test.rb index 125ba54302..e435ed4aa6 100644 --- a/activejob/test/integration/queuing_test.rb +++ b/activejob/test/integration/queuing_test.rb @@ -84,4 +84,16 @@ class QueuingTest < ActiveSupport::TestCase I18n.locale = :en end end + + test 'should run job with higher priority first' do + skip unless adapter_is?(:delayed_job, :que) + + wait_until = Time.now + 3.seconds + TestJob.set(wait_until: wait_until, priority: 20).perform_later "#{@id}.1" + TestJob.set(wait_until: wait_until, priority: 10).perform_later "#{@id}.2" + wait_for_jobs_to_finish_for(10.seconds) + assert job_executed "#{@id}.1" + assert job_executed "#{@id}.2" + assert job_executed_at("#{@id}.2") < job_executed_at("#{@id}.1") + end end diff --git a/activejob/test/support/integration/adapters/sidekiq.rb b/activejob/test/support/integration/adapters/sidekiq.rb index d9938600f2..9aa07bcb52 100644 --- a/activejob/test/support/integration/adapters/sidekiq.rb +++ b/activejob/test/support/integration/adapters/sidekiq.rb @@ -58,7 +58,7 @@ module SidekiqJobsManager timeout: 1, }) Sidekiq.average_scheduled_poll_interval = 0.5 - Sidekiq::Scheduled.const_set :INITIAL_WAIT, 1 + Sidekiq.options[:poll_interval_average] = 1 begin sidekiq.run continue_write.puts "started" diff --git a/activejob/test/support/integration/dummy_app_template.rb b/activejob/test/support/integration/dummy_app_template.rb index 4ffdb8cffa..0c062a025e 100644 --- a/activejob/test/support/integration/dummy_app_template.rb +++ b/activejob/test/support/integration/dummy_app_template.rb @@ -1,8 +1,9 @@ if ENV['AJ_ADAPTER'] == 'delayed_job' generate "delayed_job:active_record", "--quiet" - rake("db:migrate") end +rake("db:migrate") + initializer 'activejob.rb', <<-CODE require "#{File.expand_path("../jobs_manager.rb", __FILE__)}" JobsManager.current_manager.setup diff --git a/activejob/test/support/integration/helper.rb b/activejob/test/support/integration/helper.rb index 8c2e5a86c2..4a1b0bfbcb 100644 --- a/activejob/test/support/integration/helper.rb +++ b/activejob/test/support/integration/helper.rb @@ -1,4 +1,4 @@ -puts "*** rake aj:integration:#{ENV['AJ_ADAPTER']} ***\n" +puts "\n\n*** rake aj:integration:#{ENV['AJ_ADAPTER']} ***\n" ENV["RAILS_ENV"] = "test" ActiveJob::Base.queue_name_prefix = nil diff --git a/activejob/test/support/integration/test_case_helpers.rb b/activejob/test/support/integration/test_case_helpers.rb index 39aee6d407..8319d09520 100644 --- a/activejob/test/support/integration/test_case_helpers.rb +++ b/activejob/test/support/integration/test_case_helpers.rb @@ -42,8 +42,12 @@ module TestCaseHelpers end end - def job_executed - Dummy::Application.root.join("tmp/#{@id}").exist? + def job_executed(id=@id) + Dummy::Application.root.join("tmp/#{id}").exist? + end + + def job_executed_at(id=@id) + File.new(Dummy::Application.root.join("tmp/#{id}")).ctime end def job_output diff --git a/activejob/test/support/que/inline.rb b/activejob/test/support/que/inline.rb index 0232da1370..0950e52d28 100644 --- a/activejob/test/support/que/inline.rb +++ b/activejob/test/support/que/inline.rb @@ -6,6 +6,7 @@ Que::Job.class_eval do if args.last.is_a?(Hash) options = args.pop options.delete(:run_at) + options.delete(:priority) args << options unless options.empty? end self.run(*args) |