diff options
Diffstat (limited to 'activejob')
19 files changed, 224 insertions, 63 deletions
diff --git a/activejob/CHANGELOG.md b/activejob/CHANGELOG.md index d4e19274fa..1c55c1a4b8 100644 --- a/activejob/CHANGELOG.md +++ b/activejob/CHANGELOG.md @@ -1,3 +1,13 @@ +* `assert_enqueued_jobs` and `assert_performed_jobs` in block form use the + given number as expected value. This makes the error message much easier to + understand. + + *y-yagi* + +* A generated job now inherents from `app/jobs/application_job.rb` by default. + + *Jeroen van Baarsen* + * Add an `:only` option to `perform_enqueued_jobs` to filter jobs based on type. diff --git a/activejob/activejob.gemspec b/activejob/activejob.gemspec index ef8db3bcd3..24e38e495f 100644 --- a/activejob/activejob.gemspec +++ b/activejob/activejob.gemspec @@ -7,7 +7,7 @@ Gem::Specification.new do |s| s.summary = 'Job framework with pluggable queues.' s.description = 'Declare job classes that can be run by a variety of queueing backends.' - s.required_ruby_version = '>= 2.2.1' + s.required_ruby_version = '>= 2.2.2' s.license = 'MIT' diff --git a/activejob/lib/active_job/arguments.rb b/activejob/lib/active_job/arguments.rb index 622c37098e..cdb37879b6 100644 --- a/activejob/lib/active_job/arguments.rb +++ b/activejob/lib/active_job/arguments.rb @@ -5,6 +5,8 @@ module ActiveJob # # Wraps the original exception raised as +original_exception+. class DeserializationError < StandardError + # The original exception that was raised during deserialization of job + # arguments. attr_reader :original_exception def initialize(e) #:nodoc: @@ -24,6 +26,7 @@ module ActiveJob module Arguments extend self + # :nodoc: TYPE_WHITELIST = [ NilClass, Fixnum, Float, String, TrueClass, FalseClass, Bignum ] # Serializes a set of arguments. Whitelisted types are returned @@ -43,8 +46,11 @@ module ActiveJob end private + # :nodoc: GLOBALID_KEY = '_aj_globalid'.freeze + # :nodoc: SYMBOL_KEYS_KEY = '_aj_symbol_keys'.freeze + # :nodoc: WITH_INDIFFERENT_ACCESS_KEY = '_aj_hash_with_indifferent_access'.freeze private_constant :GLOBALID_KEY, :SYMBOL_KEYS_KEY, :WITH_INDIFFERENT_ACCESS_KEY @@ -113,6 +119,7 @@ module ActiveJob result end + # :nodoc: RESERVED_KEYS = [ GLOBALID_KEY, GLOBALID_KEY.to_sym, SYMBOL_KEYS_KEY, SYMBOL_KEYS_KEY.to_sym, diff --git a/activejob/lib/active_job/core.rb b/activejob/lib/active_job/core.rb index ddd7d1361c..acdfcdc791 100644 --- a/activejob/lib/active_job/core.rb +++ b/activejob/lib/active_job/core.rb @@ -1,4 +1,6 @@ module ActiveJob + # Provides general behavior that will be included into every Active Job + # object that inherits from ActiveJob::Base. module Core extend ActiveSupport::Concern diff --git a/activejob/lib/active_job/enqueuing.rb b/activejob/lib/active_job/enqueuing.rb index 430c17e1bf..98d92385dd 100644 --- a/activejob/lib/active_job/enqueuing.rb +++ b/activejob/lib/active_job/enqueuing.rb @@ -1,6 +1,7 @@ require 'active_job/arguments' module ActiveJob + # Provides behavior for enqueuing and retrying jobs. module Enqueuing extend ActiveSupport::Concern diff --git a/activejob/lib/active_job/queue_adapters.rb b/activejob/lib/active_job/queue_adapters.rb index b3d91dc562..bd69e525bb 100644 --- a/activejob/lib/active_job/queue_adapters.rb +++ b/activejob/lib/active_job/queue_adapters.rb @@ -27,13 +27,76 @@ module ActiveJob # | Sneakers | Yes | Yes | No | Queue | Queue | No | # | Sucker Punch | Yes | Yes | No | No | No | No | # | Active Job Inline | No | Yes | N/A | N/A | N/A | N/A | - # | Active Job | Yes | Yes | Yes | No | No | No | + # + # ==== Async + # + # Yes: The Queue Adapter runs the jobs in a separate or forked process. + # + # No: The job is run in the same process. + # + # ==== Queues + # + # Yes: Jobs may set which queue they are run in with queue_as or by using the set + # method. + # + # ==== Delayed + # + # Yes: The adapter will run the job in the future through perform_later. + # + # (Gem): An additional gem is required to use perform_later with this adapter. + # + # No: The adapter will run jobs at the next opportunity and cannot use perform_later. + # + # N/A: The adapter does not support queueing. # # NOTE: - # queue_classic does not support Job scheduling. However you can implement this - # yourself or you can use the queue_classic-later gem. See the documentation for - # ActiveJob::QueueAdapters::QueueClassicAdapter. + # queue_classic does not support job scheduling. + # However, you can use the queue_classic-later gem. + # See the documentation for ActiveJob::QueueAdapters::QueueClassicAdapter. + # + # ==== Priorities + # + # The order in which jobs are processed can be configured differently depending + # on the adapter. + # + # Job: Any class inheriting from the adapter may set the priority on the job + # object relative to other jobs. + # + # Queue: The adapter can set the priority for job queues, when setting a queue + # with Active Job this will be respected. + # + # Yes: Allows the priority to be set on the job object, at the queue level or + # as default configuration option. + # + # No: Does not allow the priority of jobs to be configured. + # + # N/A: The adapter does not support queueing, and therefore sorting them. + # + # ==== Timeout + # + # When a job will stop after the allotted time. + # + # Job: The timeout can be set for each instance of the job class. + # + # Queue: The timeout is set for all jobs on the queue. + # + # Global: The adapter is configured that all jobs have a maximum run time. + # + # N/A: This adapter does not run in a separate process, and therefore timeout + # is unsupported. + # + # ==== Retries + # + # Job: The number of retries can be set per instance of the job class. + # + # Yes: The Number of retries can be configured globally, for each instance or + # on the queue. This adapter may also present failed instances of the job class + # that can be restarted. + # + # Global: The adapter has a global number of retries. # + # N/A: The adapter does not run in a separate process, and therefore doesn't + # support retries. module QueueAdapters extend ActiveSupport::Autoload diff --git a/activejob/lib/active_job/queue_adapters/inline_adapter.rb b/activejob/lib/active_job/queue_adapters/inline_adapter.rb index 1d06324c18..8ad5f4de07 100644 --- a/activejob/lib/active_job/queue_adapters/inline_adapter.rb +++ b/activejob/lib/active_job/queue_adapters/inline_adapter.rb @@ -14,7 +14,7 @@ module ActiveJob end def enqueue_at(*) #:nodoc: - raise NotImplementedError.new("Use a queueing backend to enqueue jobs in the future. Read more at http://guides.rubyonrails.org/active_job_basics.html") + raise NotImplementedError, "Use a queueing backend to enqueue jobs in the future. Read more at http://guides.rubyonrails.org/active_job_basics.html" end end end diff --git a/activejob/lib/active_job/queue_adapters/qu_adapter.rb b/activejob/lib/active_job/queue_adapters/qu_adapter.rb index 94584ef9d8..36f4c14911 100644 --- a/activejob/lib/active_job/queue_adapters/qu_adapter.rb +++ b/activejob/lib/active_job/queue_adapters/qu_adapter.rb @@ -23,7 +23,7 @@ module ActiveJob end def enqueue_at(job, timestamp, *args) #:nodoc: - raise NotImplementedError + raise NotImplementedError, "This queueing backend does not support scheduling jobs. To see what features are supported go to http://api.rubyonrails.org/classes/ActiveJob/QueueAdapters.html" end class JobWrapper < Qu::Job #:nodoc: diff --git a/activejob/lib/active_job/queue_adapters/sidekiq_adapter.rb b/activejob/lib/active_job/queue_adapters/sidekiq_adapter.rb index e02593ec9c..743d5ea333 100644 --- a/activejob/lib/active_job/queue_adapters/sidekiq_adapter.rb +++ b/activejob/lib/active_job/queue_adapters/sidekiq_adapter.rb @@ -18,17 +18,19 @@ module ActiveJob def enqueue(job) #:nodoc: #Sidekiq::Client does not support symbols as keys Sidekiq::Client.push \ - 'class' => JobWrapper, - 'queue' => job.queue_name, - 'args' => [ job.serialize ] + 'class' => JobWrapper, + 'wrapped' => job.class.to_s, + 'queue' => job.queue_name, + 'args' => [ job.serialize ] end def enqueue_at(job, timestamp) #:nodoc: Sidekiq::Client.push \ - 'class' => JobWrapper, - 'queue' => job.queue_name, - 'args' => [ job.serialize ], - 'at' => timestamp + 'class' => JobWrapper, + 'wrapped' => job.class.to_s, + 'queue' => job.queue_name, + 'args' => [ job.serialize ], + 'at' => timestamp end class JobWrapper #:nodoc: diff --git a/activejob/lib/active_job/queue_adapters/sneakers_adapter.rb b/activejob/lib/active_job/queue_adapters/sneakers_adapter.rb index f5737487ca..f102c6567e 100644 --- a/activejob/lib/active_job/queue_adapters/sneakers_adapter.rb +++ b/activejob/lib/active_job/queue_adapters/sneakers_adapter.rb @@ -28,7 +28,7 @@ module ActiveJob end def enqueue_at(job, timestamp) #:nodoc: - raise NotImplementedError + raise NotImplementedError, "This queueing backend does not support scheduling jobs. To see what features are supported go to http://api.rubyonrails.org/classes/ActiveJob/QueueAdapters.html" end class JobWrapper #:nodoc: diff --git a/activejob/lib/active_job/queue_adapters/sucker_punch_adapter.rb b/activejob/lib/active_job/queue_adapters/sucker_punch_adapter.rb index 64c93e8198..c6c35f8ab4 100644 --- a/activejob/lib/active_job/queue_adapters/sucker_punch_adapter.rb +++ b/activejob/lib/active_job/queue_adapters/sucker_punch_adapter.rb @@ -23,7 +23,7 @@ module ActiveJob end def enqueue_at(job, timestamp) #:nodoc: - raise NotImplementedError + raise NotImplementedError, "This queueing backend does not support scheduling jobs. To see what features are supported go to http://api.rubyonrails.org/classes/ActiveJob/QueueAdapters.html" end class JobWrapper #:nodoc: diff --git a/activejob/lib/active_job/test_helper.rb b/activejob/lib/active_job/test_helper.rb index 4efb4b72d2..9b307e8dc8 100644 --- a/activejob/lib/active_job/test_helper.rb +++ b/activejob/lib/active_job/test_helper.rb @@ -68,7 +68,7 @@ module ActiveJob original_count = enqueued_jobs_size(only: only) yield new_count = enqueued_jobs_size(only: only) - assert_equal original_count + number, new_count, "#{number} jobs expected, but #{new_count - original_count} were enqueued" + assert_equal number, new_count - original_count, "#{number} jobs expected, but #{new_count - original_count} were enqueued" else actual_count = enqueued_jobs_size(only: only) assert_equal number, actual_count, "#{number} jobs expected, but #{actual_count} were enqueued" @@ -164,7 +164,7 @@ module ActiveJob original_count = performed_jobs.size perform_enqueued_jobs(only: only) { yield } new_count = performed_jobs.size - assert_equal original_count + number, new_count, + assert_equal number, new_count - original_count, "#{number} jobs expected, but #{new_count - original_count} were performed" else performed_jobs_size = performed_jobs.size diff --git a/activejob/lib/rails/generators/job/job_generator.rb b/activejob/lib/rails/generators/job/job_generator.rb index 979ffcb748..2115fb9f71 100644 --- a/activejob/lib/rails/generators/job/job_generator.rb +++ b/activejob/lib/rails/generators/job/job_generator.rb @@ -1,6 +1,6 @@ require 'rails/generators/named_base' -module Rails +module Rails # :nodoc: module Generators # :nodoc: class JobGenerator < Rails::Generators::NamedBase # :nodoc: desc 'This generator creates an active job file at app/jobs' @@ -18,7 +18,6 @@ module Rails def create_job_file template 'job.rb', File.join('app/jobs', class_path, "#{file_name}_job.rb") end - end end end diff --git a/activejob/lib/rails/generators/job/templates/job.rb b/activejob/lib/rails/generators/job/templates/job.rb index 462c71d917..4ad2914a45 100644 --- a/activejob/lib/rails/generators/job/templates/job.rb +++ b/activejob/lib/rails/generators/job/templates/job.rb @@ -1,5 +1,5 @@ <% module_namespacing do -%> -class <%= class_name %>Job < ActiveJob::Base +class <%= class_name %>Job < ApplicationJob queue_as :<%= options[:queue] %> def perform(*args) diff --git a/activejob/test/cases/test_helper_test.rb b/activejob/test/cases/test_helper_test.rb index 19a2820a6e..04c4c446e2 100644 --- a/activejob/test/cases/test_helper_test.rb +++ b/activejob/test/cases/test_helper_test.rb @@ -31,6 +31,17 @@ class EnqueuedJobsTest < ActiveJob::TestCase end end + def test_assert_enqueued_jobs_message + HelloJob.perform_later('sean') + e = assert_raises Minitest::Assertion do + assert_enqueued_jobs 2 do + HelloJob.perform_later('sean') + end + end + assert_match "Expected: 2", e.message + assert_match "Actual: 1", e.message + end + def test_assert_enqueued_jobs_with_no_block assert_nothing_raised do HelloJob.perform_later('rafael') @@ -230,6 +241,17 @@ class PerformedJobsTest < ActiveJob::TestCase end end + def test_assert_performed_jobs_message + HelloJob.perform_later('sean') + e = assert_raises Minitest::Assertion do + assert_performed_jobs 2 do + HelloJob.perform_later('sean') + end + end + assert_match "Expected: 2", e.message + assert_match "Actual: 1", e.message + end + def test_assert_performed_jobs_with_no_block assert_nothing_raised do perform_enqueued_jobs do diff --git a/activejob/test/integration/queuing_test.rb b/activejob/test/integration/queuing_test.rb index 38874b51a8..09f5c329cc 100644 --- a/activejob/test/integration/queuing_test.rb +++ b/activejob/test/integration/queuing_test.rb @@ -1,5 +1,6 @@ require 'helper' require 'jobs/logging_job' +require 'jobs/hello_job' require 'active_support/core_ext/numeric/time' class QueuingTest < ActiveSupport::TestCase @@ -23,6 +24,16 @@ class QueuingTest < ActiveSupport::TestCase end end + test 'should supply a wrapped class name to Sidekiq' do + skip unless adapter_is?(:sidekiq) + Sidekiq::Testing.fake! do + ::HelloJob.perform_later + hash = ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper.jobs.first + assert_equal "ActiveJob::QueueAdapters::SidekiqAdapter::JobWrapper", hash['class'] + assert_equal "HelloJob", hash['wrapped'] + end + end + test 'should not run job enqueued in the future' do begin TestJob.set(wait: 10.minutes).perform_later @id @@ -35,7 +46,7 @@ class QueuingTest < ActiveSupport::TestCase test 'should run job enqueued in the future at the specified time' do begin - TestJob.set(wait: 3.seconds).perform_later @id + TestJob.set(wait: 5.seconds).perform_later @id wait_for_jobs_to_finish_for(2.seconds) assert_not job_executed wait_for_jobs_to_finish_for(10.seconds) diff --git a/activejob/test/support/integration/adapters/que.rb b/activejob/test/support/integration/adapters/que.rb index ba7657a42a..ff5235bdc8 100644 --- a/activejob/test/support/integration/adapters/que.rb +++ b/activejob/test/support/integration/adapters/que.rb @@ -2,6 +2,15 @@ module QueJobsManager def setup require 'sequel' ActiveJob::Base.queue_adapter = :que + Que.mode = :off + Que.worker_count = 1 + end + + def clear_jobs + Que.clear! + end + + def start_workers que_url = ENV['QUE_DATABASE_URL'] || 'postgres:///active_jobs_que_int_test' uri = URI.parse(que_url) user = uri.user||ENV['USER'] @@ -11,24 +20,17 @@ module QueJobsManager %x{#{"PGPASSWORD=\"#{pass}\"" if pass} psql -c 'create database "#{db}"' -U #{user} -t template1} Que.connection = Sequel.connect(que_url) Que.migrate! - Que.mode = :off - Que.worker_count = 1 - rescue Sequel::DatabaseConnectionError - puts "Cannot run integration tests for que. To be able to run integration tests for que you need to install and start postgresql.\n" - exit - end - - def clear_jobs - Que.clear! - end - def start_workers @thread = Thread.new do loop do Que::Job.work("integration_tests") sleep 0.5 end end + + rescue Sequel::DatabaseConnectionError + puts "Cannot run integration tests for que. To be able to run integration tests for que you need to install and start postgresql.\n" + exit end def stop_workers diff --git a/activejob/test/support/integration/adapters/queue_classic.rb b/activejob/test/support/integration/adapters/queue_classic.rb index f522b2711f..29c04bf625 100644 --- a/activejob/test/support/integration/adapters/queue_classic.rb +++ b/activejob/test/support/integration/adapters/queue_classic.rb @@ -3,17 +3,7 @@ module QueueClassicJobsManager ENV['QC_DATABASE_URL'] ||= 'postgres:///active_jobs_qc_int_test' ENV['QC_RAILS_DATABASE'] = 'false' ENV['QC_LISTEN_TIME'] = "0.5" - uri = URI.parse(ENV['QC_DATABASE_URL']) - user = uri.user||ENV['USER'] - pass = uri.password - db = uri.path[1..-1] - %x{#{"PGPASSWORD=\"#{pass}\"" if pass} psql -c 'drop database if exists "#{db}"' -U #{user} -t template1} - %x{#{"PGPASSWORD=\"#{pass}\"" if pass} psql -c 'create database "#{db}"' -U #{user} -t template1} ActiveJob::Base.queue_adapter = :queue_classic - QC::Setup.create - rescue PG::ConnectionBad - puts "Cannot run integration tests for queue_classic. To be able to run integration tests for queue_classic you need to install and start postgresql.\n" - exit end def clear_jobs @@ -21,12 +11,24 @@ module QueueClassicJobsManager end def start_workers + uri = URI.parse(ENV['QC_DATABASE_URL']) + user = uri.user||ENV['USER'] + pass = uri.password + db = uri.path[1..-1] + %x{#{"PGPASSWORD=\"#{pass}\"" if pass} psql -c 'drop database if exists "#{db}"' -U #{user} -t template1} + %x{#{"PGPASSWORD=\"#{pass}\"" if pass} psql -c 'create database "#{db}"' -U #{user} -t template1} + QC::Setup.create + QC.default_conn_adapter.disconnect QC.default_conn_adapter = nil @pid = fork do worker = QC::Worker.new(q_name: 'integration_tests') worker.start end + + rescue PG::ConnectionBad + puts "Cannot run integration tests for queue_classic. To be able to run integration tests for queue_classic you need to install and start postgresql.\n" + exit end def stop_workers diff --git a/activejob/test/support/integration/adapters/sidekiq.rb b/activejob/test/support/integration/adapters/sidekiq.rb index 6ff18fb56a..4988cdb33f 100644 --- a/activejob/test/support/integration/adapters/sidekiq.rb +++ b/activejob/test/support/integration/adapters/sidekiq.rb @@ -1,6 +1,8 @@ -require 'sidekiq/cli' require 'sidekiq/api' +require 'sidekiq/testing' +Sidekiq::Testing.disable! + module SidekiqJobsManager def setup @@ -17,33 +19,71 @@ module SidekiqJobsManager end def start_workers - fork do - sidekiq = Sidekiq::CLI.instance + continue_read, continue_write = IO.pipe + death_read, death_write = IO.pipe + + @pid = fork do + continue_read.close + death_write.close + + # Celluloid & Sidekiq are not warning-clean :( + $VERBOSE = false + + $stdin.reopen('/dev/null') + $stdout.sync = true + $stderr.sync = true + logfile = Rails.root.join("log/sidekiq.log").to_s - pidfile = Rails.root.join("tmp/sidekiq.pid").to_s - sidekiq.parse([ "--require", Rails.root.to_s, - "--queue", "integration_tests", - "--logfile", logfile, - "--pidfile", pidfile, - "--environment", "test", - "--concurrency", "1", - "--timeout", "1", - "--daemon", - ]) + Sidekiq::Logging.initialize_logger(logfile) + + self_read, self_write = IO.pipe + trap "TERM" do + self_write.puts("TERM") + end + + Thread.new do + begin + death_read.read + rescue Exception + end + self_write.puts("TERM") + end + require 'celluloid' - require 'sidekiq/scheduled' + Celluloid.logger = nil + require 'sidekiq/launcher' + sidekiq = Sidekiq::Launcher.new({queues: ["integration_tests"], + environment: "test", + concurrency: 1, + timeout: 1, + }) Sidekiq.poll_interval = 0.5 Sidekiq::Scheduled.const_set :INITIAL_WAIT, 1 - sidekiq.run + begin + sidekiq.run + continue_write.puts "started" + while readable_io = IO.select([self_read]) + signal = readable_io.first[0].gets.strip + raise Interrupt if signal == "TERM" + end + rescue Interrupt + end + + sidekiq.stop + exit! end - sleep 1 + continue_write.close + death_read.close + @worker_lifeline = death_write + + raise "Failed to start worker" unless continue_read.gets == "started\n" end def stop_workers - pidfile = Rails.root.join("tmp/sidekiq.pid").to_s - Process.kill 'TERM', File.open(pidfile).read.to_i - FileUtils.rm_f pidfile - rescue + if @pid + Process.kill 'TERM', @pid + Process.wait @pid + end end def can_run? |