diff options
15 files changed, 377 insertions, 38 deletions
diff --git a/actionpack/test/dispatch/request/json_params_parsing_test.rb b/actionpack/test/dispatch/request/json_params_parsing_test.rb index c2300a0142..28ebaed663 100644 --- a/actionpack/test/dispatch/request/json_params_parsing_test.rb +++ b/actionpack/test/dispatch/request/json_params_parsing_test.rb @@ -37,6 +37,13 @@ class JsonParamsParsingTest < ActionDispatch::IntegrationTest ) end + test "parses json params for application/vnd.api+json" do + assert_parses( + {"person" => {"name" => "David"}}, + "{\"person\": {\"name\": \"David\"}}", { 'CONTENT_TYPE' => 'application/vnd.api+json' } + ) + end + test "nils are stripped from collections" do assert_parses( {"person" => []}, @@ -136,6 +143,13 @@ class RootLessJSONParamsParsingTest < ActionDispatch::IntegrationTest ) end + test "parses json params for application/vnd.api+json" do + assert_parses( + {"user" => {"username" => "sikachu"}, "username" => "sikachu"}, + "{\"username\": \"sikachu\"}", { 'CONTENT_TYPE' => 'application/vnd.api+json' } + ) + end + test "parses json with non-object JSON content" do assert_parses( {"user" => {"_json" => "string content" }, "_json" => "string content" }, diff --git a/activerecord/lib/active_record/connection_adapters/abstract_adapter.rb b/activerecord/lib/active_record/connection_adapters/abstract_adapter.rb index 402159ac13..f5b2e9fa9d 100644 --- a/activerecord/lib/active_record/connection_adapters/abstract_adapter.rb +++ b/activerecord/lib/active_record/connection_adapters/abstract_adapter.rb @@ -214,6 +214,11 @@ module ActiveRecord false end + # Does this adapter support application-enforced advisory locking? + def supports_advisory_locks? + false + end + # Should primary key values be selected from their corresponding # sequence before the insert statement? If true, next_sequence_value # is called before each insert to set the record's primary key. @@ -280,6 +285,20 @@ module ActiveRecord def enable_extension(name) end + # This is meant to be implemented by the adapters that support advisory + # locks + # + # Return true if we got the lock, otherwise false + def get_advisory_lock(key) # :nodoc: + end + + # This is meant to be implemented by the adapters that support advisory + # locks. + # + # Return true if we released the lock, otherwise false + def release_advisory_lock(key) # :nodoc: + end + # A list of extensions, to be filled in by adapters that support them. def extensions [] diff --git a/activerecord/lib/active_record/connection_adapters/abstract_mysql_adapter.rb b/activerecord/lib/active_record/connection_adapters/abstract_mysql_adapter.rb index 251acf1c83..b775c18c1b 100644 --- a/activerecord/lib/active_record/connection_adapters/abstract_mysql_adapter.rb +++ b/activerecord/lib/active_record/connection_adapters/abstract_mysql_adapter.rb @@ -220,6 +220,20 @@ module ActiveRecord version >= '5.6.4' end + # 5.0.0 definitely supports it, possibly supported by earlier versions but + # not sure + def supports_advisory_locks? + version >= '5.0.0' + end + + def get_advisory_lock(key, timeout = 0) # :nodoc: + select_value("SELECT GET_LOCK('#{key}', #{timeout});").to_s == '1' + end + + def release_advisory_lock(key) # :nodoc: + select_value("SELECT RELEASE_LOCK('#{key}')").to_s == '1' + end + def native_database_types NATIVE_DATABASE_TYPES end diff --git a/activerecord/lib/active_record/connection_adapters/postgresql_adapter.rb b/activerecord/lib/active_record/connection_adapters/postgresql_adapter.rb index 787dadfdbf..6200cc8d29 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql_adapter.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql_adapter.rb @@ -284,6 +284,10 @@ module ActiveRecord true end + def supports_advisory_locks? + true + end + def supports_explain? true end @@ -302,6 +306,20 @@ module ActiveRecord postgresql_version >= 90300 end + def get_advisory_lock(key) # :nodoc: + unless key.is_a?(Integer) && key.bit_length <= 63 + raise(ArgumentError, "Postgres requires advisory lock keys to be a signed 64 bit integer") + end + select_value("SELECT pg_try_advisory_lock(#{key});") + end + + def release_advisory_lock(key) # :nodoc: + unless key.is_a?(Integer) && key.bit_length <= 63 + raise(ArgumentError, "Postgres requires advisory lock keys to be a signed 64 bit integer") + end + select_value("SELECT pg_advisory_unlock(#{key})") + end + def enable_extension(name) exec_query("CREATE EXTENSION IF NOT EXISTS \"#{name}\"").tap { reload_type_map diff --git a/activerecord/lib/active_record/migration.rb b/activerecord/lib/active_record/migration.rb index c8b96b8de0..63ec8f6745 100644 --- a/activerecord/lib/active_record/migration.rb +++ b/activerecord/lib/active_record/migration.rb @@ -135,6 +135,14 @@ module ActiveRecord end end + class ConcurrentMigrationError < MigrationError #:nodoc: + DEFAULT_MESSAGE = "Cannot run migrations because another migration process is currently running.".freeze + + def initialize(message = DEFAULT_MESSAGE) + super + end + end + # = Active Record Migrations # # Migrations can manage the evolution of a schema used by several physical @@ -1042,32 +1050,18 @@ module ActiveRecord alias :current :current_migration def run - migration = migrations.detect { |m| m.version == @target_version } - raise UnknownMigrationVersionError.new(@target_version) if migration.nil? - unless (up? && migrated.include?(migration.version.to_i)) || (down? && !migrated.include?(migration.version.to_i)) - begin - execute_migration_in_transaction(migration, @direction) - rescue => e - canceled_msg = use_transaction?(migration) ? ", this migration was canceled" : "" - raise StandardError, "An error has occurred#{canceled_msg}:\n\n#{e}", e.backtrace - end + if use_advisory_lock? + with_advisory_lock { run_without_lock } + else + run_without_lock end end def migrate - if !target && @target_version && @target_version > 0 - raise UnknownMigrationVersionError.new(@target_version) - end - - runnable.each do |migration| - Base.logger.info "Migrating to #{migration.name} (#{migration.version})" if Base.logger - - begin - execute_migration_in_transaction(migration, @direction) - rescue => e - canceled_msg = use_transaction?(migration) ? "this and " : "" - raise StandardError, "An error has occurred, #{canceled_msg}all later migrations canceled:\n\n#{e}", e.backtrace - end + if use_advisory_lock? + with_advisory_lock { migrate_without_lock } + else + migrate_without_lock end end @@ -1092,10 +1086,45 @@ module ActiveRecord end def migrated - @migrated_versions ||= Set.new(self.class.get_all_versions) + @migrated_versions || load_migrated + end + + def load_migrated + @migrated_versions = Set.new(self.class.get_all_versions) end private + + def run_without_lock + migration = migrations.detect { |m| m.version == @target_version } + raise UnknownMigrationVersionError.new(@target_version) if migration.nil? + unless (up? && migrated.include?(migration.version.to_i)) || (down? && !migrated.include?(migration.version.to_i)) + begin + execute_migration_in_transaction(migration, @direction) + rescue => e + canceled_msg = use_transaction?(migration) ? ", this migration was canceled" : "" + raise StandardError, "An error has occurred#{canceled_msg}:\n\n#{e}", e.backtrace + end + end + end + + def migrate_without_lock + if !target && @target_version && @target_version > 0 + raise UnknownMigrationVersionError.new(@target_version) + end + + runnable.each do |migration| + Base.logger.info "Migrating to #{migration.name} (#{migration.version})" if Base.logger + + begin + execute_migration_in_transaction(migration, @direction) + rescue => e + canceled_msg = use_transaction?(migration) ? "this and " : "" + raise StandardError, "An error has occurred, #{canceled_msg}all later migrations canceled:\n\n#{e}", e.backtrace + end + end + end + def ran?(migration) migrated.include?(migration.version.to_i) end @@ -1157,5 +1186,25 @@ module ActiveRecord def use_transaction?(migration) !migration.disable_ddl_transaction && Base.connection.supports_ddl_transactions? end + + def use_advisory_lock? + Base.connection.supports_advisory_locks? + end + + def with_advisory_lock + key = generate_migrator_advisory_lock_key + got_lock = Base.connection.get_advisory_lock(key) + raise ConcurrentMigrationError unless got_lock + load_migrated # reload schema_migrations to be sure it wasn't changed by another process before we got the lock + yield + ensure + Base.connection.release_advisory_lock(key) if got_lock + end + + MIGRATOR_SALT = 2053462845 + def generate_migrator_advisory_lock_key + db_name_hash = Zlib.crc32(Base.connection.current_database) + MIGRATOR_SALT * db_name_hash + end end end diff --git a/activerecord/test/cases/adapters/mysql/connection_test.rb b/activerecord/test/cases/adapters/mysql/connection_test.rb index decac9e83b..75653ee9af 100644 --- a/activerecord/test/cases/adapters/mysql/connection_test.rb +++ b/activerecord/test/cases/adapters/mysql/connection_test.rb @@ -170,6 +170,34 @@ class MysqlConnectionTest < ActiveRecord::MysqlTestCase end end + def test_get_and_release_advisory_lock + key = "test_key" + + got_lock = @connection.get_advisory_lock(key) + assert got_lock, "get_advisory_lock should have returned true but it didn't" + + assert_equal test_lock_free(key), false, + "expected the test advisory lock to be held but it wasn't" + + released_lock = @connection.release_advisory_lock(key) + assert released_lock, "expected release_advisory_lock to return true but it didn't" + + assert test_lock_free(key), 'expected the test key to be available after releasing' + end + + def test_release_non_existent_advisory_lock + fake_key = "fake_key" + released_non_existent_lock = @connection.release_advisory_lock(fake_key) + assert_equal released_non_existent_lock, false, + 'expected release_advisory_lock to return false when there was no lock to release' + end + + protected + + def test_lock_free(key) + @connection.select_value("SELECT IS_FREE_LOCK('#{key}');") == '1' + end + private def with_example_table(&block) diff --git a/activerecord/test/cases/adapters/mysql2/connection_test.rb b/activerecord/test/cases/adapters/mysql2/connection_test.rb index 000bcadebe..71c4028675 100644 --- a/activerecord/test/cases/adapters/mysql2/connection_test.rb +++ b/activerecord/test/cases/adapters/mysql2/connection_test.rb @@ -131,4 +131,32 @@ class Mysql2ConnectionTest < ActiveRecord::Mysql2TestCase ensure @connection.execute "DROP TABLE `bar_baz`" end + + def test_get_and_release_advisory_lock + key = "test_key" + + got_lock = @connection.get_advisory_lock(key) + assert got_lock, "get_advisory_lock should have returned true but it didn't" + + assert_equal test_lock_free(key), false, + "expected the test advisory lock to be held but it wasn't" + + released_lock = @connection.release_advisory_lock(key) + assert released_lock, "expected release_advisory_lock to return true but it didn't" + + assert test_lock_free(key), 'expected the test key to be available after releasing' + end + + def test_release_non_existent_advisory_lock + fake_key = "fake_key" + released_non_existent_lock = @connection.release_advisory_lock(fake_key) + assert_equal released_non_existent_lock, false, + 'expected release_advisory_lock to return false when there was no lock to release' + end + + protected + + def test_lock_free(key) + @connection.select_value("SELECT IS_FREE_LOCK('#{key}');") == 1 + end end diff --git a/activerecord/test/cases/adapters/postgresql/connection_test.rb b/activerecord/test/cases/adapters/postgresql/connection_test.rb index 722e2377c1..b12beb91de 100644 --- a/activerecord/test/cases/adapters/postgresql/connection_test.rb +++ b/activerecord/test/cases/adapters/postgresql/connection_test.rb @@ -209,5 +209,47 @@ module ActiveRecord ActiveRecord::Base.establish_connection(orig_connection.deep_merge({:variables => {:debug_print_plan => :default}})) end end + + def test_get_and_release_advisory_lock + key = 5295901941911233559 + list_advisory_locks = <<-SQL + SELECT locktype, + (classid::bigint << 32) | objid::bigint AS lock_key + FROM pg_locks + WHERE locktype = 'advisory' + SQL + + got_lock = @connection.get_advisory_lock(key) + assert got_lock, "get_advisory_lock should have returned true but it didn't" + + advisory_lock = @connection.query(list_advisory_locks).find {|l| l[1] == key} + assert advisory_lock, + "expected to find an advisory lock with key #{key} but there wasn't one" + + released_lock = @connection.release_advisory_lock(key) + assert released_lock, "expected release_advisory_lock to return true but it didn't" + + advisory_locks = @connection.query(list_advisory_locks).select {|l| l[1] == key} + assert_empty advisory_locks, + "expected to have released advisory lock with key #{key} but it was still held" + end + + def test_release_non_existent_advisory_lock + fake_key = 2940075057017742022 + with_warning_suppression do + released_non_existent_lock = @connection.release_advisory_lock(fake_key) + assert_equal released_non_existent_lock, false, + 'expected release_advisory_lock to return false when there was no lock to release' + end + end + + protected + + def with_warning_suppression + log_level = @connection.client_min_messages + @connection.client_min_messages = 'error' + yield + @connection.client_min_messages = log_level + end end end diff --git a/activerecord/test/cases/associations/left_outer_join_association_test.rb b/activerecord/test/cases/associations/left_outer_join_association_test.rb index 847292d5a0..4af791b758 100644 --- a/activerecord/test/cases/associations/left_outer_join_association_test.rb +++ b/activerecord/test/cases/associations/left_outer_join_association_test.rb @@ -16,11 +16,11 @@ class LeftOuterJoinAssociationTest < ActiveRecord::TestCase def test_construct_finder_sql_does_not_table_name_collide_on_duplicate_associations assert_nothing_raised do - sql = capture_sql do + queries = capture_sql do Person.left_outer_joins(:agents => {:agents => :agents}) .left_outer_joins(:agents => {:agents => {:primary_contact => :agents}}).to_a - end.first - assert_match(/agents_people_4/i, sql) + end + assert queries.any? { |sql| /agents_people_4/i =~ sql } end end @@ -35,13 +35,13 @@ class LeftOuterJoinAssociationTest < ActiveRecord::TestCase end def test_construct_finder_sql_ignores_empty_left_outer_joins_hash - sql = capture_sql { Author.left_outer_joins({}) }.first - assert_no_match(/LEFT OUTER JOIN/i, sql) + queries = capture_sql { Author.left_outer_joins({}) } + assert queries.none? { |sql| /LEFT OUTER JOIN/i =~ sql } end def test_construct_finder_sql_ignores_empty_left_outer_joins_array - sql = capture_sql { Author.left_outer_joins([]) }.first - assert_no_match(/LEFT OUTER JOIN/i, sql) + queries = capture_sql { Author.left_outer_joins([]) } + assert queries.none? { |sql| /LEFT OUTER JOIN/i =~ sql } end def test_left_outer_joins_forbids_to_use_string_as_argument @@ -49,9 +49,9 @@ class LeftOuterJoinAssociationTest < ActiveRecord::TestCase end def test_join_conditions_added_to_join_clause - sql = capture_sql { Author.left_outer_joins(:essays).to_a }.first - assert_match(/writer_type.*?=.*?(Author|\?|\$1)/i, sql) - assert_no_match(/WHERE/i, sql) + queries = capture_sql { Author.left_outer_joins(:essays).to_a } + assert queries.any? { |sql| /writer_type.*?=.*?(Author|\?|\$1)/i =~ sql } + assert queries.none? { |sql| /WHERE/i =~ sql } end def test_find_with_sti_join diff --git a/activerecord/test/cases/migration_test.rb b/activerecord/test/cases/migration_test.rb index 10f1c7216f..741bec6017 100644 --- a/activerecord/test/cases/migration_test.rb +++ b/activerecord/test/cases/migration_test.rb @@ -522,6 +522,79 @@ class MigrationTest < ActiveRecord::TestCase end end + if ActiveRecord::Base.connection.supports_advisory_locks? + def test_migrator_generates_valid_lock_key + migration = Class.new(ActiveRecord::Migration).new + migrator = ActiveRecord::Migrator.new(:up, [migration], 100) + + lock_key = migrator.send(:generate_migrator_advisory_lock_key) + + assert ActiveRecord::Base.connection.get_advisory_lock(lock_key), + "the Migrator should have generated a valid lock key, but it didn't" + assert ActiveRecord::Base.connection.release_advisory_lock(lock_key), + "the Migrator should have generated a valid lock key, but it didn't" + end + + def test_generate_migrator_advisory_lock_key + # It is important we are consistent with how we generate this so that + # exclusive locking works across migrator versions + migration = Class.new(ActiveRecord::Migration).new + migrator = ActiveRecord::Migrator.new(:up, [migration], 100) + + lock_key = migrator.send(:generate_migrator_advisory_lock_key) + + current_database = ActiveRecord::Base.connection.current_database + salt = ActiveRecord::Migrator::MIGRATOR_SALT + expected_key = Zlib.crc32(current_database) * salt + + assert lock_key == expected_key, "expected lock key generated by the migrator to be #{expected_key}, but it was #{lock_key} instead" + assert lock_key.is_a?(Fixnum), "expected lock key to be a Fixnum, but it wasn't" + assert lock_key.bit_length <= 63, "lock key must be a signed integer of max 63 bits magnitude" + end + + def test_migrator_one_up_with_unavailable_lock + assert_no_column Person, :last_name + + migration = Class.new(ActiveRecord::Migration) { + def version; 100 end + def migrate(x) + add_column "people", "last_name", :string + end + }.new + + migrator = ActiveRecord::Migrator.new(:up, [migration], 100) + lock_key = migrator.send(:generate_migrator_advisory_lock_key) + + with_another_process_holding_lock(lock_key) do + assert_raise(ActiveRecord::ConcurrentMigrationError) { migrator.migrate } + end + + assert_no_column Person, :last_name, + "without an advisory lock, the Migrator should not make any changes, but it did." + end + + def test_migrator_one_up_with_unavailable_lock_using_run + assert_no_column Person, :last_name + + migration = Class.new(ActiveRecord::Migration) { + def version; 100 end + def migrate(x) + add_column "people", "last_name", :string + end + }.new + + migrator = ActiveRecord::Migrator.new(:up, [migration], 100) + lock_key = migrator.send(:generate_migrator_advisory_lock_key) + + with_another_process_holding_lock(lock_key) do + assert_raise(ActiveRecord::ConcurrentMigrationError) { migrator.run } + end + + assert_no_column Person, :last_name, + "without an advisory lock, the Migrator should not make any changes, but it did." + end + end + protected # This is needed to isolate class_attribute assignments like `table_name_prefix` # for each test case. @@ -531,6 +604,30 @@ class MigrationTest < ActiveRecord::TestCase def self.base_class; self; end } end + + def with_another_process_holding_lock(lock_key) + thread_lock = Concurrent::CountDownLatch.new + test_terminated = Concurrent::CountDownLatch.new + + other_process = Thread.new do + begin + conn = ActiveRecord::Base.connection_pool.checkout + conn.get_advisory_lock(lock_key) + thread_lock.count_down + test_terminated.wait # hold the lock open until we tested everything + ensure + conn.release_advisory_lock(lock_key) + ActiveRecord::Base.connection_pool.checkin(conn) + end + end + + thread_lock.wait # wait until the 'other process' has the lock + + yield + + test_terminated.count_down + other_process.join + end end class ReservedWordsMigrationTest < ActiveRecord::TestCase diff --git a/activesupport/test/core_ext/hash_ext_test.rb b/activesupport/test/core_ext/hash_ext_test.rb index 265416dce7..2119352df0 100644 --- a/activesupport/test/core_ext/hash_ext_test.rb +++ b/activesupport/test/core_ext/hash_ext_test.rb @@ -1613,7 +1613,6 @@ class HashToXmlTest < ActiveSupport::TestCase assert_not_same hash_wia, hash_wia.with_indifferent_access end - def test_allows_setting_frozen_array_values_with_indifferent_access value = [1, 2, 3].freeze hash = HashWithIndifferentAccess.new diff --git a/railties/CHANGELOG.md b/railties/CHANGELOG.md index f43b73cb9d..967f850ea0 100644 --- a/railties/CHANGELOG.md +++ b/railties/CHANGELOG.md @@ -1,3 +1,7 @@ +* Allow passing an environment to `config_for`. + + *Simon Eskildsen* + * Allow rake:stats to account for rake tasks in lib/tasks *Kevin Deisz* diff --git a/railties/lib/rails/application.rb b/railties/lib/rails/application.rb index e81ec62a1d..77efe3248d 100644 --- a/railties/lib/rails/application.rb +++ b/railties/lib/rails/application.rb @@ -218,12 +218,12 @@ module Rails # Rails.application.configure do # config.middleware.use ExceptionNotifier, config_for(:exception_notification) # end - def config_for(name) + def config_for(name, env: Rails.env) yaml = Pathname.new("#{paths["config"].existent.first}/#{name}.yml") if yaml.exist? require "erb" - (YAML.load(ERB.new(yaml.read).result) || {})[Rails.env] || {} + (YAML.load(ERB.new(yaml.read).result) || {})[env] || {} else raise "Could not load configuration. No such file - #{yaml}" end diff --git a/railties/test/application/configuration_test.rb b/railties/test/application/configuration_test.rb index ebcfcb1c3a..22b615023d 100644 --- a/railties/test/application/configuration_test.rb +++ b/railties/test/application/configuration_test.rb @@ -1367,5 +1367,21 @@ module ApplicationTests assert_match 'YAML syntax error occurred while parsing', exception.message end + + test "config_for allows overriding the environment" do + app_file 'config/custom.yml', <<-RUBY + test: + key: 'walrus' + production: + key: 'unicorn' + RUBY + + add_to_config <<-RUBY + config.my_custom_config = config_for('custom', env: 'production') + RUBY + require "#{app_path}/config/environment" + + assert_equal 'unicorn', Rails.application.config.my_custom_config['key'] + end end end diff --git a/tasks/release.rb b/tasks/release.rb index 2c7e927679..4711974b63 100644 --- a/tasks/release.rb +++ b/tasks/release.rb @@ -66,13 +66,24 @@ directory "pkg" end namespace :changelog do + task :header do + (FRAMEWORKS + ['guides']).each do |fw| + require 'date' + fname = File.join fw, 'CHANGELOG.md' + + header = "## Rails #{version} (#{Date.today.strftime('%B %d, %Y')}) ##\n\n* No changes.\n\n\n" + contents = header + File.read(fname) + File.open(fname, 'wb') { |f| f.write contents } + end + end + task :release_date do (FRAMEWORKS + ['guides']).each do |fw| require 'date' - replace = '\1(' + Date.today.strftime('%B %d, %Y') + ')' + replace = "## Rails #{version} (#{Date.today.strftime('%B %d, %Y')}) ##\n" fname = File.join fw, 'CHANGELOG.md' - contents = File.read(fname).sub(/^([^(]*)\(unreleased\)/, replace) + contents = File.read(fname).sub(/^(## Rails .*)\n/, replace) File.open(fname, 'wb') { |f| f.write contents } end end |