diff options
26 files changed, 768 insertions, 87 deletions
diff --git a/.rubocop.yml b/.rubocop.yml index 618817daca..54dcaf80f9 100644 --- a/.rubocop.yml +++ b/.rubocop.yml @@ -179,6 +179,12 @@ Layout/TrailingWhitespace: Style/UnneededPercentQ: Enabled: true +Lint/AmbiguousOperator: + Enabled: true + +Lint/AmbiguousRegexpLiteral: + Enabled: true + Lint/ErbNewArguments: Enabled: true @@ -237,5 +243,8 @@ Performance/EndWith: Performance/RegexpMatch: Enabled: true +Performance/ReverseEach: + Enabled: true + Performance/UnfreezeString: Enabled: true diff --git a/activerecord/CHANGELOG.md b/activerecord/CHANGELOG.md index 04449a3ebc..cff6a3147b 100644 --- a/activerecord/CHANGELOG.md +++ b/activerecord/CHANGELOG.md @@ -1,3 +1,13 @@ +* Add `insert_all`/`insert_all!`/`upsert_all` methods to `ActiveRecord::Persistence`, + allowing bulk inserts akin to the bulk updates provided by `update_all` and + bulk deletes by `delete_all`. + + Supports skipping or upserting duplicates through the `ON CONFLICT` syntax + for Postgres (9.5+) and Sqlite (3.24+) and `ON DUPLICATE KEY UPDATE` syntax + for MySQL. + + *Bob Lail* + * Add `rails db:seed:replant` that truncates tables of each database for current environment and loads the seeds. diff --git a/activerecord/lib/active_record/connection_adapters/abstract/database_statements.rb b/activerecord/lib/active_record/connection_adapters/abstract/database_statements.rb index b5e6d03cf5..3c872d6c1b 100644 --- a/activerecord/lib/active_record/connection_adapters/abstract/database_statements.rb +++ b/activerecord/lib/active_record/connection_adapters/abstract/database_statements.rb @@ -404,6 +404,17 @@ module ActiveRecord end end + # Fixture value is quoted by Arel, however scalar values + # are not quotable. In this case we want to convert + # the column value to YAML. + def with_yaml_fallback(value) # :nodoc: + if value.is_a?(Hash) || value.is_a?(Array) + YAML.dump(value) + else + value + end + end + private def default_insert_value(column) Arel.sql("DEFAULT") @@ -473,17 +484,6 @@ module ActiveRecord relation end end - - # Fixture value is quoted by Arel, however scalar values - # are not quotable. In this case we want to convert - # the column value to YAML. - def with_yaml_fallback(value) - if value.is_a?(Hash) || value.is_a?(Array) - YAML.dump(value) - else - value - end - end end end end diff --git a/activerecord/lib/active_record/connection_adapters/abstract/schema_definitions.rb b/activerecord/lib/active_record/connection_adapters/abstract/schema_definitions.rb index 11d4b4a503..4861872129 100644 --- a/activerecord/lib/active_record/connection_adapters/abstract/schema_definitions.rb +++ b/activerecord/lib/active_record/connection_adapters/abstract/schema_definitions.rb @@ -105,13 +105,9 @@ module ActiveRecord !ActiveRecord::SchemaDumper.fk_ignore_pattern.match?(name) if name end - def defined_for?(to_table_ord = nil, to_table: nil, **options) - if to_table_ord - self.to_table == to_table_ord.to_s - else - (to_table.nil? || to_table.to_s == self.to_table) && - options.all? { |k, v| self.options[k].to_s == v.to_s } - end + def defined_for?(to_table: nil, **options) + (to_table.nil? || to_table.to_s == self.to_table) && + options.all? { |k, v| self.options[k].to_s == v.to_s } end private diff --git a/activerecord/lib/active_record/connection_adapters/abstract/schema_statements.rb b/activerecord/lib/active_record/connection_adapters/abstract/schema_statements.rb index b2cc60f363..3dc81ffccf 100644 --- a/activerecord/lib/active_record/connection_adapters/abstract/schema_statements.rb +++ b/activerecord/lib/active_record/connection_adapters/abstract/schema_statements.rb @@ -1002,10 +1002,10 @@ module ActiveRecord # with an addition of # [<tt>:to_table</tt>] # The name of the table that contains the referenced primary key. - def remove_foreign_key(from_table, options_or_to_table = {}) + def remove_foreign_key(from_table, to_table = nil, **options) return unless supports_foreign_keys? - fk_name_to_delete = foreign_key_for!(from_table, options_or_to_table).name + fk_name_to_delete = foreign_key_for!(from_table, to_table: to_table, **options).name at = create_alter_table from_table at.drop_foreign_key fk_name_to_delete @@ -1024,8 +1024,8 @@ module ActiveRecord # # Checks to see if a foreign key with a custom name exists. # foreign_key_exists?(:accounts, name: "special_fk_name") # - def foreign_key_exists?(from_table, options_or_to_table = {}) - foreign_key_for(from_table, options_or_to_table).present? + def foreign_key_exists?(from_table, to_table = nil, **options) + foreign_key_for(from_table, to_table: to_table, **options).present? end def foreign_key_column_for(table_name) # :nodoc: @@ -1341,14 +1341,14 @@ module ActiveRecord end end - def foreign_key_for(from_table, options_or_to_table = {}) + def foreign_key_for(from_table, **options) return unless supports_foreign_keys? - foreign_keys(from_table).detect { |fk| fk.defined_for? options_or_to_table } + foreign_keys(from_table).detect { |fk| fk.defined_for?(options) } end - def foreign_key_for!(from_table, options_or_to_table = {}) - foreign_key_for(from_table, options_or_to_table) || \ - raise(ArgumentError, "Table '#{from_table}' has no foreign key for #{options_or_to_table}") + def foreign_key_for!(from_table, to_table: nil, **options) + foreign_key_for(from_table, to_table: to_table, **options) || + raise(ArgumentError, "Table '#{from_table}' has no foreign key for #{to_table || options}") end def extract_foreign_key_action(specifier) diff --git a/activerecord/lib/active_record/connection_adapters/abstract_adapter.rb b/activerecord/lib/active_record/connection_adapters/abstract_adapter.rb index 823efee301..c2087b8216 100644 --- a/activerecord/lib/active_record/connection_adapters/abstract_adapter.rb +++ b/activerecord/lib/active_record/connection_adapters/abstract_adapter.rb @@ -120,6 +120,7 @@ module ActiveRecord @quoted_column_names, @quoted_table_names = {}, {} @prevent_writes = false @visitor = arel_visitor + @statements = build_statement_pool @lock = ActiveSupport::Concurrency::LoadInterlockAwareMonitor.new if self.class.type_cast_config_to_boolean(config.fetch(:prepared_statements) { true }) @@ -387,6 +388,22 @@ module ActiveRecord false end + def supports_insert_returning? + false + end + + def supports_insert_on_duplicate_skip? + false + end + + def supports_insert_on_duplicate_update? + false + end + + def supports_insert_conflict_target? + false + end + # This is meant to be implemented by the adapters that support extensions def disable_extension(name) end @@ -476,11 +493,9 @@ module ActiveRecord # this should be overridden by concrete adapters end - ### - # Clear any caching the database adapter may be doing, for example - # clearing the prepared statement cache. This is database specific. + # Clear any caching the database adapter may be doing. def clear_cache! - # this should be overridden by concrete adapters + @lock.synchronize { @statements.clear } if @statements end # Returns true if its required to reload the connection between requests for development mode. @@ -542,6 +557,19 @@ module ActiveRecord index.using.nil? end + # Called by ActiveRecord::InsertAll, + # Passed an instance of ActiveRecord::InsertAll::Builder, + # This method implements standard bulk inserts for all databases, but + # should be overridden by adapters to implement common features with + # non-standard syntax like handling duplicates or returning values. + def build_insert_sql(insert) # :nodoc: + if insert.skip_duplicates? || insert.update_duplicates? + raise NotImplementedError, "#{self.class} should define `build_insert_sql` to implement adapter-specific logic for handling duplicates during INSERT" + end + + "INSERT #{insert.into} #{insert.values_list}" + end + private def check_version end @@ -689,6 +717,9 @@ module ActiveRecord def arel_visitor Arel::Visitors::ToSql.new(self) end + + def build_statement_pool + end end end end diff --git a/activerecord/lib/active_record/connection_adapters/abstract_mysql_adapter.rb b/activerecord/lib/active_record/connection_adapters/abstract_mysql_adapter.rb index 96f902792e..d99c9b9d02 100644 --- a/activerecord/lib/active_record/connection_adapters/abstract_mysql_adapter.rb +++ b/activerecord/lib/active_record/connection_adapters/abstract_mysql_adapter.rb @@ -53,8 +53,6 @@ module ActiveRecord def initialize(connection, logger, connection_options, config) super(connection, logger, config) - - @statements = StatementPool.new(self.class.type_cast_config_to_integer(config[:statement_limit])) end def version #:nodoc: @@ -109,6 +107,14 @@ module ActiveRecord true end + def supports_insert_on_duplicate_skip? + true + end + + def supports_insert_on_duplicate_update? + true + end + def get_advisory_lock(lock_name, timeout = 0) # :nodoc: query_value("SELECT GET_LOCK(#{quote(lock_name.to_s)}, #{timeout})") == 1 end @@ -154,10 +160,9 @@ module ActiveRecord # CONNECTION MANAGEMENT ==================================== - # Clears the prepared statements cache. - def clear_cache! + def clear_cache! # :nodoc: reload_type_map - @statements.clear + super end #-- @@ -511,6 +516,20 @@ module ActiveRecord end end + def build_insert_sql(insert) # :nodoc: + sql = +"INSERT #{insert.into} #{insert.values_list}" + + if insert.skip_duplicates? + any_column = quote_column_name(insert.model.columns.first.name) + sql << " ON DUPLICATE KEY UPDATE #{any_column}=#{any_column}" + elsif insert.update_duplicates? + sql << " ON DUPLICATE KEY UPDATE " + sql << insert.updatable_columns.map { |column| "#{column}=VALUES(#{column})" }.join(",") + end + + sql + end + private def check_version if version < "5.5.8" @@ -784,6 +803,10 @@ module ActiveRecord Arel::Visitors::MySQL.new(self) end + def build_statement_pool + StatementPool.new(self.class.type_cast_config_to_integer(@config[:statement_limit])) + end + def mismatched_foreign_key(message, sql:, binds:) match = %r/ (?:CREATE|ALTER)\s+TABLE\s*(?:`?\w+`?\.)?`?(?<table>\w+)`?.+? diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/schema_statements.rb b/activerecord/lib/active_record/connection_adapters/postgresql/schema_statements.rb index d694a4f47d..a38c1325c0 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql/schema_statements.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql/schema_statements.rb @@ -623,10 +623,10 @@ module ActiveRecord # validate_foreign_key :accounts, name: :special_fk_name # # The +options+ hash accepts the same keys as SchemaStatements#add_foreign_key. - def validate_foreign_key(from_table, options_or_to_table = {}) + def validate_foreign_key(from_table, to_table = nil, **options) return unless supports_validate_constraints? - fk_name_to_validate = foreign_key_for!(from_table, options_or_to_table).name + fk_name_to_validate = foreign_key_for!(from_table, to_table: to_table, **options).name validate_constraint from_table, fk_name_to_validate end diff --git a/activerecord/lib/active_record/connection_adapters/postgresql_adapter.rb b/activerecord/lib/active_record/connection_adapters/postgresql_adapter.rb index 95beeb4cae..05d8d5ac00 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql_adapter.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql_adapter.rb @@ -196,6 +196,17 @@ module ActiveRecord true end + def supports_insert_returning? + true + end + + def supports_insert_on_conflict? + postgresql_version >= 90500 + end + alias supports_insert_on_duplicate_skip? supports_insert_on_conflict? + alias supports_insert_on_duplicate_update? supports_insert_on_conflict? + alias supports_insert_conflict_target? supports_insert_on_conflict? + def index_algorithms { concurrently: "CONCURRENTLY" } end @@ -242,9 +253,6 @@ module ActiveRecord configure_connection add_pg_encoders - @statements = StatementPool.new @connection, - self.class.type_cast_config_to_integer(config[:statement_limit]) - add_pg_decoders @type_map = Type::HashLookupTypeMap.new @@ -253,13 +261,6 @@ module ActiveRecord @use_insert_returning = @config.key?(:insert_returning) ? self.class.type_cast_config_to_boolean(@config[:insert_returning]) : true end - # Clears the prepared statements cache. - def clear_cache! - @lock.synchronize do - @statements.clear - end - end - def truncate(table_name, name = nil) exec_query "TRUNCATE TABLE #{quote_table_name(table_name)}", name, [] end @@ -425,6 +426,20 @@ module ActiveRecord index.using == :btree || super end + def build_insert_sql(insert) # :nodoc: + sql = +"INSERT #{insert.into} #{insert.values_list}" + + if insert.skip_duplicates? + sql << " ON CONFLICT #{insert.conflict_target} DO NOTHING" + elsif insert.update_duplicates? + sql << " ON CONFLICT #{insert.conflict_target} DO UPDATE SET " + sql << insert.updatable_columns.map { |column| "#{column}=excluded.#{column}" }.join(",") + end + + sql << " RETURNING #{insert.returning}" if insert.returning + sql + end + private def check_version if postgresql_version < 90300 @@ -803,6 +818,10 @@ module ActiveRecord Arel::Visitors::PostgreSQL.new(self) end + def build_statement_pool + StatementPool.new(@connection, self.class.type_cast_config_to_integer(@config[:statement_limit])) + end + def can_perform_case_insensitive_comparison_for?(column) @case_insensitive_cache ||= {} @case_insensitive_cache[column.sql_type] ||= begin diff --git a/activerecord/lib/active_record/connection_adapters/sqlite3/schema_statements.rb b/activerecord/lib/active_record/connection_adapters/sqlite3/schema_statements.rb index bd649451d6..e64e995e1a 100644 --- a/activerecord/lib/active_record/connection_adapters/sqlite3/schema_statements.rb +++ b/activerecord/lib/active_record/connection_adapters/sqlite3/schema_statements.rb @@ -72,7 +72,7 @@ module ActiveRecord table = strip_table_name_prefix_and_suffix(table) fk_to_table = strip_table_name_prefix_and_suffix(fk.to_table) fk_to_table == table && options.all? { |k, v| fk.options[k].to_s == v.to_s } - end || raise(ArgumentError, "Table '#{from_table}' has no foreign key for #{to_table}") + end || raise(ArgumentError, "Table '#{from_table}' has no foreign key for #{to_table || options}") foreign_keys.delete(fkey) alter_table(from_table, foreign_keys) diff --git a/activerecord/lib/active_record/connection_adapters/sqlite3_adapter.rb b/activerecord/lib/active_record/connection_adapters/sqlite3_adapter.rb index 63e07932d9..cb3d34a740 100644 --- a/activerecord/lib/active_record/connection_adapters/sqlite3_adapter.rb +++ b/activerecord/lib/active_record/connection_adapters/sqlite3_adapter.rb @@ -96,8 +96,7 @@ module ActiveRecord def initialize(connection, logger, connection_options, config) super(connection, logger, config) - @active = true - @statements = StatementPool.new(self.class.type_cast_config_to_integer(config[:statement_limit])) + @active = true configure_connection end @@ -137,6 +136,13 @@ module ActiveRecord true end + def supports_insert_on_conflict? + sqlite_version >= "3.24.0" + end + alias supports_insert_on_duplicate_skip? supports_insert_on_conflict? + alias supports_insert_on_duplicate_update? supports_insert_on_conflict? + alias supports_insert_conflict_target? supports_insert_on_conflict? + def active? @active end @@ -149,11 +155,6 @@ module ActiveRecord @connection.close rescue nil end - # Clears the prepared statements cache. - def clear_cache! - @statements.clear - end - def truncate(table_name, name = nil) execute "DELETE FROM #{quote_table_name(table_name)}", name end @@ -397,6 +398,19 @@ module ActiveRecord end end + def build_insert_sql(insert) # :nodoc: + sql = +"INSERT #{insert.into} #{insert.values_list}" + + if insert.skip_duplicates? + sql << " ON CONFLICT #{insert.conflict_target} DO NOTHING" + elsif insert.update_duplicates? + sql << " ON CONFLICT #{insert.conflict_target} DO UPDATE SET " + sql << insert.updatable_columns.map { |column| "#{column}=excluded.#{column}" }.join(",") + end + + sql + end + private # See https://www.sqlite.org/limits.html, # the default value is 999 when not configured. @@ -593,6 +607,10 @@ module ActiveRecord Arel::Visitors::SQLite.new(self) end + def build_statement_pool + StatementPool.new(self.class.type_cast_config_to_integer(@config[:statement_limit])) + end + def configure_connection execute("PRAGMA foreign_keys = ON", "SCHEMA") end diff --git a/activerecord/lib/active_record/insert_all.rb b/activerecord/lib/active_record/insert_all.rb new file mode 100644 index 0000000000..5baaea344b --- /dev/null +++ b/activerecord/lib/active_record/insert_all.rb @@ -0,0 +1,153 @@ +# frozen_string_literal: true + +module ActiveRecord + class InsertAll + attr_reader :model, :connection, :inserts, :on_duplicate, :returning, :unique_by + + def initialize(model, inserts, on_duplicate:, returning: nil, unique_by: nil) + raise ArgumentError, "Empty list of attributes passed" if inserts.blank? + + @model, @connection, @inserts, @on_duplicate, @returning, @unique_by = model, model.connection, inserts, on_duplicate, returning, unique_by + + @returning = (connection.supports_insert_returning? ? primary_keys : false) if @returning.nil? + @returning = false if @returning == [] + + @on_duplicate = :skip if @on_duplicate == :update && updatable_columns.empty? + + ensure_valid_options_for_connection! + end + + def execute + connection.exec_query to_sql, "Bulk Insert" + end + + def keys + inserts.first.keys.map(&:to_s) + end + + def updatable_columns + keys - readonly_columns - unique_by_columns + end + + def skip_duplicates? + on_duplicate == :skip + end + + def update_duplicates? + on_duplicate == :update + end + + private + def ensure_valid_options_for_connection! + if returning && !connection.supports_insert_returning? + raise ArgumentError, "#{connection.class} does not support :returning" + end + + unless %i{ raise skip update }.member?(on_duplicate) + raise NotImplementedError, "#{on_duplicate.inspect} is an unknown value for :on_duplicate. Valid values are :raise, :skip, and :update" + end + + if on_duplicate == :skip && !connection.supports_insert_on_duplicate_skip? + raise ArgumentError, "#{connection.class} does not support skipping duplicates" + end + + if on_duplicate == :update && !connection.supports_insert_on_duplicate_update? + raise ArgumentError, "#{connection.class} does not support upsert" + end + + if unique_by && !connection.supports_insert_conflict_target? + raise ArgumentError, "#{connection.class} does not support :unique_by" + end + end + + def to_sql + connection.build_insert_sql(ActiveRecord::InsertAll::Builder.new(self)) + end + + def readonly_columns + primary_keys + model.readonly_attributes.to_a + end + + def unique_by_columns + unique_by ? unique_by.fetch(:columns).map(&:to_s) : [] + end + + def primary_keys + Array.wrap(model.primary_key) + end + + + class Builder + attr_reader :model + + delegate :skip_duplicates?, :update_duplicates?, to: :insert_all + + def initialize(insert_all) + @insert_all, @model, @connection = insert_all, insert_all.model, insert_all.connection + end + + def into + "INTO #{model.quoted_table_name}(#{columns_list})" + end + + def values_list + columns = connection.schema_cache.columns_hash(model.table_name) + keys = insert_all.keys.to_set + types = keys.map { |key| [ key, connection.lookup_cast_type_from_column(columns[key]) ] }.to_h + + values_list = insert_all.inserts.map do |attributes| + attributes = attributes.stringify_keys + + unless attributes.keys.to_set == keys + raise ArgumentError, "All objects being inserted must have the same keys" + end + + keys.map do |key| + bind = Relation::QueryAttribute.new(key, attributes[key], types[key]) + connection.with_yaml_fallback(bind.value_for_database) + end + end + + Arel::InsertManager.new.create_values_list(values_list).to_sql + end + + def returning + quote_columns(insert_all.returning).join(",") if insert_all.returning + end + + def conflict_target + return unless conflict_columns + sql = +"(#{quote_columns(conflict_columns).join(',')})" + sql << " WHERE #{where}" if where + sql + end + + def updatable_columns + quote_columns(insert_all.updatable_columns) + end + + private + attr_reader :connection, :insert_all + + def columns_list + quote_columns(insert_all.keys).join(",") + end + + def quote_columns(columns) + columns.map(&connection.method(:quote_column_name)) + end + + def conflict_columns + @conflict_columns ||= begin + conflict_columns = insert_all.unique_by.fetch(:columns) if insert_all.unique_by + conflict_columns ||= Array.wrap(model.primary_key) if update_duplicates? + conflict_columns + end + end + + def where + insert_all.unique_by && insert_all.unique_by[:where] + end + end + end +end diff --git a/activerecord/lib/active_record/migration.rb b/activerecord/lib/active_record/migration.rb index c20274420f..997b7f763a 100644 --- a/activerecord/lib/active_record/migration.rb +++ b/activerecord/lib/active_record/migration.rb @@ -308,7 +308,7 @@ module ActiveRecord # named +column_name+ from the table called +table_name+. # * <tt>remove_columns(table_name, *column_names)</tt>: Removes the given # columns from the table definition. - # * <tt>remove_foreign_key(from_table, options_or_to_table)</tt>: Removes the + # * <tt>remove_foreign_key(from_table, to_table = nil, **options)</tt>: Removes the # given foreign key from the table called +table_name+. # * <tt>remove_index(table_name, column: column_names)</tt>: Removes the index # specified by +column_names+. diff --git a/activerecord/lib/active_record/migration/command_recorder.rb b/activerecord/lib/active_record/migration/command_recorder.rb index 82f5121d94..8e7f596076 100644 --- a/activerecord/lib/active_record/migration/command_recorder.rb +++ b/activerecord/lib/active_record/migration/command_recorder.rb @@ -231,24 +231,15 @@ module ActiveRecord end def invert_remove_foreign_key(args) - from_table, options_or_to_table, options_or_nil = args + options = args.extract_options! + from_table, to_table = args - to_table = if options_or_to_table.is_a?(Hash) - options_or_to_table[:to_table] - else - options_or_to_table - end - - remove_options = if options_or_to_table.is_a?(Hash) - options_or_to_table.except(:to_table) - else - options_or_nil - end + to_table ||= options.delete(:to_table) raise ActiveRecord::IrreversibleMigration, "remove_foreign_key is only reversible if given a second table" if to_table.nil? reversed_args = [from_table, to_table] - reversed_args << remove_options if remove_options.present? + reversed_args << options unless options.empty? [:add_foreign_key, reversed_args] end diff --git a/activerecord/lib/active_record/persistence.rb b/activerecord/lib/active_record/persistence.rb index 10148d0dca..72ccf7739f 100644 --- a/activerecord/lib/active_record/persistence.rb +++ b/activerecord/lib/active_record/persistence.rb @@ -1,5 +1,7 @@ # frozen_string_literal: true +require "active_record/insert_all" + module ActiveRecord # = Active Record \Persistence module Persistence @@ -55,6 +57,201 @@ module ActiveRecord end end + # Inserts a single record into the databases. This method constructs a single SQL INSERT + # statement and sends it straight to the database. It does not instantiate the involved + # models and it does not trigger Active Record callbacks or validations. However, values + # passed to #insert will still go through Active Record's normal type casting and + # serialization. + # + # See <tt>ActiveRecord::Persistence#insert_all</tt> for documentation. + def insert(attributes, returning: nil, unique_by: nil) + insert_all([ attributes ], returning: returning, unique_by: unique_by) + end + + # Inserts multiple records into the database. This method constructs a single SQL INSERT + # statement and sends it straight to the database. It does not instantiate the involved + # models and it does not trigger Active Record callbacks or validations. However, values + # passed to #insert_all will still go through Active Record's normal type casting and + # serialization. + # + # The +attributes+ parameter is an Array of Hashes. These Hashes describe the + # attributes on the objects that are to be created. All of the Hashes must have + # same keys. + # + # Records that would violate a unique constraint on the table are skipped. + # + # Returns an <tt>ActiveRecord::Result</tt>. The contents of the result depend on the + # value of <tt>:returning</tt> (see below). + # + # ==== Options + # + # [:returning] + # (Postgres-only) An array of attributes that should be returned for all successfully + # inserted records. For databases that support <tt>INSERT ... RETURNING</tt>, this will default + # to returning the primary keys of the successfully inserted records. Pass + # <tt>returning: %w[ id name ]</tt> to return the id and name of every successfully inserted + # record or pass <tt>returning: false</tt> to omit the clause. + # + # [:unique_by] + # (Postgres and SQLite only) In a table with more than one unique constaint or index, + # new records may considered duplicates according to different criteria. By default, + # new rows will be skipped if they violate _any_ unique constraint/index. By defining + # <tt>:unique_by</tt>, you can skip rows that would create duplicates according to the given + # constraint but raise <tt>ActiveRecord::RecordNotUnique</tt> if rows violate other constraints. + # + # (For example, maybe you assume a client will try to import the same ISBNs more than + # once and want to silently ignore the duplicate records, but you don't except any of + # your code to attempt to create two rows with the same primary key and would appreciate + # an exception report in that scenario.) + # + # Indexes can be identified by an array of columns: + # + # unique_by: { columns: %w[ isbn ] } + # + # Partial indexes can be identified by an array of columns and a <tt>:where</tt> condition: + # + # unique_by: { columns: %w[ isbn ], where: "published_on IS NOT NULL" } + # + # ==== Example + # + # # Insert multiple records and skip duplicates + # # ('Eloquent Ruby' will be skipped because its id is duplicate) + # Book.insert_all([ + # { id: 1, title: 'Rework', author: 'David' }, + # { id: 1, title: 'Eloquent Ruby', author: 'Russ' } + # ]) + # + def insert_all(attributes, returning: nil, unique_by: nil) + InsertAll.new(self, attributes, on_duplicate: :skip, returning: returning, unique_by: unique_by).execute + end + + # Inserts a single record into the databases. This method constructs a single SQL INSERT + # statement and sends it straight to the database. It does not instantiate the involved + # models and it does not trigger Active Record callbacks or validations. However, values + # passed to #insert! will still go through Active Record's normal type casting and + # serialization. + # + # See <tt>ActiveRecord::Persistence#insert_all!</tt> for documentation. + def insert!(attributes, returning: nil) + insert_all!([ attributes ], returning: returning) + end + + # Inserts multiple records into the database. This method constructs a single SQL INSERT + # statement and sends it straight to the database. It does not instantiate the involved + # models and it does not trigger Active Record callbacks or validations. However, values + # passed to #insert_all! will still go through Active Record's normal type casting and + # serialization. + # + # The +attributes+ parameter is an Array of Hashes. These Hashes describe the + # attributes on the objects that are to be created. All of the Hashes must have + # same keys. + # + # #insert_all! will raise <tt>ActiveRecord::RecordNotUnique</tt> if any of the records being + # inserts would violate a unique constraint on the table. In that case, no records + # would be inserted. + # + # To skip duplicate records, see <tt>ActiveRecord::Persistence#insert_all</tt>. + # To replace them, see <tt>ActiveRecord::Persistence#upsert_all</tt>. + # + # Returns an <tt>ActiveRecord::Result</tt>. The contents of the result depend on the + # value of <tt>:returning</tt> (see below). + # + # ==== Options + # + # [:returning] + # (Postgres-only) An array of attributes that should be returned for all successfully + # inserted records. For databases that support <tt>INSERT ... RETURNING</tt>, this will default + # to returning the primary keys of the successfully inserted records. Pass + # <tt>returning: %w[ id name ]</tt> to return the id and name of every successfully inserted + # record or pass <tt>returning: false</tt> to omit the clause. + # + # ==== Examples + # + # # Insert multiple records + # Book.insert_all!([ + # { title: 'Rework', author: 'David' }, + # { title: 'Eloquent Ruby', author: 'Russ' } + # ]) + # + # # raises ActiveRecord::RecordNotUnique because 'Eloquent Ruby' + # # does not have a unique ID + # Book.insert_all!([ + # { id: 1, title: 'Rework', author: 'David' }, + # { id: 1, title: 'Eloquent Ruby', author: 'Russ' } + # ]) + # + def insert_all!(attributes, returning: nil) + InsertAll.new(self, attributes, on_duplicate: :raise, returning: returning).execute + end + + # Upserts (inserts-or-creates) a single record into the databases. This method constructs + # a single SQL INSERT statement and sends it straight to the database. It does not + # instantiate the involved models and it does not trigger Active Record callbacks or + # validations. However, values passed to #upsert will still go through Active Record's + # normal type casting and serialization. + # + # See <tt>ActiveRecord::Persistence#upsert_all</tt> for documentation. + def upsert(attributes, returning: nil, unique_by: nil) + upsert_all([ attributes ], returning: returning, unique_by: unique_by) + end + + # Upserts (creates-or-updates) multiple records into the database. This method constructs + # a single SQL INSERT statement and sends it straight to the database. It does not + # instantiate the involved models and it does not trigger Active Record callbacks or + # validations. However, values passed to #upsert_all will still go through Active Record's + # normal type casting and serialization. + # + # The +attributes+ parameter is an Array of Hashes. These Hashes describe the + # attributes on the objects that are to be created. All of the Hashes must have + # same keys. + # + # Returns an <tt>ActiveRecord::Result</tt>. The contents of the result depend on the + # value of <tt>:returning</tt> (see below). + # + # ==== Options + # + # [:returning] + # (Postgres-only) An array of attributes that should be returned for all successfully + # inserted records. For databases that support <tt>INSERT ... RETURNING</tt>, this will default + # to returning the primary keys of the successfully inserted records. Pass + # <tt>returning: %w[ id name ]</tt> to return the id and name of every successfully inserted + # record or pass <tt>returning: false</tt> to omit the clause. + # + # [:unique_by] + # (Postgres and SQLite only) In a table with more than one unique constaint or index, + # new records may considered duplicates according to different criteria. For MySQL, + # an upsert will take place if a new record violates _any_ unique constraint. For + # Postgres and SQLite, new rows will replace existing rows when the new row has the + # same primary key as the existing row. By defining <tt>:unique_by</tt>, you can supply + # a different key for matching new records to existing ones than the primary key. + # + # (For example, if you have a unique index on the ISBN column and use that as + # the <tt>:unique_by</tt>, a new record with the same ISBN as an existing record + # will replace the existing record but a new record with the same primary key + # as an existing record will raise <tt>ActiveRecord::RecordNotUnique</tt>.) + # + # Indexes can be identified by an array of columns: + # + # unique_by: { columns: %w[ isbn ] } + # + # Partial indexes can be identified by an array of columns and a <tt>:where</tt> condition: + # + # unique_by: { columns: %w[ isbn ], where: "published_on IS NOT NULL" } + # + # ==== Examples + # + # # Insert multiple records, performing an upsert when records have duplicate ISBNs + # # ('Eloquent Ruby' will overwrite 'Rework' because its ISBN is duplicate) + # Book.upsert_all([ + # { title: 'Rework', author: 'David', isbn: '1' }, + # { title: 'Eloquent Ruby', author: 'Russ', isbn: '1' } + # ], + # unique_by: { columns: %w[ isbn ] }) + # + def upsert_all(attributes, returning: nil, unique_by: nil) + InsertAll.new(self, attributes, on_duplicate: :update, returning: returning, unique_by: unique_by).execute + end + # Given an attributes hash, +instantiate+ returns a new instance of # the appropriate class. Accepts only keys as strings. # diff --git a/activerecord/test/cases/bind_parameter_test.rb b/activerecord/test/cases/bind_parameter_test.rb index 03cc0dffbc..eb8b45431f 100644 --- a/activerecord/test/cases/bind_parameter_test.rb +++ b/activerecord/test/cases/bind_parameter_test.rb @@ -41,6 +41,10 @@ if ActiveRecord::Base.connection.prepared_statements topics = Topic.where(id: 1) assert_equal [1], topics.map(&:id) assert_includes statement_cache, to_sql_key(topics.arel) + + @connection.clear_cache! + + assert_not_includes statement_cache, to_sql_key(topics.arel) end def test_statement_cache_with_query_cache diff --git a/activerecord/test/cases/helper.rb b/activerecord/test/cases/helper.rb index 730cd663a2..a9ec667eba 100644 --- a/activerecord/test/cases/helper.rb +++ b/activerecord/test/cases/helper.rb @@ -57,8 +57,17 @@ def supports_default_expression? end end -def supports_savepoints? - ActiveRecord::Base.connection.supports_savepoints? +%w[ + supports_savepoints? + supports_partial_index? + supports_insert_returning? + supports_insert_on_duplicate_skip? + supports_insert_on_duplicate_update? + supports_insert_conflict_target? +].each do |method_name| + define_method method_name do + ActiveRecord::Base.connection.public_send(method_name) + end end def with_env_tz(new_tz = "US/Eastern") diff --git a/activerecord/test/cases/insert_all_test.rb b/activerecord/test/cases/insert_all_test.rb new file mode 100644 index 0000000000..63245719c0 --- /dev/null +++ b/activerecord/test/cases/insert_all_test.rb @@ -0,0 +1,139 @@ +# frozen_string_literal: true + +require "cases/helper" +require "models/book" + +class ReadonlyNameBook < Book + attr_readonly :name +end + +class InsertAllTest < ActiveRecord::TestCase + fixtures :books + + def test_insert + assert_difference "Book.count", +1 do + Book.insert! name: "Rework", author_id: 1 + end + end + + def test_insert_all + assert_difference "Book.count", +10 do + Book.insert_all! [ + { name: "Rework", author_id: 1 }, + { name: "Patterns of Enterprise Application Architecture", author_id: 1 }, + { name: "Design of Everyday Things", author_id: 1 }, + { name: "Practical Object-Oriented Design in Ruby", author_id: 1 }, + { name: "Clean Code", author_id: 1 }, + { name: "Ruby Under a Microscope", author_id: 1 }, + { name: "The Principles of Product Development Flow", author_id: 1 }, + { name: "Peopleware", author_id: 1 }, + { name: "About Face", author_id: 1 }, + { name: "Eloquent Ruby", author_id: 1 }, + ] + end + end + + def test_insert_all_should_handle_empty_arrays + assert_raise ArgumentError do + Book.insert_all! [] + end + end + + def test_insert_all_raises_on_duplicate_records + assert_raise ActiveRecord::RecordNotUnique do + Book.insert_all! [ + { name: "Rework", author_id: 1 }, + { name: "Patterns of Enterprise Application Architecture", author_id: 1 }, + { name: "Agile Web Development with Rails", author_id: 1 }, + ] + end + end + + def test_insert_all_returns_ActiveRecord_Result + result = Book.insert_all! [{ name: "Rework", author_id: 1 }] + assert_kind_of ActiveRecord::Result, result + end + + def test_insert_all_returns_primary_key_if_returning_is_supported + skip unless supports_insert_returning? + + result = Book.insert_all! [{ name: "Rework", author_id: 1 }] + assert_equal %w[ id ], result.columns + end + + def test_insert_all_returns_nothing_if_returning_is_empty + skip unless supports_insert_returning? + + result = Book.insert_all! [{ name: "Rework", author_id: 1 }], returning: [] + assert_equal [], result.columns + end + + def test_insert_all_returns_nothing_if_returning_is_false + skip unless supports_insert_returning? + + result = Book.insert_all! [{ name: "Rework", author_id: 1 }], returning: false + assert_equal [], result.columns + end + + def test_insert_all_returns_requested_fields + skip unless supports_insert_returning? + + result = Book.insert_all! [{ name: "Rework", author_id: 1 }], returning: [:id, :name] + assert_equal %w[ Rework ], result.pluck("name") + end + + def test_insert_all_can_skip_duplicate_records + skip unless supports_insert_on_duplicate_skip? + + assert_no_difference "Book.count" do + Book.insert_all [{ id: 1, name: "Agile Web Development with Rails" }] + end + end + + def test_insert_all_will_raise_if_duplicates_are_skipped_only_for_a_certain_conflict_target + skip unless supports_insert_on_duplicate_skip? && supports_insert_conflict_target? + + assert_raise ActiveRecord::RecordNotUnique do + Book.insert_all [{ id: 1, name: "Agile Web Development with Rails" }], + unique_by: { columns: %i{author_id name} } + end + end + + def test_upsert_all_updates_existing_records + skip unless supports_insert_on_duplicate_update? + + new_name = "Agile Web Development with Rails, 4th Edition" + Book.upsert_all [{ id: 1, name: new_name }] + assert_equal new_name, Book.find(1).name + end + + def test_upsert_all_does_not_update_readonly_attributes + skip unless supports_insert_on_duplicate_update? + + new_name = "Agile Web Development with Rails, 4th Edition" + ReadonlyNameBook.upsert_all [{ id: 1, name: new_name }] + assert_not_equal new_name, Book.find(1).name + end + + def test_upsert_all_does_not_update_primary_keys + skip unless supports_insert_on_duplicate_update? && supports_insert_conflict_target? + + Book.upsert_all [{ id: 101, name: "Perelandra", author_id: 7 }] + Book.upsert_all [{ id: 103, name: "Perelandra", author_id: 7, isbn: "1974522598" }], + unique_by: { columns: %i{author_id name} } + + book = Book.find_by(name: "Perelandra") + assert_equal 101, book.id, "Should not have updated the ID" + assert_equal "1974522598", book.isbn, "Should have updated the isbn" + end + + def test_upsert_all_does_not_perform_an_upsert_if_a_partial_index_doesnt_apply + skip unless supports_insert_on_duplicate_update? && supports_insert_conflict_target? && supports_partial_index? + + Book.upsert_all [{ name: "Out of the Silent Planet", author_id: 7, isbn: "1974522598", published_on: Date.new(1938, 4, 1) }] + Book.upsert_all [{ name: "Perelandra", author_id: 7, isbn: "1974522598" }], + unique_by: { columns: %w[ isbn ], where: "published_on IS NOT NULL" } + + assert_equal ["Out of the Silent Planet", "Perelandra"], Book.where(isbn: "1974522598").order(:name).pluck(:name) + end +end diff --git a/activerecord/test/cases/instrumentation_test.rb b/activerecord/test/cases/instrumentation_test.rb index c09ea32991..06382b6c7c 100644 --- a/activerecord/test/cases/instrumentation_test.rb +++ b/activerecord/test/cases/instrumentation_test.rb @@ -41,8 +41,8 @@ module ActiveRecord assert_equal "Book Update", event.payload[:name] end end - book = Book.create(name: "test book") - book.update_attribute(:name, "new name") + book = Book.create(name: "test book", format: "paperback") + book.update_attribute(:format, "ebook") ensure ActiveSupport::Notifications.unsubscribe(subscriber) if subscriber end @@ -54,8 +54,8 @@ module ActiveRecord assert_equal "Book Update All", event.payload[:name] end end - Book.create(name: "test book") - Book.update_all(name: "new name") + Book.create(name: "test book", format: "paperback") + Book.update_all(format: "ebook") ensure ActiveSupport::Notifications.unsubscribe(subscriber) if subscriber end diff --git a/activerecord/test/cases/migration/foreign_key_test.rb b/activerecord/test/cases/migration/foreign_key_test.rb index 6547ebb5d1..da82f62406 100644 --- a/activerecord/test/cases/migration/foreign_key_test.rb +++ b/activerecord/test/cases/migration/foreign_key_test.rb @@ -385,6 +385,18 @@ if ActiveRecord::Base.connection.supports_foreign_keys? assert_equal "Table 'astronauts' has no foreign key for rockets", e.message end + def test_remove_foreign_key_by_the_select_one_on_the_same_table + @connection.add_foreign_key :astronauts, :rockets + @connection.add_reference :astronauts, :myrocket, foreign_key: { to_table: :rockets } + + assert_equal 2, @connection.foreign_keys("astronauts").size + + @connection.remove_foreign_key :astronauts, :rockets, column: "myrocket_id" + + assert_equal [["astronauts", "rockets", "rocket_id"]], + @connection.foreign_keys("astronauts").map { |fk| [fk.from_table, fk.to_table, fk.column] } + end + if ActiveRecord::Base.connection.supports_validate_constraints? def test_add_invalid_foreign_key @connection.add_foreign_key :astronauts, :rockets, column: "rocket_id", validate: false diff --git a/activerecord/test/schema/schema.rb b/activerecord/test/schema/schema.rb index 5d46bd47d9..ead4de2a13 100644 --- a/activerecord/test/schema/schema.rb +++ b/activerecord/test/schema/schema.rb @@ -114,6 +114,10 @@ ActiveRecord::Schema.define do t.column :font_size, :integer, **default_zero t.column :difficulty, :integer, **default_zero t.column :cover, :string, default: "hard" + t.string :isbn + t.datetime :published_on + t.index [:author_id, :name], unique: true + t.index :isbn, where: "published_on IS NOT NULL", unique: true end create_table :booleans, force: true do |t| diff --git a/activesupport/CHANGELOG.md b/activesupport/CHANGELOG.md index 3f0c6fbd4e..ea84b54b3f 100644 --- a/activesupport/CHANGELOG.md +++ b/activesupport/CHANGELOG.md @@ -1,3 +1,21 @@ +* Allow Array#excluding and Enumerable#excluding to deal with a passed array gracefully. + + [ 1, 2, 3, 4, 5 ].excluding([4, 5]) => [ 1, 2, 3 ] + + *DHH* + +* Renamed Array#without and Enumerable#without to Array#excluding and Enumerable#excluding, to create parity with + Array#including and Enumerable#including. Retained the old names as aliases. + + *DHH* + +* Added Array#including and Enumerable#including to conveniently enlarge a collection with more members using a method rather than an operator: + + [ 1, 2, 3 ].including(4, 5) => [ 1, 2, 3, 4, 5 ] + post.authors.including(Current.person) => All the authors plus the current person! + + *DHH* + ## Rails 6.0.0.beta2 (February 25, 2019) ## * New autoloading based on [Zeitwerk](https://github.com/fxn/zeitwerk). diff --git a/activesupport/lib/active_support/core_ext/array/access.rb b/activesupport/lib/active_support/core_ext/array/access.rb index b7ff7a3907..c9eecc55f9 100644 --- a/activesupport/lib/active_support/core_ext/array/access.rb +++ b/activesupport/lib/active_support/core_ext/array/access.rb @@ -29,16 +29,28 @@ class Array end end - # Returns a copy of the Array without the specified elements. + # Returns a new array that includes the passed elements. + # + # Example: [ 1, 2, 3 ].including(4, 5) => [ 1, 2, 3, 4, 5 ] + def including(*elements) + self + elements.flatten + end + + # Returns a copy of the Array excluding the specified elements. # # people = ["David", "Rafael", "Aaron", "Todd"] - # people.without "Aaron", "Todd" + # people.excluding "Aaron", "Todd" # # => ["David", "Rafael"] # - # Note: This is an optimization of <tt>Enumerable#without</tt> that uses <tt>Array#-</tt> + # Note: This is an optimization of <tt>Enumerable#excluding</tt> that uses <tt>Array#-</tt> # instead of <tt>Array#reject</tt> for performance reasons. + def excluding(*elements) + self - elements.flatten + end + + # Alias for #excluding. def without(*elements) - self - elements + excluding(*elements) end # Equal to <tt>self[1]</tt>. diff --git a/activesupport/lib/active_support/core_ext/enumerable.rb b/activesupport/lib/active_support/core_ext/enumerable.rb index d87d63f287..d6fb89e588 100644 --- a/activesupport/lib/active_support/core_ext/enumerable.rb +++ b/activesupport/lib/active_support/core_ext/enumerable.rb @@ -97,23 +97,43 @@ module Enumerable end end + # Returns a new array that includes the passed elements. + # + # [ 1, 2, 3 ].including(4, 5) + # # => [ 1, 2, 3, 4, 5 ] + # + # ["David", "Rafael"].including %w[ Aaron Todd ] + # # => ["David", "Rafael", "Aaron", "Todd"] + def including(*elements) + to_a.including(*elements) + end + # The negative of the <tt>Enumerable#include?</tt>. Returns +true+ if the # collection does not include the object. def exclude?(object) !include?(object) end - # Returns a copy of the enumerable without the specified elements. + # Returns a copy of the enumerable excluding the specified elements. + # + # ["David", "Rafael", "Aaron", "Todd"].excluding "Aaron", "Todd" + # # => ["David", "Rafael"] # - # ["David", "Rafael", "Aaron", "Todd"].without "Aaron", "Todd" + # ["David", "Rafael", "Aaron", "Todd"].excluding %w[ Aaron Todd ] # # => ["David", "Rafael"] # - # {foo: 1, bar: 2, baz: 3}.without :bar + # {foo: 1, bar: 2, baz: 3}.excluding :bar # # => {foo: 1, baz: 3} - def without(*elements) + def excluding(*elements) + elements.flatten! reject { |element| elements.include?(element) } end + # Alias for #excluding. + def without(*elements) + excluding(*elements) + end + # Convert an enumerable to an array based on the given key. # # [{ name: "David" }, { name: "Rafael" }, { name: "Aaron" }].pluck(:name) diff --git a/activesupport/test/core_ext/array/access_test.rb b/activesupport/test/core_ext/array/access_test.rb index 8c217023cf..8f89c3f25c 100644 --- a/activesupport/test/core_ext/array/access_test.rb +++ b/activesupport/test/core_ext/array/access_test.rb @@ -32,6 +32,16 @@ class AccessTest < ActiveSupport::TestCase assert_equal array[-2], array.second_to_last end + def test_including + assert_equal [1, 2, 3, 4, 5], [1, 2, 4].including(3, 5).sort + assert_equal [1, 2, 3, 4, 5], [1, 2, 4].including([3, 5]).sort + end + + def test_excluding + assert_equal [1, 2, 4], [1, 2, 3, 4, 5].excluding(3, 5) + assert_equal [1, 2, 4], [1, 2, 3, 4, 5].excluding([3, 5]) + end + def test_without assert_equal [1, 2, 4], [1, 2, 3, 4, 5].without(3, 5) end diff --git a/activesupport/test/core_ext/enumerable_test.rb b/activesupport/test/core_ext/enumerable_test.rb index b63464a36a..4e9cf3848d 100644 --- a/activesupport/test/core_ext/enumerable_test.rb +++ b/activesupport/test/core_ext/enumerable_test.rb @@ -217,11 +217,17 @@ class EnumerableTests < ActiveSupport::TestCase assert_equal false, GenericEnumerable.new([ 1 ]).exclude?(1) end + def test_excluding + assert_equal [1, 2, 4], GenericEnumerable.new((1..5).to_a).excluding(3, 5) + assert_equal [3, 4, 5], GenericEnumerable.new((1..5).to_a).excluding([1, 2]) + assert_equal [1, 2, 4], (1..5).to_a.excluding(3, 5) + assert_equal [1, 2, 4], (1..5).to_set.excluding(3, 5) + assert_equal({ foo: 1, baz: 3 }, { foo: 1, bar: 2, baz: 3 }.excluding(:bar)) + end + def test_without assert_equal [1, 2, 4], GenericEnumerable.new((1..5).to_a).without(3, 5) - assert_equal [1, 2, 4], (1..5).to_a.without(3, 5) - assert_equal [1, 2, 4], (1..5).to_set.without(3, 5) - assert_equal({ foo: 1, baz: 3 }, { foo: 1, bar: 2, baz: 3 }.without(:bar)) + assert_equal [3, 4, 5], GenericEnumerable.new((1..5).to_a).without([1, 2]) end def test_pluck |