Merge branch 'master' into feature/reselect-method

8 files changed, 118 insertions, 54 deletions

diff --git a/activerecord/lib/active_record/connection_adapters/abstract/connection_pool.rb b/activerecord/lib/active_record/connection_adapters/abstract/connection_pool.rb
index f721e91203..99934a0e31 100644
--- a/activerecord/lib/active_record/connection_adapters/abstract/connection_pool.rb
+++ b/activerecord/lib/active_record/connection_adapters/abstract/connection_pool.rb
@@ -188,7 +188,9 @@ module ActiveRecord
             t0 = Time.now
             elapsed = 0
             loop do
-              @cond.wait(timeout - elapsed)
+              ActiveSupport::Dependencies.interlock.permit_concurrent_loads do
+                @cond.wait(timeout - elapsed)
+              end
 
               return remove if any?
 
@@ -729,7 +731,7 @@ module ActiveRecord
           # this block can't be easily moved into attempt_to_checkout_all_existing_connections's
           # rescue block, because doing so would put it outside of synchronize section, without
           # being in a critical section thread_report might become inaccurate
-          msg = "could not obtain ownership of all database connections in #{checkout_timeout} seconds".dup
+          msg = +"could not obtain ownership of all database connections in #{checkout_timeout} seconds"
 
           thread_report = []
           @connections.each do |conn|
diff --git a/activerecord/lib/active_record/connection_adapters/abstract/database_limits.rb b/activerecord/lib/active_record/connection_adapters/abstract/database_limits.rb
index 7a9e7add24..1305216be2 100644
--- a/activerecord/lib/active_record/connection_adapters/abstract/database_limits.rb
+++ b/activerecord/lib/active_record/connection_adapters/abstract/database_limits.rb
@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require "active_support/deprecation"
+
 module ActiveRecord
   module ConnectionAdapters # :nodoc:
     module DatabaseLimits
@@ -12,11 +14,13 @@ module ActiveRecord
       def column_name_length
         64
       end
+      deprecate :column_name_length
 
       # Returns the maximum length of a table name.
       def table_name_length
         64
       end
+      deprecate :table_name_length
 
       # Returns the maximum allowed length for an index name. This
       # limit is enforced by \Rails and is less than or equal to
@@ -36,16 +40,19 @@ module ActiveRecord
       def columns_per_table
         1024
       end
+      deprecate :columns_per_table
 
       # Returns the maximum number of indexes per table.
       def indexes_per_table
         16
       end
+      deprecate :indexes_per_table
 
       # Returns the maximum number of columns in a multicolumn index.
       def columns_per_multicolumn_index
         16
       end
+      deprecate :columns_per_multicolumn_index
 
       # Returns the maximum number of elements in an IN (x,y,z) clause.
       # +nil+ means no limit.
@@ -57,11 +64,18 @@ module ActiveRecord
       def sql_query_length
         1048575
       end
+      deprecate :sql_query_length
 
       # Returns maximum number of joins in a single query.
       def joins_per_query
         256
       end
+      deprecate :joins_per_query
+
+      private
+        def bind_params_length
+          65535
+        end
     end
   end
 end
diff --git a/activerecord/lib/active_record/connection_adapters/abstract/database_statements.rb b/activerecord/lib/active_record/connection_adapters/abstract/database_statements.rb
index 41553cfa83..feacdf6931 100644
--- a/activerecord/lib/active_record/connection_adapters/abstract/database_statements.rb
+++ b/activerecord/lib/active_record/connection_adapters/abstract/database_statements.rb
@@ -20,7 +20,7 @@ module ActiveRecord
             raise "Passing bind parameters with an arel AST is forbidden. " \
               "The values must be stored on the AST directly"
           end
-          sql, binds = visitor.accept(arel_or_sql_string.ast, collector).value
+          sql, binds = visitor.compile(arel_or_sql_string.ast, collector)
           [sql.freeze, binds || []]
         else
           [arel_or_sql_string.dup.freeze, binds]
@@ -32,11 +32,11 @@ module ActiveRecord
       # can be used to query the database repeatedly.
       def cacheable_query(klass, arel) # :nodoc:
         if prepared_statements
-          sql, binds = visitor.accept(arel.ast, collector).value
+          sql, binds = visitor.compile(arel.ast, collector)
           query = klass.query(sql)
         else
-          collector = PartialQueryCollector.new
-          parts, binds = visitor.accept(arel.ast, collector).value
+          collector = klass.partial_query_collector
+          parts, binds = visitor.compile(arel.ast, collector)
           query = klass.partial_query(parts)
         end
         [query, binds]
@@ -46,11 +46,16 @@ module ActiveRecord
       def select_all(arel, name = nil, binds = [], preparable: nil)
         arel = arel_from_relation(arel)
         sql, binds = to_sql_and_binds(arel, binds)
+
         if !prepared_statements || (arel.is_a?(String) && preparable.nil?)
           preparable = false
+        elsif binds.length > bind_params_length
+          sql, binds = unprepared_statement { to_sql_and_binds(arel) }
+          preparable = false
         else
           preparable = visitor.preparable
         end
+
         if prepared_statements && preparable
           select_prepared(sql, name, binds)
         else
@@ -259,7 +264,9 @@ module ActiveRecord
 
       attr_reader :transaction_manager #:nodoc:
 
-      delegate :within_new_transaction, :open_transactions, :current_transaction, :begin_transaction, :commit_transaction, :rollback_transaction, to: :transaction_manager
+      delegate :within_new_transaction, :open_transactions, :current_transaction, :begin_transaction,
+               :commit_transaction, :rollback_transaction, :materialize_transactions,
+               :disable_lazy_transactions!, :enable_lazy_transactions!, to: :transaction_manager
 
       def transaction_open?
         current_transaction.open?
@@ -372,7 +379,7 @@ module ActiveRecord
           build_fixture_sql(fixtures, table_name)
         end.compact
 
-        table_deletes = tables_to_delete.map { |table| "DELETE FROM #{quote_table_name table}".dup }
+        table_deletes = tables_to_delete.map { |table| +"DELETE FROM #{quote_table_name table}" }
         total_sql = Array.wrap(combine_multi_statements(table_deletes + fixture_inserts))
 
         disable_referential_integrity do
@@ -500,28 +507,6 @@ module ActiveRecord
             value
           end
         end
-
-        class PartialQueryCollector
-          def initialize
-            @parts = []
-            @binds = []
-          end
-
-          def <<(str)
-            @parts << str
-            self
-          end
-
-          def add_bind(obj)
-            @binds << obj
-            @parts << Arel::Nodes::BindParam.new(1)
-            self
-          end
-
-          def value
-            [@parts, @binds]
-          end
-        end
     end
   end
 end
diff --git a/activerecord/lib/active_record/connection_adapters/abstract/quoting.rb b/activerecord/lib/active_record/connection_adapters/abstract/quoting.rb
index 98b1348135..07e86afe9a 100644
--- a/activerecord/lib/active_record/connection_adapters/abstract/quoting.rb
+++ b/activerecord/lib/active_record/connection_adapters/abstract/quoting.rb
@@ -60,7 +60,7 @@ module ActiveRecord
       # Quotes a string, escaping any ' (single quote) and \ (backslash)
       # characters.
       def quote_string(s)
-        s.gsub('\\'.freeze, '\&\&'.freeze).gsub("'".freeze, "''".freeze) # ' (for ruby-mode)
+        s.gsub('\\', '\&\&').gsub("'", "''") # ' (for ruby-mode)
       end
 
       # Quotes the column name. Defaults to no quoting.
@@ -95,7 +95,7 @@ module ActiveRecord
       end
 
       def quoted_true
-        "TRUE".freeze
+        "TRUE"
       end
 
       def unquoted_true
@@ -103,7 +103,7 @@ module ActiveRecord
       end
 
       def quoted_false
-        "FALSE".freeze
+        "FALSE"
       end
 
       def unquoted_false
diff --git a/activerecord/lib/active_record/connection_adapters/abstract/schema_creation.rb b/activerecord/lib/active_record/connection_adapters/abstract/schema_creation.rb
index 529c9d8ca6..9d9e8a4110 100644
--- a/activerecord/lib/active_record/connection_adapters/abstract/schema_creation.rb
+++ b/activerecord/lib/active_record/connection_adapters/abstract/schema_creation.rb
@@ -21,7 +21,7 @@ module ActiveRecord
         private
 
           def visit_AlterTable(o)
-            sql = "ALTER TABLE #{quote_table_name(o.name)} ".dup
+            sql = +"ALTER TABLE #{quote_table_name(o.name)} "
             sql << o.adds.map { |col| accept col }.join(" ")
             sql << o.foreign_key_adds.map { |fk| visit_AddForeignKey fk }.join(" ")
             sql << o.foreign_key_drops.map { |fk| visit_DropForeignKey fk }.join(" ")
@@ -29,17 +29,17 @@ module ActiveRecord
 
           def visit_ColumnDefinition(o)
             o.sql_type = type_to_sql(o.type, o.options)
-            column_sql = "#{quote_column_name(o.name)} #{o.sql_type}".dup
+            column_sql = +"#{quote_column_name(o.name)} #{o.sql_type}"
             add_column_options!(column_sql, column_options(o)) unless o.type == :primary_key
             column_sql
           end
 
           def visit_AddColumnDefinition(o)
-            "ADD #{accept(o.column)}".dup
+            +"ADD #{accept(o.column)}"
           end
 
           def visit_TableDefinition(o)
-            create_sql = "CREATE#{' TEMPORARY' if o.temporary} TABLE #{quote_table_name(o.name)} ".dup
+            create_sql = +"CREATE#{' TEMPORARY' if o.temporary} TABLE #{quote_table_name(o.name)} "
 
             statements = o.columns.map { |c| accept c }
             statements << accept(o.primary_keys) if o.primary_keys
diff --git a/activerecord/lib/active_record/connection_adapters/abstract/schema_definitions.rb b/activerecord/lib/active_record/connection_adapters/abstract/schema_definitions.rb
index 582ac516c7..015204c056 100644
--- a/activerecord/lib/active_record/connection_adapters/abstract/schema_definitions.rb
+++ b/activerecord/lib/active_record/connection_adapters/abstract/schema_definitions.rb
@@ -527,7 +527,9 @@ module ActiveRecord
       #
       # See TableDefinition#column for details of the options you can use.
       def column(column_name, type, options = {})
+        index_options = options.delete(:index)
         @base.add_column(name, column_name, type, options)
+        index(column_name, index_options.is_a?(Hash) ? index_options : {}) if index_options
       end
 
       # Checks to see if a column exists.
diff --git a/activerecord/lib/active_record/connection_adapters/abstract/schema_statements.rb b/activerecord/lib/active_record/connection_adapters/abstract/schema_statements.rb
index 3be0906f2a..723d8c318d 100644
--- a/activerecord/lib/active_record/connection_adapters/abstract/schema_statements.rb
+++ b/activerecord/lib/active_record/connection_adapters/abstract/schema_statements.rb
@@ -211,13 +211,13 @@ module ActiveRecord
       #
       # ====== Add a backend specific option to the generated SQL (MySQL)
       #
-      #   create_table(:suppliers, options: 'ENGINE=InnoDB DEFAULT CHARSET=utf8')
+      #   create_table(:suppliers, options: 'ENGINE=InnoDB DEFAULT CHARSET=utf8mb4')
       #
       # generates:
       #
       #   CREATE TABLE suppliers (
       #     id bigint auto_increment PRIMARY KEY
-      #   ) ENGINE=InnoDB DEFAULT CHARSET=utf8
+      #   ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
       #
       # ====== Rename the primary key column
       #
@@ -522,6 +522,9 @@ module ActiveRecord
       #   Specifies the precision for the <tt>:decimal</tt> and <tt>:numeric</tt> columns.
       # * <tt>:scale</tt> -
       #   Specifies the scale for the <tt>:decimal</tt> and <tt>:numeric</tt> columns.
+      # * <tt>:collation</tt> -
+      #   Specifies the collation for a <tt>:string</tt> or <tt>:text</tt> column. If not specified, the
+      #   column will have the same collation as the table.
       # * <tt>:comment</tt> -
       #   Specifies the comment for the column. This option is ignored by some backends.
       #
@@ -599,6 +602,7 @@ module ActiveRecord
       # The +type+ and +options+ parameters will be ignored if present. It can be helpful
       # to provide these in a migration's +change+ method so it can be reverted.
       # In that case, +type+ and +options+ will be used by #add_column.
+      # Indexes on the column are automatically removed.
       def remove_column(table_name, column_name, type = nil, options = {})
         execute "ALTER TABLE #{quote_table_name(table_name)} #{remove_column_for_alter(table_name, column_name, type, options)}"
       end
@@ -980,11 +984,18 @@ module ActiveRecord
       #
       #   remove_foreign_key :accounts, column: :owner_id
       #
+      # Removes the foreign key on +accounts.owner_id+.
+      #
+      #   remove_foreign_key :accounts, to_table: :owners
+      #
       # Removes the foreign key named +special_fk_name+ on the +accounts+ table.
       #
       #   remove_foreign_key :accounts, name: :special_fk_name
       #
-      # The +options+ hash accepts the same keys as SchemaStatements#add_foreign_key.
+      # The +options+ hash accepts the same keys as SchemaStatements#add_foreign_key
+      # with an addition of
+      # [<tt>:to_table</tt>]
+      #   The name of the table that contains the referenced primary key.
       def remove_foreign_key(from_table, options_or_to_table = {})
         return unless supports_foreign_keys?
 
@@ -1369,7 +1380,7 @@ module ActiveRecord
           sm_table = quote_table_name(ActiveRecord::SchemaMigration.table_name)
 
           if versions.is_a?(Array)
-            sql = "INSERT INTO #{sm_table} (version) VALUES\n".dup
+            sql = +"INSERT INTO #{sm_table} (version) VALUES\n"
             sql << versions.map { |v| "(#{quote(v)})" }.join(",\n")
             sql << ";\n\n"
             sql
diff --git a/activerecord/lib/active_record/connection_adapters/abstract/transaction.rb b/activerecord/lib/active_record/connection_adapters/abstract/transaction.rb
index b59df2fff7..564b226b39 100644
--- a/activerecord/lib/active_record/connection_adapters/abstract/transaction.rb
+++ b/activerecord/lib/active_record/connection_adapters/abstract/transaction.rb
@@ -91,12 +91,14 @@ module ActiveRecord
     end
 
     class Transaction #:nodoc:
-      attr_reader :connection, :state, :records, :savepoint_name
+      attr_reader :connection, :state, :records, :savepoint_name, :isolation_level
 
       def initialize(connection, options, run_commit_callbacks: false)
         @connection = connection
         @state = TransactionState.new
         @records = []
+        @isolation_level = options[:isolation]
+        @materialized = false
         @joinable = options.fetch(:joinable, true)
         @run_commit_callbacks = run_commit_callbacks
       end
@@ -105,6 +107,14 @@ module ActiveRecord
         records << record
       end
 
+      def materialize!
+        @materialized = true
+      end
+
+      def materialized?
+        @materialized
+      end
+
       def rollback_records
         ite = records.uniq
         while record = ite.shift
@@ -141,24 +151,30 @@ module ActiveRecord
     end
 
     class SavepointTransaction < Transaction
-      def initialize(connection, savepoint_name, parent_transaction, options, *args)
-        super(connection, options, *args)
+      def initialize(connection, savepoint_name, parent_transaction, *args)
+        super(connection, *args)
 
         parent_transaction.state.add_child(@state)
 
-        if options[:isolation]
+        if isolation_level
           raise ActiveRecord::TransactionIsolationError, "cannot set transaction isolation in a nested transaction"
         end
-        connection.create_savepoint(@savepoint_name = savepoint_name)
+
+        @savepoint_name = savepoint_name
+      end
+
+      def materialize!
+        connection.create_savepoint(savepoint_name)
+        super
       end
 
       def rollback
-        connection.rollback_to_savepoint(savepoint_name)
+        connection.rollback_to_savepoint(savepoint_name) if materialized?
         @state.rollback!
       end
 
       def commit
-        connection.release_savepoint(savepoint_name)
+        connection.release_savepoint(savepoint_name) if materialized?
         @state.commit!
       end
 
@@ -166,22 +182,23 @@ module ActiveRecord
     end
 
     class RealTransaction < Transaction
-      def initialize(connection, options, *args)
-        super
-        if options[:isolation]
-          connection.begin_isolated_db_transaction(options[:isolation])
+      def materialize!
+        if isolation_level
+          connection.begin_isolated_db_transaction(isolation_level)
         else
           connection.begin_db_transaction
         end
+
+        super
       end
 
       def rollback
-        connection.rollback_db_transaction
+        connection.rollback_db_transaction if materialized?
         @state.full_rollback!
       end
 
       def commit
-        connection.commit_db_transaction
+        connection.commit_db_transaction if materialized?
         @state.full_commit!
       end
     end
@@ -190,6 +207,9 @@ module ActiveRecord
       def initialize(connection)
         @stack = []
         @connection = connection
+        @has_unmaterialized_transactions = false
+        @materializing_transactions = false
+        @lazy_transactions_enabled = true
       end
 
       def begin_transaction(options = {})
@@ -203,11 +223,41 @@ module ActiveRecord
                                        run_commit_callbacks: run_commit_callbacks)
             end
 
+          transaction.materialize! unless @connection.supports_lazy_transactions? && lazy_transactions_enabled?
           @stack.push(transaction)
+          @has_unmaterialized_transactions = true if @connection.supports_lazy_transactions?
           transaction
         end
       end
 
+      def disable_lazy_transactions!
+        materialize_transactions
+        @lazy_transactions_enabled = false
+      end
+
+      def enable_lazy_transactions!
+        @lazy_transactions_enabled = true
+      end
+
+      def lazy_transactions_enabled?
+        @lazy_transactions_enabled
+      end
+
+      def materialize_transactions
+        return if @materializing_transactions
+        return unless @has_unmaterialized_transactions
+
+        @connection.lock.synchronize do
+          begin
+            @materializing_transactions = true
+            @stack.each { |t| t.materialize! unless t.materialized? }
+          ensure
+            @materializing_transactions = false
+          end
+          @has_unmaterialized_transactions = false
+        end
+      end
+
       def commit_transaction
         @connection.lock.synchronize do
           transaction = @stack.last