diff options
Diffstat (limited to 'activerecord/lib/active_record/connection_adapters')
35 files changed, 752 insertions, 508 deletions
diff --git a/activerecord/lib/active_record/connection_adapters/abstract/connection_pool.rb b/activerecord/lib/active_record/connection_adapters/abstract/connection_pool.rb index d0c5bbe17d..5ec2fc073e 100644 --- a/activerecord/lib/active_record/connection_adapters/abstract/connection_pool.rb +++ b/activerecord/lib/active_record/connection_adapters/abstract/connection_pool.rb @@ -69,7 +69,7 @@ module ActiveRecord # threads, which can occur if a programmer forgets to close a # connection at the end of a thread or a thread dies unexpectedly. # Regardless of this setting, the Reaper will be invoked before every - # blocking wait. (Default nil, which means don't schedule the Reaper). + # blocking wait. (Default +nil+, which means don't schedule the Reaper). # #-- # Synchronization policy: @@ -116,7 +116,7 @@ module ActiveRecord end end - # If +element+ is in the queue, remove and return it, or nil. + # If +element+ is in the queue, remove and return it, or +nil+. def delete(element) synchronize do @queue.delete(element) @@ -135,7 +135,7 @@ module ActiveRecord # If +timeout+ is not given, remove and return the head the # queue if the number of available elements is strictly # greater than the number of threads currently waiting (that - # is, don't jump ahead in line). Otherwise, return nil. + # is, don't jump ahead in line). Otherwise, return +nil+. # # If +timeout+ is given, block if there is no element # available, waiting up to +timeout+ seconds for an element to @@ -158,33 +158,33 @@ module ActiveRecord @lock.synchronize(&block) end - # Test if the queue currently contains any elements. + # Test if the queue currently contains any elements. def any? !@queue.empty? end - # A thread can remove an element from the queue without - # waiting if and only if the number of currently available - # connections is strictly greater than the number of waiting - # threads. + # A thread can remove an element from the queue without + # waiting if and only if the number of currently available + # connections is strictly greater than the number of waiting + # threads. def can_remove_no_wait? @queue.size > @num_waiting end - # Removes and returns the head of the queue if possible, or nil. + # Removes and returns the head of the queue if possible, or +nil+. def remove @queue.shift end - # Remove and return the head the queue if the number of - # available elements is strictly greater than the number of - # threads currently waiting. Otherwise, return nil. + # Remove and return the head the queue if the number of + # available elements is strictly greater than the number of + # threads currently waiting. Otherwise, return +nil+. def no_wait_poll remove if can_remove_no_wait? end - # Waits on the queue up to +timeout+ seconds, then removes and - # returns the head of the queue. + # Waits on the queue up to +timeout+ seconds, then removes and + # returns the head of the queue. def wait_poll(timeout) @num_waiting += 1 @@ -282,7 +282,7 @@ module ActiveRecord end # Every +frequency+ seconds, the reaper will call +reap+ on +pool+. - # A reaper instantiated with a nil frequency will never reap the + # A reaper instantiated with a +nil+ frequency will never reap the # connection pool. # # Configure the frequency by setting "reaping_frequency" in your @@ -307,6 +307,7 @@ module ActiveRecord end include MonitorMixin + include QueryCache::ConnectionPoolConfiguration attr_accessor :automatic_reconnect, :checkout_timeout, :schema_cache attr_reader :spec, :connections, :size, :reaper @@ -349,8 +350,7 @@ module ActiveRecord # currently in the process of independently establishing connections to the DB. @now_connecting = 0 - # A boolean toggle that allows/disallows new connections. - @new_cons_enabled = true + @threads_blocking_new_connections = 0 @available = ConnectionLeasingQueue.new self end @@ -445,8 +445,6 @@ module ActiveRecord # connections in the pool within a timeout interval (default duration is # <tt>spec.config[:checkout_timeout] * 2</tt> seconds). def clear_reloadable_connections(raise_on_acquisition_timeout = true) - num_new_conns_required = 0 - with_exclusively_acquired_all_connections(raise_on_acquisition_timeout) do synchronize do @connections.each do |conn| @@ -457,24 +455,9 @@ module ActiveRecord conn.disconnect! if conn.requires_reloading? end @connections.delete_if(&:requires_reloading?) - @available.clear - - if @connections.size < @size - # because of the pruning done by this method, we might be running - # low on connections, while threads stuck in queue are helpless - # (not being able to establish new connections for themselves), - # see also more detailed explanation in +remove+ - num_new_conns_required = num_waiting_in_queue - @connections.size - end - - @connections.each do |conn| - @available.add conn - end end end - - bulk_make_new_connections(num_new_conns_required) if num_new_conns_required > 0 end # Clears the cache which maps classes and re-connects connections that @@ -581,9 +564,27 @@ module ActiveRecord @available.num_waiting end + # Return connection pool's usage statistic + # Example: + # + # ActiveRecord::Base.connection_pool.stat # => { size: 15, connections: 1, busy: 1, dead: 0, idle: 0, waiting: 0, checkout_timeout: 5 } + def stat + synchronize do + { + size: size, + connections: @connections.size, + busy: @connections.count { |c| c.in_use? && c.owner.alive? }, + dead: @connections.count { |c| c.in_use? && !c.owner.alive? }, + idle: @connections.count { |c| !c.in_use? }, + waiting: num_waiting_in_queue, + checkout_timeout: checkout_timeout + } + end + end + private - #-- - # this is unfortunately not concurrent + #-- + # this is unfortunately not concurrent def bulk_make_new_connections(num_new_conns_needed) num_new_conns_needed.times do # try_to_checkout_new_connection will not exceed pool's @size limit @@ -594,19 +595,19 @@ module ActiveRecord end end - #-- - # From the discussion on GitHub: - # https://github.com/rails/rails/pull/14938#commitcomment-6601951 - # This hook-in method allows for easier monkey-patching fixes needed by - # JRuby users that use Fibers. + #-- + # From the discussion on GitHub: + # https://github.com/rails/rails/pull/14938#commitcomment-6601951 + # This hook-in method allows for easier monkey-patching fixes needed by + # JRuby users that use Fibers. def connection_cache_key(thread) thread end - # Take control of all existing connections so a "group" action such as - # reload/disconnect can be performed safely. It is no longer enough to - # wrap it in +synchronize+ because some pool's actions are allowed - # to be performed outside of the main +synchronize+ block. + # Take control of all existing connections so a "group" action such as + # reload/disconnect can be performed safely. It is no longer enough to + # wrap it in +synchronize+ because some pool's actions are allowed + # to be performed outside of the main +synchronize+ block. def with_exclusively_acquired_all_connections(raise_on_acquisition_timeout = true) with_new_connections_blocked do attempt_to_checkout_all_existing_connections(raise_on_acquisition_timeout) @@ -658,8 +659,8 @@ module ActiveRecord end end - #-- - # Must be called in a synchronize block. + #-- + # Must be called in a synchronize block. def checkout_for_exclusive_access(checkout_timeout) checkout(checkout_timeout) rescue ConnectionTimeoutError @@ -681,26 +682,45 @@ module ActiveRecord end def with_new_connections_blocked - previous_value = nil synchronize do - previous_value, @new_cons_enabled = @new_cons_enabled, false + @threads_blocking_new_connections += 1 end + yield ensure - synchronize { @new_cons_enabled = previous_value } + num_new_conns_required = 0 + + synchronize do + @threads_blocking_new_connections -= 1 + + if @threads_blocking_new_connections.zero? + @available.clear + + num_new_conns_required = num_waiting_in_queue + + @connections.each do |conn| + next if conn.in_use? + + @available.add conn + num_new_conns_required -= 1 + end + end + end + + bulk_make_new_connections(num_new_conns_required) if num_new_conns_required > 0 end - # Acquire a connection by one of 1) immediately removing one - # from the queue of available connections, 2) creating a new - # connection if the pool is not at capacity, 3) waiting on the - # queue for a connection to become available. - # - # Raises: - # - ActiveRecord::ConnectionTimeoutError if a connection could not be acquired - # - #-- - # Implementation detail: the connection returned by +acquire_connection+ - # will already be "+connection.lease+ -ed" to the current thread. + # Acquire a connection by one of 1) immediately removing one + # from the queue of available connections, 2) creating a new + # connection if the pool is not at capacity, 3) waiting on the + # queue for a connection to become available. + # + # Raises: + # - ActiveRecord::ConnectionTimeoutError if a connection could not be acquired + # + #-- + # Implementation detail: the connection returned by +acquire_connection+ + # will already be "+connection.lease+ -ed" to the current thread. def acquire_connection(checkout_timeout) # NOTE: we rely on +@available.poll+ and +try_to_checkout_new_connection+ to # +conn.lease+ the returned connection (and to do this in a +synchronized+ @@ -716,8 +736,8 @@ module ActiveRecord end end - #-- - # if owner_thread param is omitted, this must be called in synchronize block + #-- + # if owner_thread param is omitted, this must be called in synchronize block def remove_connection_from_thread_cache(conn, owner_thread = conn.owner) @thread_cached_conns.delete_pair(connection_cache_key(owner_thread), conn) end @@ -729,17 +749,17 @@ module ActiveRecord end end - # If the pool is not at a +@size+ limit, establish new connection. Connecting - # to the DB is done outside main synchronized section. - #-- - # Implementation constraint: a newly established connection returned by this - # method must be in the +.leased+ state. + # If the pool is not at a +@size+ limit, establish new connection. Connecting + # to the DB is done outside main synchronized section. + #-- + # Implementation constraint: a newly established connection returned by this + # method must be in the +.leased+ state. def try_to_checkout_new_connection # first in synchronized section check if establishing new conns is allowed # and increment @now_connecting, to prevent overstepping this pool's @size # constraint do_checkout = synchronize do - if @new_cons_enabled && (@connections.size + @now_connecting) < @size + if @threads_blocking_new_connections.zero? && (@connections.size + @now_connecting) < @size @now_connecting += 1 end end @@ -833,7 +853,7 @@ module ActiveRecord class ConnectionHandler def initialize # These caches are keyed by spec.name (ConnectionSpecification#name). - @owner_to_pool = Concurrent::Map.new(initial_capacity: 2) do |h,k| + @owner_to_pool = Concurrent::Map.new(initial_capacity: 2) do |h, k| h[k] = Concurrent::Map.new(initial_capacity: 2) end end diff --git a/activerecord/lib/active_record/connection_adapters/abstract/database_limits.rb b/activerecord/lib/active_record/connection_adapters/abstract/database_limits.rb index 95c72f1e20..407e019326 100644 --- a/activerecord/lib/active_record/connection_adapters/abstract/database_limits.rb +++ b/activerecord/lib/active_record/connection_adapters/abstract/database_limits.rb @@ -46,7 +46,7 @@ module ActiveRecord end # Returns the maximum number of elements in an IN (x,y,z) clause. - # nil means no limit. + # +nil+ means no limit. def in_clause_length nil end diff --git a/activerecord/lib/active_record/connection_adapters/abstract/database_statements.rb b/activerecord/lib/active_record/connection_adapters/abstract/database_statements.rb index def1dadb6a..769f488469 100644 --- a/activerecord/lib/active_record/connection_adapters/abstract/database_statements.rb +++ b/activerecord/lib/active_record/connection_adapters/abstract/database_statements.rb @@ -10,9 +10,9 @@ module ActiveRecord def to_sql(arel, binds = []) if arel.respond_to?(:ast) collected = visitor.accept(arel.ast, collector) - collected.compile(binds, self) + collected.compile(binds, self).freeze else - arel + arel.dup.freeze end end @@ -51,8 +51,7 @@ module ActiveRecord # Returns a single value from a record def select_value(arel, name = nil, binds = []) - arel, binds = binds_from_relation arel, binds - if result = select_rows(to_sql(arel, binds), name, binds).first + if result = select_rows(arel, name, binds).first result.first end end @@ -60,14 +59,13 @@ module ActiveRecord # Returns an array of the values of the first column in a select: # select_values("SELECT id FROM companies LIMIT 3") => [1,2,3] def select_values(arel, name = nil, binds = []) - arel, binds = binds_from_relation arel, binds - select_rows(to_sql(arel, binds), name, binds).map(&:first) + select_rows(arel, name, binds).map(&:first) end # Returns an array of arrays containing the field values. # Order is the same as that returned by +columns+. - def select_rows(sql, name = nil, binds = []) - exec_query(sql, name, binds).rows + def select_rows(arel, name = nil, binds = []) + select_all(arel, name, binds).rows end # Executes the SQL statement in the context of this connection and returns @@ -115,7 +113,7 @@ module ActiveRecord # Executes an INSERT query and returns the new record's ID # - # +id_value+ will be returned unless the value is nil, in + # +id_value+ will be returned unless the value is +nil+, in # which case the database will attempt to calculate the last inserted # id and return that value. # @@ -126,22 +124,16 @@ module ActiveRecord id_value || last_inserted_id(value) end alias create insert - alias insert_sql insert - deprecate insert_sql: :insert # Executes the update statement and returns the number of rows affected. def update(arel, name = nil, binds = []) exec_update(to_sql(arel, binds), name, binds) end - alias update_sql update - deprecate update_sql: :update # Executes the delete statement and returns the number of rows affected. def delete(arel, name = nil, binds = []) exec_delete(to_sql(arel, binds), name, binds) end - alias delete_sql delete - deprecate delete_sql: :delete # Returns +true+ when the connection adapter supports prepared statement # caching, otherwise returns +false+ @@ -245,7 +237,7 @@ module ActiveRecord end def reset_transaction #:nodoc: - @transaction_manager = TransactionManager.new(self) + @transaction_manager = ConnectionAdapters::TransactionManager.new(self) end # Register a record with the current transaction so that its after_commit and after_rollback callbacks @@ -334,17 +326,12 @@ module ActiveRecord # Sanitizes the given LIMIT parameter in order to prevent SQL injection. # # The +limit+ may be anything that can evaluate to a string via #to_s. It - # should look like an integer, or a comma-delimited list of integers, or - # an Arel SQL literal. + # should look like an integer, or an Arel SQL literal. # # Returns Integer and Arel::Nodes::SqlLiteral limits as is. - # Returns the sanitized limit parameter, either as an integer, or as a - # string which contains a comma-delimited list of integers. def sanitize_limit(limit) if limit.is_a?(Integer) || limit.is_a?(Arel::Nodes::SqlLiteral) limit - elsif limit.to_s.include?(",") - Arel.sql limit.to_s.split(",").map { |i| Integer(i) }.join(",") else Integer(limit) end @@ -360,7 +347,7 @@ module ActiveRecord end alias join_to_delete join_to_update - protected + private # Returns a subquery for the given key using the join information. def subquery_for(key, select) diff --git a/activerecord/lib/active_record/connection_adapters/abstract/query_cache.rb b/activerecord/lib/active_record/connection_adapters/abstract/query_cache.rb index 10c60080d5..7eab7de5d3 100644 --- a/activerecord/lib/active_record/connection_adapters/abstract/query_cache.rb +++ b/activerecord/lib/active_record/connection_adapters/abstract/query_cache.rb @@ -4,6 +4,9 @@ module ActiveRecord class << self def included(base) #:nodoc: dirties_query_cache base, :insert, :update, :delete, :rollback_to_savepoint, :rollback_db_transaction + + base.set_callback :checkout, :after, :configure_query_cache! + base.set_callback :checkin, :after, :disable_query_cache! end def dirties_query_cache(base, *method_names) @@ -18,11 +21,32 @@ module ActiveRecord end end + module ConnectionPoolConfiguration + def initialize(*) + super + @query_cache_enabled = Concurrent::Map.new { false } + end + + def enable_query_cache! + @query_cache_enabled[connection_cache_key(Thread.current)] = true + connection.enable_query_cache! if active_connection? + end + + def disable_query_cache! + @query_cache_enabled.delete connection_cache_key(Thread.current) + connection.disable_query_cache! if active_connection? + end + + def query_cache_enabled + @query_cache_enabled[connection_cache_key(Thread.current)] + end + end + attr_reader :query_cache, :query_cache_enabled def initialize(*) super - @query_cache = Hash.new { |h,sql| h[sql] = {} } + @query_cache = Hash.new { |h, sql| h[sql] = {} } @query_cache_enabled = false end @@ -41,6 +65,7 @@ module ActiveRecord def disable_query_cache! @query_cache_enabled = false + clear_query_cache end # Disable the query cache within the block. @@ -65,7 +90,7 @@ module ActiveRecord if @query_cache_enabled && !locked?(arel) arel, binds = binds_from_relation arel, binds sql = to_sql(arel, binds) - cache_sql(sql, binds) { super(sql, name, binds, preparable: preparable) } + cache_sql(sql, name, binds) { super(sql, name, binds, preparable: preparable) } else super end @@ -73,11 +98,17 @@ module ActiveRecord private - def cache_sql(sql, binds) + def cache_sql(sql, name, binds) result = if @query_cache[sql].key?(binds) - ActiveSupport::Notifications.instrument("sql.active_record", - sql: sql, binds: binds, name: "CACHE", connection_id: object_id) + ActiveSupport::Notifications.instrument( + "sql.active_record", + sql: sql, + binds: binds, + name: name, + connection_id: object_id, + cached: true, + ) @query_cache[sql][binds] else @query_cache[sql][binds] = yield @@ -85,11 +116,15 @@ module ActiveRecord result.dup end - # If arel is locked this is a SELECT ... FOR UPDATE or somesuch. Such - # queries should not be cached. + # If arel is locked this is a SELECT ... FOR UPDATE or somesuch. Such + # queries should not be cached. def locked?(arel) arel.respond_to?(:locked) && arel.locked end + + def configure_query_cache! + enable_query_cache! if pool.query_cache_enabled + end end end end diff --git a/activerecord/lib/active_record/connection_adapters/abstract/quoting.rb b/activerecord/lib/active_record/connection_adapters/abstract/quoting.rb index bbd52b8a91..0c6bc16e6f 100644 --- a/activerecord/lib/active_record/connection_adapters/abstract/quoting.rb +++ b/activerecord/lib/active_record/connection_adapters/abstract/quoting.rb @@ -5,20 +5,10 @@ module ActiveRecord module Quoting # Quotes the column value to help prevent # {SQL injection attacks}[http://en.wikipedia.org/wiki/SQL_injection]. - def quote(value, column = nil) + def quote(value) # records are quoted as their primary key return value.quoted_id if value.respond_to?(:quoted_id) - if column - ActiveSupport::Deprecation.warn(<<-MSG.squish) - Passing a column to `quote` has been deprecated. It is only used - for type casting, which should be handled elsewhere. See - https://github.com/rails/arel/commit/6160bfbda1d1781c3b08a33ec4955f170e95be11 - for more information. - MSG - value = type_cast_from_column(column, value) - end - _quote(value) end diff --git a/activerecord/lib/active_record/connection_adapters/abstract/schema_definitions.rb b/activerecord/lib/active_record/connection_adapters/abstract/schema_definitions.rb index ffde4f2c93..9b324c090b 100644 --- a/activerecord/lib/active_record/connection_adapters/abstract/schema_definitions.rb +++ b/activerecord/lib/active_record/connection_adapters/abstract/schema_definitions.rb @@ -71,7 +71,7 @@ module ActiveRecord polymorphic: false, index: true, foreign_key: false, - type: :integer, + type: :bigint, **options ) @name = name @@ -100,6 +100,8 @@ module ActiveRecord end end + # TODO Change this to private once we've dropped Ruby 2.2 support. + # Workaround for Ruby 2.2 "private attribute?" warning. protected attr_reader :name, :polymorphic, :index, :foreign_key, :type, :options @@ -475,7 +477,7 @@ module ActiveRecord # Checks to see if a column exists. # - # t.string(:name) unless t.column_exists?(:name, :string) + # t.string(:name) unless t.column_exists?(:name, :string) # # See {connection.column_exists?}[rdoc-ref:SchemaStatements#column_exists?] def column_exists?(column_name, type = nil, options = {}) @@ -496,9 +498,9 @@ module ActiveRecord # Checks to see if an index exists. # - # unless t.index_exists?(:branch_id) - # t.index(:branch_id) - # end + # unless t.index_exists?(:branch_id) + # t.index(:branch_id) + # end # # See {connection.index_exists?}[rdoc-ref:SchemaStatements#index_exists?] def index_exists?(column_name, options = {}) diff --git a/activerecord/lib/active_record/connection_adapters/abstract/schema_dumper.rb b/activerecord/lib/active_record/connection_adapters/abstract/schema_dumper.rb index 8bb7362c2e..b912d24626 100644 --- a/activerecord/lib/active_record/connection_adapters/abstract/schema_dumper.rb +++ b/activerecord/lib/active_record/connection_adapters/abstract/schema_dumper.rb @@ -7,10 +7,7 @@ module ActiveRecord # Adapter level by over-writing this code inside the database specific adapters module ColumnDumper def column_spec(column) - spec = Hash[prepare_column_options(column).map { |k, v| [k, "#{k}: #{v}"] }] - spec[:name] = column.name.inspect - spec[:type] = schema_type(column).to_s - spec + [schema_type(column), prepare_column_options(column)] end def column_spec_for_primary_key(column) @@ -38,7 +35,7 @@ module ActiveRecord end default = schema_default(column) if column.has_default? - spec[:default] = default unless default.nil? + spec[:default] = default unless default.nil? spec[:null] = "false" unless column.null @@ -53,13 +50,13 @@ module ActiveRecord # Lists the valid migration options def migration_keys - [:name, :limit, :precision, :scale, :default, :null, :collation, :comment] + [:limit, :precision, :scale, :default, :null, :collation, :comment] end private def default_primary_key?(column) - schema_type(column) == :integer + schema_type(column) == :bigint end def schema_type(column) diff --git a/activerecord/lib/active_record/connection_adapters/abstract/schema_statements.rb b/activerecord/lib/active_record/connection_adapters/abstract/schema_statements.rb index 0bd5ec4b26..1bdc086380 100644 --- a/activerecord/lib/active_record/connection_adapters/abstract/schema_statements.rb +++ b/activerecord/lib/active_record/connection_adapters/abstract/schema_statements.rb @@ -43,7 +43,7 @@ module ActiveRecord end # Returns an array of table names defined in the database. - def tables(name = nil) + def tables raise NotImplementedError, "#tables is not implemented" end @@ -69,7 +69,9 @@ module ActiveRecord end # Returns an array of indexes for the given table. - # def indexes(table_name, name = nil) end + def indexes(table_name, name = nil) + raise NotImplementedError, "#indexes is not implemented" + end # Checks to see if an index exists on a table for a given index definition. # @@ -120,7 +122,7 @@ module ActiveRecord checks = [] checks << lambda { |c| c.name == column_name } checks << lambda { |c| c.type == type } if type - (migration_keys - [:name]).each do |attr| + migration_keys.each do |attr| checks << lambda { |c| c.send(attr) == options[attr] } if options.key?(attr) end @@ -284,10 +286,10 @@ module ActiveRecord end if supports_comments? && !supports_comments_in_create? - change_table_comment(table_name, comment) if comment + change_table_comment(table_name, comment) if comment.present? td.columns.each do |column| - change_column_comment(table_name, column.name, column.comment) if column.comment + change_column_comment(table_name, column.name, column.comment) if column.comment.present? end end @@ -511,7 +513,7 @@ module ActiveRecord # Default is (38,0). # * DB2: <tt>:precision</tt> [1..63], <tt>:scale</tt> [0..62]. # Default unknown. - # * SqlServer?: <tt>:precision</tt> [1..38], <tt>:scale</tt> [0..38]. + # * SqlServer: <tt>:precision</tt> [1..38], <tt>:scale</tt> [0..38]. # Default (38,0). # # == Examples @@ -994,17 +996,15 @@ module ActiveRecord end def insert_versions_sql(versions) # :nodoc: - sm_table = ActiveRecord::Migrator.schema_migrations_table_name + sm_table = quote_table_name(ActiveRecord::Migrator.schema_migrations_table_name) - if supports_multi_insert? + if versions.is_a?(Array) sql = "INSERT INTO #{sm_table} (version) VALUES\n" - sql << versions.map { |v| "('#{v}')" }.join(",\n") + sql << versions.map { |v| "(#{quote(v)})" }.join(",\n") sql << ";\n\n" sql else - versions.map { |version| - "INSERT INTO #{sm_table} (version) VALUES ('#{version}');" - }.join "\n\n" + "INSERT INTO #{sm_table} (version) VALUES (#{quote(versions)});" end end @@ -1034,7 +1034,7 @@ module ActiveRecord end unless migrated.include?(version) - execute "INSERT INTO #{sm_table} (version) VALUES ('#{version}')" + execute "INSERT INTO #{sm_table} (version) VALUES (#{quote(version)})" end inserting = (versions - migrated).select { |v| v < version } @@ -1042,7 +1042,13 @@ module ActiveRecord if (duplicate = inserting.detect { |v| inserting.count(v) > 1 }) raise "Duplicate migration #{duplicate}. Please renumber your migrations to resolve the conflict." end - execute insert_versions_sql(inserting) + if supports_multi_insert? + execute insert_versions_sql(inserting) + else + inserting.each do |v| + execute insert_versions_sql(v) + end + end end end @@ -1116,7 +1122,7 @@ module ActiveRecord end def add_index_options(table_name, column_name, comment: nil, **options) # :nodoc: - if column_name.is_a?(String) && /\W/ === column_name + if column_name.is_a?(String) && /\W/.match?(column_name) column_names = column_name else column_names = Array(column_name) @@ -1165,12 +1171,13 @@ module ActiveRecord raise NotImplementedError, "#{self.class} does not support changing column comments" end - protected + private def add_index_sort_order(quoted_columns, **options) if order = options[:order] case order when Hash + order = order.symbolize_keys quoted_columns.each { |name, column| column << " #{order[name].upcase}" if order[name].present? } when String quoted_columns.each { |name, column| column << " #{order.upcase}" if order.present? } @@ -1192,17 +1199,13 @@ module ActiveRecord def quoted_columns_for_index(column_names, **options) return [column_names] if column_names.is_a?(String) - quoted_columns = Hash[column_names.map { |name| [name, quote_column_name(name).dup] }] + quoted_columns = Hash[column_names.map { |name| [name.to_sym, quote_column_name(name).dup] }] add_options_for_index_columns(quoted_columns, options).values end def index_name_for_remove(table_name, options = {}) return options[:name] if can_remove_index_by_name?(options) - # if the adapter doesn't support the indexes call the best we can do - # is return the default index name for the options provided - return index_name(table_name, options) unless respond_to?(:indexes) - checks = [] if options.is_a?(Hash) @@ -1252,7 +1255,6 @@ module ActiveRecord end end - private def create_table_definition(*args) TableDefinition.new(*args) end @@ -1261,7 +1263,7 @@ module ActiveRecord AlterTable.new create_table_definition(name) end - def index_name_options(column_names) # :nodoc: + def index_name_options(column_names) if column_names.is_a?(String) column_names = column_names.scan(/\w+/).join("_") end @@ -1269,7 +1271,7 @@ module ActiveRecord { column: column_names } end - def foreign_key_name(table_name, options) # :nodoc: + def foreign_key_name(table_name, options) identifier = "#{table_name}_#{options.fetch(:column)}_fk" hashed_identifier = Digest::SHA256.hexdigest(identifier).first(10) options.fetch(:name) do @@ -1277,7 +1279,7 @@ module ActiveRecord end end - def validate_index_length!(table_name, new_name, internal = false) # :nodoc: + def validate_index_length!(table_name, new_name, internal = false) max_index_length = internal ? index_name_length : allowed_index_name_length if new_name.length > max_index_length diff --git a/activerecord/lib/active_record/connection_adapters/abstract_adapter.rb b/activerecord/lib/active_record/connection_adapters/abstract_adapter.rb index 0c7197a002..4046b3829d 100644 --- a/activerecord/lib/active_record/connection_adapters/abstract_adapter.rb +++ b/activerecord/lib/active_record/connection_adapters/abstract_adapter.rb @@ -62,17 +62,17 @@ module ActiveRecord # notably, the instance methods provided by SchemaStatements are very useful. class AbstractAdapter ADAPTER_NAME = "Abstract".freeze + include ActiveSupport::Callbacks + define_callbacks :checkout, :checkin + include Quoting, DatabaseStatements, SchemaStatements include DatabaseLimits include QueryCache - include ActiveSupport::Callbacks include ColumnDumper include Savepoints SIMPLE_INT = /\A\d+\z/ - define_callbacks :checkout, :checkin - attr_accessor :visitor, :pool attr_reader :schema_cache, :owner, :logger alias :in_use? :owner @@ -106,7 +106,7 @@ module ActiveRecord @pool = nil @schema_cache = SchemaCache.new self @quoted_column_names, @quoted_table_names = {}, {} - @visitor = arel_visitor + @visitor = arel_visitor if self.class.type_cast_config_to_boolean(config.fetch(:prepared_statements) { true }) @prepared_statements = true @@ -161,6 +161,14 @@ module ActiveRecord SchemaCreation.new self end + # Returns an array of +Column+ objects for the table specified by +table_name+. + def columns(table_name) # :nodoc: + table_name = table_name.to_s + column_definitions(table_name).map do |field| + new_column_from_field(table_name, field) + end + end + # this method must only be called while holding connection pool's mutex def lease if in_use? @@ -491,9 +499,9 @@ module ActiveRecord result end - protected + private - def initialize_type_map(m) # :nodoc: + def initialize_type_map(m) register_class_with_limit m, %r(boolean)i, Type::Boolean register_class_with_limit m, %r(char)i, Type::String register_class_with_limit m, %r(binary)i, Type::Binary @@ -524,37 +532,37 @@ module ActiveRecord end end - def reload_type_map # :nodoc: + def reload_type_map type_map.clear initialize_type_map(type_map) end - def register_class_with_limit(mapping, key, klass) # :nodoc: + def register_class_with_limit(mapping, key, klass) mapping.register_type(key) do |*args| limit = extract_limit(args.last) klass.new(limit: limit) end end - def register_class_with_precision(mapping, key, klass) # :nodoc: + def register_class_with_precision(mapping, key, klass) mapping.register_type(key) do |*args| precision = extract_precision(args.last) klass.new(precision: precision) end end - def extract_scale(sql_type) # :nodoc: + def extract_scale(sql_type) case sql_type when /\((\d+)\)/ then 0 when /\((\d+)(,(\d+))\)/ then $3.to_i end end - def extract_precision(sql_type) # :nodoc: + def extract_precision(sql_type) $1.to_i if sql_type =~ /\((\d+)(,\d+)?\)/ end - def extract_limit(sql_type) # :nodoc: + def extract_limit(sql_type) case sql_type when /^bigint/i 8 @@ -575,7 +583,7 @@ module ActiveRecord exception end - def log(sql, name = "SQL", binds = [], type_casted_binds = [], statement_name = nil) + def log(sql, name = "SQL", binds = [], type_casted_binds = [], statement_name = nil) # :doc: @instrumenter.instrument( "sql.active_record", sql: sql, @@ -590,14 +598,19 @@ module ActiveRecord def translate_exception(exception, message) # override in derived class - ActiveRecord::StatementInvalid.new(message) + case exception + when RuntimeError + exception + else + ActiveRecord::StatementInvalid.new(message) + end end def without_prepared_statement?(binds) !prepared_statements || binds.empty? end - def column_for(table_name, column_name) # :nodoc: + def column_for(table_name, column_name) column_name = column_name.to_s columns(table_name).detect { |c| c.name == column_name } || raise(ActiveRecordError, "No such column: #{table_name}.#{column_name}") diff --git a/activerecord/lib/active_record/connection_adapters/abstract_mysql_adapter.rb b/activerecord/lib/active_record/connection_adapters/abstract_mysql_adapter.rb index d0ea1ce0cf..fbc510bc0e 100644 --- a/activerecord/lib/active_record/connection_adapters/abstract_mysql_adapter.rb +++ b/activerecord/lib/active_record/connection_adapters/abstract_mysql_adapter.rb @@ -9,7 +9,6 @@ require "active_record/connection_adapters/mysql/schema_dumper" require "active_record/connection_adapters/mysql/type_metadata" require "active_support/core_ext/string/strip" -require "active_support/core_ext/regexp" module ActiveRecord module ConnectionAdapters @@ -40,7 +39,7 @@ module ActiveRecord self.emulate_booleans = true NATIVE_DATABASE_TYPES = { - primary_key: "int auto_increment PRIMARY KEY", + primary_key: "bigint auto_increment PRIMARY KEY", string: { name: "varchar", limit: 255 }, text: { name: "text", limit: 65535 }, integer: { name: "int", limit: 4 }, @@ -68,8 +67,8 @@ module ActiveRecord @statements = StatementPool.new(self.class.type_cast_config_to_integer(config[:statement_limit])) - if version < "5.0.0" - raise "Your version of MySQL (#{full_version.match(/^\d+\.\d+\.\d+/)[0]}) is too old. Active Record supports MySQL >= 5.0." + if version < "5.1.10" + raise "Your version of MySQL (#{full_version.match(/^\d+\.\d+\.\d+/)[0]}) is too old. Active Record supports MySQL >= 5.1.10." end end @@ -216,7 +215,11 @@ module ActiveRecord # Executes the SQL statement in the context of this connection. def execute(sql, name = nil) - log(sql, name) { @connection.query(sql) } + log(sql, name) do + ActiveSupport::Dependencies.interlock.permit_concurrent_loads do + @connection.query(sql) + end + end end # Mysql2Adapter doesn't have to free a result after using it, but we use this method @@ -307,45 +310,36 @@ module ActiveRecord show_variable "collation_database" end - def tables(name = nil) # :nodoc: - ActiveSupport::Deprecation.warn(<<-MSG.squish) - #tables currently returns both tables and views. - This behavior is deprecated and will be changed with Rails 5.1 to only return tables. - Use #data_sources instead. - MSG + def tables # :nodoc: + sql = "SELECT table_name FROM information_schema.tables WHERE table_type = 'BASE TABLE'" + sql << " AND table_schema = #{quote(@config[:database])}" - if name - ActiveSupport::Deprecation.warn(<<-MSG.squish) - Passing arguments to #tables is deprecated without replacement. - MSG - end + select_values(sql, "SCHEMA") + end - data_sources + def views # :nodoc: + select_values("SHOW FULL TABLES WHERE table_type = 'VIEW'", "SCHEMA") end - def data_sources + def data_sources # :nodoc: sql = "SELECT table_name FROM information_schema.tables " sql << "WHERE table_schema = #{quote(@config[:database])}" select_values(sql, "SCHEMA") end - def truncate(table_name, name = nil) - execute "TRUNCATE TABLE #{quote_table_name(table_name)}", name - end + def table_exists?(table_name) # :nodoc: + return false unless table_name.present? - def table_exists?(table_name) - # Update lib/active_record/internal_metadata.rb when this gets removed - ActiveSupport::Deprecation.warn(<<-MSG.squish) - #table_exists? currently checks both tables and views. - This behavior is deprecated and will be changed with Rails 5.1 to only check tables. - Use #data_source_exists? instead. - MSG + schema, name = extract_schema_qualified_name(table_name) + + sql = "SELECT table_name FROM information_schema.tables WHERE table_type = 'BASE TABLE'" + sql << " AND table_schema = #{quote(schema)} AND table_name = #{quote(name)}" - data_source_exists?(table_name) + select_values(sql, "SCHEMA").any? end - def data_source_exists?(table_name) + def data_source_exists?(table_name) # :nodoc: return false unless table_name.present? schema, name = extract_schema_qualified_name(table_name) @@ -356,10 +350,6 @@ module ActiveRecord select_values(sql, "SCHEMA").any? end - def views # :nodoc: - select_values("SHOW FULL TABLES WHERE table_type = 'VIEW'", "SCHEMA") - end - def view_exists?(view_name) # :nodoc: return false unless view_name.present? @@ -371,8 +361,18 @@ module ActiveRecord select_values(sql, "SCHEMA").any? end + def truncate(table_name, name = nil) + execute "TRUNCATE TABLE #{quote_table_name(table_name)}", name + end + # Returns an array of indexes for the given table. def indexes(table_name, name = nil) #:nodoc: + if name + ActiveSupport::Deprecation.warn(<<-MSG.squish) + Passing name to #indexes is deprecated without replacement. + MSG + end + indexes = [] current_index = nil execute_and_free("SHOW KEYS FROM #{quote_table_name(table_name)}", "SCHEMA") do |result| @@ -384,29 +384,25 @@ module ActiveRecord mysql_index_type = row[:Index_type].downcase.to_sym index_type = INDEX_TYPES.include?(mysql_index_type) ? mysql_index_type : nil index_using = INDEX_USINGS.include?(mysql_index_type) ? mysql_index_type : nil - indexes << IndexDefinition.new(row[:Table], row[:Key_name], row[:Non_unique].to_i == 0, [], [], nil, nil, index_type, index_using, row[:Index_comment].presence) + indexes << IndexDefinition.new(row[:Table], row[:Key_name], row[:Non_unique].to_i == 0, [], {}, nil, nil, index_type, index_using, row[:Index_comment].presence) end indexes.last.columns << row[:Column_name] - indexes.last.lengths << row[:Sub_part] + indexes.last.lengths.merge!(row[:Column_name] => row[:Sub_part].to_i) if row[:Sub_part] end end indexes end - # Returns an array of +Column+ objects for the table specified by +table_name+. - def columns(table_name) # :nodoc: - table_name = table_name.to_s - column_definitions(table_name).map do |field| - type_metadata = fetch_type_metadata(field[:Type], field[:Extra]) - if type_metadata.type == :datetime && field[:Default] == "CURRENT_TIMESTAMP" - default, default_function = nil, field[:Default] - else - default, default_function = field[:Default], nil - end - new_column(field[:Field], default, type_metadata, field[:Null] == "YES", table_name, default_function, field[:Collation], comment: field[:Comment].presence) + def new_column_from_field(table_name, field) # :nodoc: + type_metadata = fetch_type_metadata(field[:Type], field[:Extra]) + if type_metadata.type == :datetime && field[:Default] == "CURRENT_TIMESTAMP" + default, default_function = nil, field[:Default] + else + default, default_function = field[:Default], nil end + new_column(field[:Field], default, type_metadata, field[:Null] == "YES", table_name, default_function, field[:Collation], comment: field[:Comment].presence) end def table_comment(table_name) # :nodoc: @@ -509,7 +505,7 @@ module ActiveRecord end def add_sql_comment!(sql, comment) # :nodoc: - sql << " COMMENT #{quote(comment)}" if comment + sql << " COMMENT #{quote(comment)}" if comment.present? sql end @@ -650,9 +646,9 @@ module ActiveRecord !native_database_types[type].nil? end - protected + private - def initialize_type_map(m) # :nodoc: + def initialize_type_map(m) super register_class_with_limit m, %r(char)i, MysqlString @@ -692,9 +688,9 @@ module ActiveRecord end end - def register_integer_type(mapping, key, options) # :nodoc: + def register_integer_type(mapping, key, options) mapping.register_type(key) do |sql_type| - if /\bunsigned\z/ === sql_type + if /\bunsigned\b/.match?(sql_type) Type::UnsignedInteger.new(options) else Type::Integer.new(options) @@ -703,7 +699,7 @@ module ActiveRecord end def extract_precision(sql_type) - if /time/ === sql_type + if /time/.match?(sql_type) super || 0 else super @@ -718,6 +714,7 @@ module ActiveRecord if length = options[:length] case length when Hash + length = length.symbolize_keys quoted_columns.each { |name, column| column << "(#{length[name]})" if length[name].present? } when Integer quoted_columns.each { |name, column| column << "(#{length})" } @@ -734,9 +731,14 @@ module ActiveRecord # See https://dev.mysql.com/doc/refman/5.7/en/error-messages-server.html ER_DUP_ENTRY = 1062 + ER_NOT_NULL_VIOLATION = 1048 + ER_DO_NOT_HAVE_DEFAULT = 1364 ER_NO_REFERENCED_ROW_2 = 1452 ER_DATA_TOO_LONG = 1406 + ER_OUT_OF_RANGE = 1264 ER_LOCK_DEADLOCK = 1213 + ER_CANNOT_ADD_FOREIGN = 1215 + ER_CANNOT_CREATE_TABLE = 1005 def translate_exception(exception, message) case error_number(exception) @@ -744,8 +746,20 @@ module ActiveRecord RecordNotUnique.new(message) when ER_NO_REFERENCED_ROW_2 InvalidForeignKey.new(message) + when ER_CANNOT_ADD_FOREIGN + mismatched_foreign_key(message) + when ER_CANNOT_CREATE_TABLE + if message.include?("errno: 150") + mismatched_foreign_key(message) + else + super + end when ER_DATA_TOO_LONG ValueTooLong.new(message) + when ER_OUT_OF_RANGE + RangeError.new(message) + when ER_NOT_NULL_VIOLATION, ER_DO_NOT_HAVE_DEFAULT + NotNullViolation.new(message) when ER_LOCK_DEADLOCK Deadlocked.new(message) else @@ -770,6 +784,10 @@ module ActiveRecord options[:null] = column.null end + unless options.key?(:comment) + options[:comment] = column.comment + end + td = create_table_definition(table_name) cd = td.new_column_definition(column.name, type, options) schema_creation.accept(ChangeColumnDefinition.new(cd, column.name)) @@ -816,10 +834,8 @@ module ActiveRecord [remove_column_sql(table_name, :updated_at), remove_column_sql(table_name, :created_at)] end - private - - # MySQL is too stupid to create a temporary table for use subquery, so we have - # to give it some prompting in the form of a subsubquery. Ugh! + # MySQL is too stupid to create a temporary table for use subquery, so we have + # to give it some prompting in the form of a subsubquery. Ugh! def subquery_for(key, select) subsubselect = select.clone subsubselect.projections = [key] @@ -887,7 +903,7 @@ module ActiveRecord end.compact.join(", ") # ...and send them all in one query - @connection.query "SET #{encoding} #{sql_mode_assignment} #{variable_assignments}" + execute "SET #{encoding} #{sql_mode_assignment} #{variable_assignments}" end def column_definitions(table_name) # :nodoc: @@ -911,6 +927,18 @@ module ActiveRecord MySQL::TableDefinition.new(*args) end + def mismatched_foreign_key(message) + parts = message.scan(/`(\w+)`[ $)]/).flatten + MismatchedForeignKey.new( + self, + message: message, + table: parts[0], + foreign_key: parts[1], + target_table: parts[2], + primary_key: parts[3], + ) + end + def extract_schema_qualified_name(string) # :nodoc: schema, name = string.to_s.scan(/[^`.\s]+|`[^`]*`/) schema, name = @config[:database], schema unless name diff --git a/activerecord/lib/active_record/connection_adapters/column.rb b/activerecord/lib/active_record/connection_adapters/column.rb index 1808173592..61cd7ae4cc 100644 --- a/activerecord/lib/active_record/connection_adapters/column.rb +++ b/activerecord/lib/active_record/connection_adapters/column.rb @@ -29,7 +29,7 @@ module ActiveRecord end def bigint? - /\Abigint\b/ === sql_type + /\Abigint\b/.match?(sql_type) end # Returns the human name of the column name. @@ -40,6 +40,28 @@ module ActiveRecord Base.human_attribute_name(@name) end + def init_with(coder) + @name = coder["name"] + @table_name = coder["table_name"] + @sql_type_metadata = coder["sql_type_metadata"] + @null = coder["null"] + @default = coder["default"] + @default_function = coder["default_function"] + @collation = coder["collation"] + @comment = coder["comment"] + end + + def encode_with(coder) + coder["name"] = @name + coder["table_name"] = @table_name + coder["sql_type_metadata"] = @sql_type_metadata + coder["null"] = @null + coder["default"] = @default + coder["default_function"] = @default_function + coder["collation"] = @collation + coder["comment"] = @comment + end + def ==(other) other.is_a?(Column) && attributes_for_hash == other.attributes_for_hash diff --git a/activerecord/lib/active_record/connection_adapters/connection_specification.rb b/activerecord/lib/active_record/connection_adapters/connection_specification.rb index be6b55e53c..dcf56997db 100644 --- a/activerecord/lib/active_record/connection_adapters/connection_specification.rb +++ b/activerecord/lib/active_record/connection_adapters/connection_specification.rb @@ -48,8 +48,8 @@ module ActiveRecord # Converts the given URL to a full connection hash. def to_hash - config = raw_config.reject { |_,value| value.blank? } - config.map { |key,value| config[key] = uri_parser.unescape(value) if value.is_a? String } + config = raw_config.reject { |_, value| value.blank? } + config.map { |key, value| config[key] = uri_parser.unescape(value) if value.is_a? String } config end @@ -63,15 +63,15 @@ module ActiveRecord @uri_parser ||= URI::Parser.new end - # Converts the query parameters of the URI into a hash. - # - # "localhost?pool=5&reaping_frequency=2" - # # => { "pool" => "5", "reaping_frequency" => "2" } - # - # returns empty hash if no query present. - # - # "localhost" - # # => {} + # Converts the query parameters of the URI into a hash. + # + # "localhost?pool=5&reaping_frequency=2" + # # => { "pool" => "5", "reaping_frequency" => "2" } + # + # returns empty hash if no query present. + # + # "localhost" + # # => {} def query_hash Hash[(@query || "").split("&").map { |pair| pair.split("=") }] end @@ -92,7 +92,7 @@ module ActiveRecord end end - # Returns name of the database. + # Returns name of the database. def database_from_path if @adapter == "sqlite3" # 'sqlite3:/foo' is absolute, because that makes sense. The @@ -192,26 +192,26 @@ module ActiveRecord private - # Returns fully resolved connection, accepts hash, string or symbol. - # Always returns a hash. - # - # == Examples - # - # Symbol representing current environment. - # - # Resolver.new("production" => {}).resolve_connection(:production) - # # => {} - # - # One layer deep hash of connection values. - # - # Resolver.new({}).resolve_connection("adapter" => "sqlite3") - # # => { "adapter" => "sqlite3" } - # - # Connection URL. - # - # Resolver.new({}).resolve_connection("postgresql://localhost/foo") - # # => { "host" => "localhost", "database" => "foo", "adapter" => "postgresql" } - # + # Returns fully resolved connection, accepts hash, string or symbol. + # Always returns a hash. + # + # == Examples + # + # Symbol representing current environment. + # + # Resolver.new("production" => {}).resolve_connection(:production) + # # => {} + # + # One layer deep hash of connection values. + # + # Resolver.new({}).resolve_connection("adapter" => "sqlite3") + # # => { "adapter" => "sqlite3" } + # + # Connection URL. + # + # Resolver.new({}).resolve_connection("postgresql://localhost/foo") + # # => { "host" => "localhost", "database" => "foo", "adapter" => "postgresql" } + # def resolve_connection(spec) case spec when Symbol @@ -223,13 +223,13 @@ module ActiveRecord end end - # Takes the environment such as +:production+ or +:development+. - # This requires that the @configurations was initialized with a key that - # matches. - # - # Resolver.new("production" => {}).resolve_symbol_connection(:production) - # # => {} - # + # Takes the environment such as +:production+ or +:development+. + # This requires that the @configurations was initialized with a key that + # matches. + # + # Resolver.new("production" => {}).resolve_symbol_connection(:production) + # # => {} + # def resolve_symbol_connection(spec) if config = configurations[spec.to_s] resolve_connection(config).merge("name" => spec.to_s) @@ -238,10 +238,10 @@ module ActiveRecord end end - # Accepts a hash. Expands the "url" key that contains a - # URL database connection to a full connection - # hash and merges with the rest of the hash. - # Connection details inside of the "url" key win any merge conflicts + # Accepts a hash. Expands the "url" key that contains a + # URL database connection to a full connection + # hash and merges with the rest of the hash. + # Connection details inside of the "url" key win any merge conflicts def resolve_hash_connection(spec) if spec["url"] && spec["url"] !~ /^jdbc:/ connection_hash = resolve_url_connection(spec.delete("url")) @@ -250,11 +250,11 @@ module ActiveRecord spec end - # Takes a connection URL. - # - # Resolver.new({}).resolve_url_connection("postgresql://localhost/foo") - # # => { "host" => "localhost", "database" => "foo", "adapter" => "postgresql" } - # + # Takes a connection URL. + # + # Resolver.new({}).resolve_url_connection("postgresql://localhost/foo") + # # => { "host" => "localhost", "database" => "foo", "adapter" => "postgresql" } + # def resolve_url_connection(url) ConnectionUrlResolver.new(url).to_hash end diff --git a/activerecord/lib/active_record/connection_adapters/mysql/column.rb b/activerecord/lib/active_record/connection_adapters/mysql/column.rb index 296d9a15f8..1499c1681f 100644 --- a/activerecord/lib/active_record/connection_adapters/mysql/column.rb +++ b/activerecord/lib/active_record/connection_adapters/mysql/column.rb @@ -5,11 +5,11 @@ module ActiveRecord delegate :extra, to: :sql_type_metadata, allow_nil: true def unsigned? - /\bunsigned\z/ === sql_type + /\bunsigned(?: zerofill)?\z/.match?(sql_type) end def case_sensitive? - collation && collation !~ /_ci\z/ + collation && !/_ci\z/.match?(collation) end def auto_increment? diff --git a/activerecord/lib/active_record/connection_adapters/mysql/database_statements.rb b/activerecord/lib/active_record/connection_adapters/mysql/database_statements.rb index 56800f7590..8c67a7a80b 100644 --- a/activerecord/lib/active_record/connection_adapters/mysql/database_statements.rb +++ b/activerecord/lib/active_record/connection_adapters/mysql/database_statements.rb @@ -3,7 +3,7 @@ module ActiveRecord module MySQL module DatabaseStatements # Returns an ActiveRecord::Result instance. - def select_all(arel, name = nil, binds = [], preparable: nil) + def select_all(arel, name = nil, binds = [], preparable: nil) # :nodoc: result = if ExplainRegistry.collect? && prepared_statements unprepared_statement { super } else @@ -15,8 +15,8 @@ module ActiveRecord # Returns an array of arrays containing the field values. # Order is the same as that returned by +columns+. - def select_rows(sql, name = nil, binds = []) - select_result(sql, name, binds) do |result| + def select_rows(arel, name = nil, binds = []) # :nodoc: + select_result(arel, name, binds) do |result| @connection.next_result while @connection.more_results? result.to_a end @@ -52,15 +52,15 @@ module ActiveRecord end alias :exec_update :exec_delete - protected + private def last_inserted_id(result) @connection.last_id end - private - - def select_result(sql, name = nil, binds = []) + def select_result(arel, name, binds) + arel, binds = binds_from_relation(arel, binds) + sql = to_sql(arel, binds) if without_prepared_statement?(binds) execute_and_free(sql, name) { |result| yield result } else @@ -86,7 +86,9 @@ module ActiveRecord end begin - result = stmt.execute(*type_casted_binds) + result = ActiveSupport::Dependencies.interlock.permit_concurrent_loads do + stmt.execute(*type_casted_binds) + end rescue Mysql2::Error => e if cache_stmt @statements.delete(sql) diff --git a/activerecord/lib/active_record/connection_adapters/mysql/explain_pretty_printer.rb b/activerecord/lib/active_record/connection_adapters/mysql/explain_pretty_printer.rb index 925555703d..9691060cd3 100644 --- a/activerecord/lib/active_record/connection_adapters/mysql/explain_pretty_printer.rb +++ b/activerecord/lib/active_record/connection_adapters/mysql/explain_pretty_printer.rb @@ -47,7 +47,7 @@ module ActiveRecord def build_separator(widths) padding = 1 - "+" + widths.map { |w| "-" * (w + (padding*2)) }.join("+") + "+" + "+" + widths.map { |w| "-" * (w + (padding * 2)) }.join("+") + "+" end def build_cells(items, widths) diff --git a/activerecord/lib/active_record/connection_adapters/mysql/schema_definitions.rb b/activerecord/lib/active_record/connection_adapters/mysql/schema_definitions.rb index ce773ed75b..0cf40de70f 100644 --- a/activerecord/lib/active_record/connection_adapters/mysql/schema_definitions.rb +++ b/activerecord/lib/active_record/connection_adapters/mysql/schema_definitions.rb @@ -3,7 +3,10 @@ module ActiveRecord module MySQL module ColumnMethods def primary_key(name, type = :primary_key, **options) - options[:auto_increment] = true if type == :bigint && !options.key?(:default) + if type == :primary_key && !options.key?(:default) + options[:auto_increment] = true + options[:limit] = 8 + end super end diff --git a/activerecord/lib/active_record/connection_adapters/mysql/schema_dumper.rb b/activerecord/lib/active_record/connection_adapters/mysql/schema_dumper.rb index 39221eeb0c..2065816501 100644 --- a/activerecord/lib/active_record/connection_adapters/mysql/schema_dumper.rb +++ b/activerecord/lib/active_record/connection_adapters/mysql/schema_dumper.rb @@ -3,11 +3,9 @@ module ActiveRecord module MySQL module ColumnDumper def column_spec_for_primary_key(column) - if column.bigint? - spec = { id: :bigint.inspect } - spec[:default] = schema_default(column) || "nil" unless column.auto_increment? - else - spec = super + spec = super + if column.type == :integer && !column.auto_increment? + spec[:default] = schema_default(column) || "nil" end spec[:unsigned] = "true" if column.unsigned? spec @@ -38,7 +36,7 @@ module ActiveRecord end def schema_precision(column) - super unless /time/ === column.sql_type && column.precision == 0 + super unless /time/.match?(column.sql_type) && column.precision == 0 end def schema_collation(column) diff --git a/activerecord/lib/active_record/connection_adapters/mysql2_adapter.rb b/activerecord/lib/active_record/connection_adapters/mysql2_adapter.rb index a3e2c913c5..45e400b75b 100644 --- a/activerecord/lib/active_record/connection_adapters/mysql2_adapter.rb +++ b/activerecord/lib/active_record/connection_adapters/mysql2_adapter.rb @@ -14,12 +14,10 @@ module ActiveRecord config[:username] = "root" if config[:username].nil? config[:flags] ||= 0 - if Mysql2::Client.const_defined? :FOUND_ROWS - if config[:flags].kind_of? Array - config[:flags].push "FOUND_ROWS".freeze - else - config[:flags] |= Mysql2::Client::FOUND_ROWS - end + if config[:flags].kind_of? Array + config[:flags].push "FOUND_ROWS".freeze + else + config[:flags] |= Mysql2::Client::FOUND_ROWS end client = Mysql2::Client.new(config) diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/database_statements.rb b/activerecord/lib/active_record/connection_adapters/postgresql/database_statements.rb index 7414eba6c5..705e6063dc 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql/database_statements.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql/database_statements.rb @@ -7,18 +7,14 @@ module ActiveRecord PostgreSQL::ExplainPrettyPrinter.new.pp(exec_query(sql, "EXPLAIN", binds)) end - def select_value(arel, name = nil, binds = []) - arel, binds = binds_from_relation arel, binds - sql = to_sql(arel, binds) - execute_and_clear(sql, name, binds) do |result| + def select_value(arel, name = nil, binds = []) # :nodoc: + select_result(arel, name, binds) do |result| result.getvalue(0, 0) if result.ntuples > 0 && result.nfields > 0 end end - def select_values(arel, name = nil, binds = []) - arel, binds = binds_from_relation arel, binds - sql = to_sql(arel, binds) - execute_and_clear(sql, name, binds) do |result| + def select_values(arel, name = nil, binds = []) # :nodoc: + select_result(arel, name, binds) do |result| if result.nfields > 0 result.column_values(0) else @@ -29,8 +25,8 @@ module ActiveRecord # Executes a SELECT query and returns an array of rows. Each row is an # array of field values. - def select_rows(sql, name = nil, binds = []) - execute_and_clear(sql, name, binds) do |result| + def select_rows(arel, name = nil, binds = []) # :nodoc: + select_result(arel, name, binds) do |result| result.values end end @@ -85,17 +81,21 @@ module ActiveRecord # Queries the database and returns the results in an Array-like object def query(sql, name = nil) #:nodoc: log(sql, name) do - result_as_array @connection.async_exec(sql) + ActiveSupport::Dependencies.interlock.permit_concurrent_loads do + result_as_array @connection.async_exec(sql) + end end end - # Executes an SQL statement, returning a PGresult object on success - # or raising a PGError exception otherwise. - # Note: the PGresult object is manually memory managed; if you don't - # need it specifically, you many want consider the exec_query wrapper. + # Executes an SQL statement, returning a PG::Result object on success + # or raising a PG::Error exception otherwise. + # Note: the PG::Result object is manually memory managed; if you don't + # need it specifically, you may want consider the <tt>exec_query</tt> wrapper. def execute(sql, name = nil) log(sql, name) do - @connection.async_exec(sql) + ActiveSupport::Dependencies.interlock.permit_concurrent_loads do + @connection.async_exec(sql) + end end end @@ -130,7 +130,7 @@ module ActiveRecord super end - protected :sql_for_insert + private :sql_for_insert def exec_insert(sql, name = nil, binds = [], pk = nil, sequence_name = nil) if use_insert_returning? || pk == false @@ -175,6 +175,14 @@ module ActiveRecord def suppress_composite_primary_key(pk) pk unless pk.is_a?(Array) end + + def select_result(arel, name, binds) + arel, binds = binds_from_relation(arel, binds) + sql = to_sql(arel, binds) + execute_and_clear(sql, name, binds) do |result| + yield result + end + end end end end diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/oid/array.rb b/activerecord/lib/active_record/connection_adapters/postgresql/oid/array.rb index b969503178..e1a75f8e5e 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql/oid/array.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql/oid/array.rb @@ -5,8 +5,10 @@ module ActiveRecord class Array < Type::Value # :nodoc: include Type::Helpers::Mutable + Data = Struct.new(:encoder, :values) # :nodoc: + attr_reader :subtype, :delimiter - delegate :type, :user_input_in_time_zone, :limit, to: :subtype + delegate :type, :user_input_in_time_zone, :limit, :precision, :scale, to: :subtype def initialize(subtype, delimiter = ",") @subtype = subtype @@ -17,8 +19,11 @@ module ActiveRecord end def deserialize(value) - if value.is_a?(::String) + case value + when ::String type_cast_array(@pg_decoder.decode(value), :deserialize) + when Data + deserialize(value.values) else super end @@ -33,11 +38,8 @@ module ActiveRecord def serialize(value) if value.is_a?(::Array) - result = @pg_encoder.encode(type_cast_array(value, :serialize)) - if encoding = determine_encoding_of_strings(value) - result.encode!(encoding) - end - result + casted_values = type_cast_array(value, :serialize) + Data.new(@pg_encoder, casted_values) else super end @@ -58,6 +60,10 @@ module ActiveRecord value.map(&block) end + def changed_in_place?(raw_old_value, new_value) + deserialize(raw_old_value) != new_value + end + private def type_cast_array(value, method) @@ -67,13 +73,6 @@ module ActiveRecord @subtype.public_send(method, value) end end - - def determine_encoding_of_strings(value) - case value - when ::Array then determine_encoding_of_strings(value.first) - when ::String then value.encoding - end - end end end end diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/oid/bit.rb b/activerecord/lib/active_record/connection_adapters/postgresql/oid/bit.rb index 74bff229ea..0a505f46a7 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql/oid/bit.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql/oid/bit.rb @@ -34,13 +34,15 @@ module ActiveRecord end def binary? - /\A[01]*\Z/ === value + /\A[01]*\Z/.match?(value) end def hex? - /\A[0-9A-F]*\Z/i === value + /\A[0-9A-F]*\Z/i.match?(value) end + # TODO Change this to private once we've dropped Ruby 2.2 support. + # Workaround for Ruby 2.2 "private attribute?" warning. protected attr_reader :value diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/oid/hstore.rb b/activerecord/lib/active_record/connection_adapters/postgresql/oid/hstore.rb index 2d3e6a925d..49dd4fc73f 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql/oid/hstore.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql/oid/hstore.rb @@ -12,8 +12,8 @@ module ActiveRecord def deserialize(value) if value.is_a?(::String) ::Hash[value.scan(HstorePair).map { |k, v| - v = v.upcase == "NULL" ? nil : v.gsub(/\A"(.*)"\Z/m,'\1').gsub(/\\(.)/, '\1') - k = k.gsub(/\A"(.*)"\Z/m,'\1').gsub(/\\(.)/, '\1') + v = v.upcase == "NULL" ? nil : v.gsub(/\A"(.*)"\Z/m, '\1').gsub(/\\(.)/, '\1') + k = k.gsub(/\A"(.*)"\Z/m, '\1').gsub(/\\(.)/, '\1') [k, v] }] else @@ -24,6 +24,8 @@ module ActiveRecord def serialize(value) if value.is_a?(::Hash) value.map { |k, v| "#{escape_hstore(k)}=>#{escape_hstore(v)}" }.join(", ") + elsif value.respond_to?(:to_unsafe_h) + serialize(value.to_unsafe_h) else value end @@ -33,6 +35,14 @@ module ActiveRecord ActiveRecord::Store::StringKeyedHashAccessor end + # Will compare the Hash equivalents of +raw_old_value+ and +new_value+. + # By comparing hashes, this avoids an edge case where the order of + # the keys change between the two hashes, and they would not be marked + # as equal. + def changed_in_place?(raw_old_value, new_value) + deserialize(raw_old_value) != new_value + end + private HstorePair = begin diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/oid/range.rb b/activerecord/lib/active_record/connection_adapters/postgresql/oid/range.rb index 2c714f4018..54d5d0902e 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql/oid/range.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql/oid/range.rb @@ -1,5 +1,3 @@ -require "active_support/core_ext/string/filters" - module ActiveRecord module ConnectionAdapters module PostgreSQL diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/quoting.rb b/activerecord/lib/active_record/connection_adapters/postgresql/quoting.rb index b5031d890f..3783925954 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql/quoting.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql/quoting.rb @@ -92,6 +92,8 @@ module ActiveRecord else super end + when OID::Array::Data + _quote(encode_array(value)) else super end @@ -106,10 +108,37 @@ module ActiveRecord { value: value.to_s, format: 1 } when OID::Xml::Data, OID::Bit::Data value.to_s + when OID::Array::Data + encode_array(value) else super end end + + def encode_array(array_data) + encoder = array_data.encoder + values = type_cast_array(array_data.values) + + result = encoder.encode(values) + if encoding = determine_encoding_of_strings_in_array(values) + result.force_encoding(encoding) + end + result + end + + def determine_encoding_of_strings_in_array(value) + case value + when ::Array then determine_encoding_of_strings_in_array(value.first) + when ::String then value.encoding + end + end + + def type_cast_array(values) + case values + when ::Array then values.map { |item| type_cast_array(item) } + else _type_cast(values) + end + end end end end diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/schema_definitions.rb b/activerecord/lib/active_record/connection_adapters/postgresql/schema_definitions.rb index a11dbe7dce..4afb4733eb 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql/schema_definitions.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql/schema_definitions.rb @@ -11,11 +11,22 @@ module ActiveRecord # t.timestamps # end # - # By default, this will use the +uuid_generate_v4()+ function from the - # +uuid-ossp+ extension, which MUST be enabled on your database. To enable - # the +uuid-ossp+ extension, you can use the +enable_extension+ method in your - # migrations. To use a UUID primary key without +uuid-ossp+ enabled, you can - # set the +:default+ option to +nil+: + # By default, this will use the +gen_random_uuid()+ function from the + # +pgcrypto+ extension. As that extension is only available in + # PostgreSQL 9.4+, for earlier versions an explicit default can be set + # to use +uuid_generate_v4()+ from the +uuid-ossp+ extension instead: + # + # create_table :stuffs, id: false do |t| + # t.primary_key :id, :uuid, default: "uuid_generate_v4()" + # t.uuid :foo_id + # t.timestamps + # end + # + # To enable the appropriate extension, which is a requirement, use + # the +enable_extension+ method in your migrations. + # + # To use a UUID primary key without any of the extensions, set the + # +:default+ option to +nil+: # # create_table :stuffs, id: false do |t| # t.primary_key :id, :uuid, default: nil @@ -23,15 +34,24 @@ module ActiveRecord # t.timestamps # end # - # You may also pass a different UUID generation function from +uuid-ossp+ - # or another library. + # You may also pass a custom stored procedure that returns a UUID or use a + # different UUID generation function from another library. # # Note that setting the UUID primary key default value to +nil+ will # require you to assure that you always provide a UUID value before saving # a record (as primary keys cannot be +nil+). This might be done via the # +SecureRandom.uuid+ method and a +before_save+ callback, for instance. def primary_key(name, type = :primary_key, **options) - options[:default] = options.fetch(:default, "uuid_generate_v4()") if type == :uuid + if type == :uuid + options[:default] = options.fetch(:default, "gen_random_uuid()") + elsif options.delete(:auto_increment) == true && %i(integer bigint).include?(type) + type = if type == :bigint || options[:limit] == 8 + :bigserial + else + :serial + end + end + super end diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/schema_dumper.rb b/activerecord/lib/active_record/connection_adapters/postgresql/schema_dumper.rb index c20baf655c..7808d37deb 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql/schema_dumper.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql/schema_dumper.rb @@ -25,7 +25,7 @@ module ActiveRecord private def default_primary_key?(column) - schema_type(column) == :serial + schema_type(column) == :bigserial end def schema_type(column) diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/schema_statements.rb b/activerecord/lib/active_record/connection_adapters/postgresql/schema_statements.rb index 29a77580f5..bfda113e40 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql/schema_statements.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql/schema_statements.rb @@ -71,13 +71,7 @@ module ActiveRecord end # Returns the list of all tables in the schema search path. - def tables(name = nil) - if name - ActiveSupport::Deprecation.warn(<<-MSG.squish) - Passing arguments to #tables is deprecated without replacement. - MSG - end - + def tables select_values("SELECT tablename FROM pg_tables WHERE schemaname = ANY(current_schemas(false))", "SCHEMA") end @@ -86,7 +80,17 @@ module ActiveRecord SELECT c.relname FROM pg_class c LEFT JOIN pg_namespace n ON n.oid = c.relnamespace - WHERE c.relkind IN ('r', 'v','m') -- (r)elation/table, (v)iew, (m)aterialized view + WHERE c.relkind IN ('r','v','m') -- (r)elation/table, (v)iew, (m)aterialized view + AND n.nspname = ANY (current_schemas(false)) + SQL + end + + def views # :nodoc: + select_values(<<-SQL, "SCHEMA") + SELECT c.relname + FROM pg_class c + LEFT JOIN pg_namespace n ON n.oid = c.relnamespace + WHERE c.relkind IN ('v','m') -- (v)iew, (m)aterialized view AND n.nspname = ANY (current_schemas(false)) SQL end @@ -95,36 +99,28 @@ module ActiveRecord # If the schema is not specified as part of +name+ then it will only find tables within # the current schema search path (regardless of permissions to access tables in other schemas) def table_exists?(name) - ActiveSupport::Deprecation.warn(<<-MSG.squish) - #table_exists? currently checks both tables and views. - This behavior is deprecated and will be changed with Rails 5.1 to only check tables. - Use #data_source_exists? instead. - MSG - - data_source_exists?(name) - end - - def data_source_exists?(name) name = Utils.extract_schema_qualified_name(name.to_s) return false unless name.identifier - select_value(<<-SQL, "SCHEMA").to_i > 0 - SELECT COUNT(*) - FROM pg_class c - LEFT JOIN pg_namespace n ON n.oid = c.relnamespace - WHERE c.relkind IN ('r','v','m') -- (r)elation/table, (v)iew, (m)aterialized view - AND c.relname = '#{name.identifier}' - AND n.nspname = #{name.schema ? "'#{name.schema}'" : 'ANY (current_schemas(false))'} + select_values(<<-SQL, "SCHEMA").any? + SELECT tablename + FROM pg_tables + WHERE tablename = #{quote(name.identifier)} + AND schemaname = #{name.schema ? quote(name.schema) : "ANY (current_schemas(false))"} SQL end - def views # :nodoc: - select_values(<<-SQL, "SCHEMA") + def data_source_exists?(name) # :nodoc: + name = Utils.extract_schema_qualified_name(name.to_s) + return false unless name.identifier + + select_values(<<-SQL, "SCHEMA").any? SELECT c.relname FROM pg_class c LEFT JOIN pg_namespace n ON n.oid = c.relnamespace - WHERE c.relkind IN ('v','m') -- (v)iew, (m)aterialized view - AND n.nspname = ANY (current_schemas(false)) + WHERE c.relkind IN ('r','v','m') -- (r)elation/table, (v)iew, (m)aterialized view + AND c.relname = #{quote(name.identifier)} + AND n.nspname = #{name.schema ? quote(name.schema) : "ANY (current_schemas(false))"} SQL end @@ -137,8 +133,8 @@ module ActiveRecord FROM pg_class c LEFT JOIN pg_namespace n ON n.oid = c.relnamespace WHERE c.relkind IN ('v','m') -- (v)iew, (m)aterialized view - AND c.relname = '#{name.identifier}' - AND n.nspname = #{name.schema ? "'#{name.schema}'" : 'ANY (current_schemas(false))'} + AND c.relname = #{quote(name.identifier)} + AND n.nspname = #{name.schema ? quote(name.schema) : "ANY (current_schemas(false))"} SQL end @@ -170,7 +166,13 @@ module ActiveRecord end # Returns an array of indexes for the given table. - def indexes(table_name, name = nil) + def indexes(table_name, name = nil) # :nodoc: + if name + ActiveSupport::Deprecation.warn(<<-MSG.squish) + Passing name to #indexes is deprecated without replacement. + MSG + end + table = Utils.extract_schema_qualified_name(table_name.to_s) result = query(<<-SQL, "SCHEMA") @@ -221,25 +223,29 @@ module ActiveRecord end.compact end - # Returns the list of all column definitions for a table. - def columns(table_name) # :nodoc: - table_name = table_name.to_s - column_definitions(table_name).map do |column_name, type, default, notnull, oid, fmod, collation, comment| - oid = oid.to_i - fmod = fmod.to_i - type_metadata = fetch_type_metadata(column_name, type, oid, fmod) - default_value = extract_value_from_default(default) - default_function = extract_default_function(default_value, default) - new_column(column_name, default_value, type_metadata, !notnull, table_name, default_function, collation, comment: comment.presence) - end - end - - def new_column(*args) # :nodoc: - PostgreSQLColumn.new(*args) + def new_column_from_field(table_name, field) # :nondoc: + column_name, type, default, notnull, oid, fmod, collation, comment = field + oid = oid.to_i + fmod = fmod.to_i + type_metadata = fetch_type_metadata(column_name, type, oid, fmod) + default_value = extract_value_from_default(default) + default_function = extract_default_function(default_value, default) + PostgreSQLColumn.new( + column_name, + default_value, + type_metadata, + !notnull, + table_name, + default_function, + collation, + comment: comment.presence + ) end def table_options(table_name) # :nodoc: - { comment: table_comment(table_name) } + if comment = table_comment(table_name) + { comment: comment } + end end # Returns a comment stored in database for given table @@ -439,7 +445,7 @@ module ActiveRecord WITH pk_constraint AS ( SELECT conrelid, unnest(conkey) AS connum FROM pg_constraint WHERE contype = 'p' - AND conrelid = '#{quote_table_name(table_name)}'::regclass + AND conrelid = #{quote(quote_table_name(table_name))}::regclass ), cons AS ( SELECT conrelid, connum, row_number() OVER() AS rownum FROM pk_constraint ) diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/type_metadata.rb b/activerecord/lib/active_record/connection_adapters/postgresql/type_metadata.rb index bcef8ac715..311988625f 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql/type_metadata.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql/type_metadata.rb @@ -8,7 +8,7 @@ module ActiveRecord @type_metadata = type_metadata @oid = oid @fmod = fmod - @array = /\[\]$/ === type_metadata.sql_type + @array = /\[\]$/.match?(type_metadata.sql_type) end def sql_type diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/utils.rb b/activerecord/lib/active_record/connection_adapters/postgresql/utils.rb index 9a0b80d7d3..a3f9ce6d64 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql/utils.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql/utils.rb @@ -35,6 +35,12 @@ module ActiveRecord end protected + + def parts + @parts ||= [@schema, @identifier].compact + end + + private def unquote(part) if part && part.start_with?('"') part[1..-2] @@ -42,10 +48,6 @@ module ActiveRecord part end end - - def parts - @parts ||= [@schema, @identifier].compact - end end module Utils # :nodoc: @@ -53,7 +55,7 @@ module ActiveRecord # Returns an instance of <tt>ActiveRecord::ConnectionAdapters::PostgreSQL::Name</tt> # extracted from +string+. - # +schema+ is nil if not specified in +string+. + # +schema+ is +nil+ if not specified in +string+. # +schema+ and +identifier+ exclude surrounding quotes (regardless of whether provided in +string+) # +string+ supports the range of schema/table references understood by PostgreSQL, for example: # diff --git a/activerecord/lib/active_record/connection_adapters/postgresql_adapter.rb b/activerecord/lib/active_record/connection_adapters/postgresql_adapter.rb index 03ee0eec5b..0ebd907cc0 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql_adapter.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql_adapter.rb @@ -70,7 +70,7 @@ module ActiveRecord ADAPTER_NAME = "PostgreSQL".freeze NATIVE_DATABASE_TYPES = { - primary_key: "serial primary key", + primary_key: "bigserial primary key", string: { name: "character varying" }, text: { name: "text" }, integer: { name: "integer" }, @@ -315,6 +315,10 @@ module ActiveRecord postgresql_version >= 90300 end + def supports_pgcrypto_uuid? + postgresql_version >= 90400 + end + def get_advisory_lock(lock_id) # :nodoc: unless lock_id.is_a?(Integer) && lock_id.bit_length <= 63 raise(ArgumentError, "Postgres requires advisory lock ids to be a signed 64 bit integer") @@ -400,10 +404,12 @@ module ActiveRecord @connection.server_version end - protected + private # See http://www.postgresql.org/docs/current/static/errcodes-appendix.html VALUE_LIMIT_VIOLATION = "22001" + NUMERIC_VALUE_OUT_OF_RANGE = "22003" + NOT_NULL_VIOLATION = "23502" FOREIGN_KEY_VIOLATION = "23503" UNIQUE_VIOLATION = "23505" SERIALIZATION_FAILURE = "40001" @@ -419,6 +425,10 @@ module ActiveRecord InvalidForeignKey.new(message) when VALUE_LIMIT_VIOLATION ValueTooLong.new(message) + when NUMERIC_VALUE_OUT_OF_RANGE + RangeError.new(message) + when NOT_NULL_VIOLATION + NotNullViolation.new(message) when SERIALIZATION_FAILURE SerializationFailure.new(message) when DEADLOCK_DETECTED @@ -428,9 +438,7 @@ module ActiveRecord end end - private - - def get_oid_type(oid, fmod, column_name, sql_type = "") # :nodoc: + def get_oid_type(oid, fmod, column_name, sql_type = "") if !type_map.key?(oid) load_additional_types(type_map, [oid]) end @@ -443,7 +451,7 @@ module ActiveRecord } end - def initialize_type_map(m) # :nodoc: + def initialize_type_map(m) register_class_with_limit m, "int2", Type::Integer register_class_with_limit m, "int4", Type::Integer register_class_with_limit m, "int8", Type::Integer @@ -511,7 +519,7 @@ module ActiveRecord load_additional_types(m) end - def extract_limit(sql_type) # :nodoc: + def extract_limit(sql_type) case sql_type when /^bigint/i, /^int8/i 8 @@ -523,11 +531,11 @@ module ActiveRecord end # Extracts the value from a PostgreSQL column default definition. - def extract_value_from_default(default) # :nodoc: + def extract_value_from_default(default) case default # Quoted types when /\A[\(B]?'(.*)'.*::"?([\w. ]+)"?(?:\[\])?\z/m - # The default 'now'::date is CURRENT_DATE + # The default 'now'::date is CURRENT_DATE if $1 == "now".freeze && $2 == "date".freeze nil else @@ -542,22 +550,22 @@ module ActiveRecord # Object identifier types when /\A-?\d+\z/ $1 - else - # Anything else is blank, some user type, or some function - # and we can't know the value of that, so return nil. + else + # Anything else is blank, some user type, or some function + # and we can't know the value of that, so return nil. nil end end - def extract_default_function(default_value, default) # :nodoc: + def extract_default_function(default_value, default) default if has_default_function?(default_value, default) end - def has_default_function?(default_value, default) # :nodoc: + def has_default_function?(default_value, default) !default_value && (%r{\w+\(.*\)|\(.*\)::\w+} === default) end - def load_additional_types(type_map, oids = nil) # :nodoc: + def load_additional_types(type_map, oids = nil) initializer = OID::TypeMapInitializer.new(type_map) if supports_ranges? @@ -601,7 +609,11 @@ module ActiveRecord def exec_no_cache(sql, name, binds) type_casted_binds = type_casted_binds(binds) - log(sql, name, binds, type_casted_binds) { @connection.async_exec(sql, type_casted_binds) } + log(sql, name, binds, type_casted_binds) do + ActiveSupport::Dependencies.interlock.permit_concurrent_loads do + @connection.async_exec(sql, type_casted_binds) + end + end end def exec_cache(sql, name, binds) @@ -609,7 +621,9 @@ module ActiveRecord type_casted_binds = type_casted_binds(binds) log(sql, name, binds, type_casted_binds, stmt_key) do - @connection.exec_prepared(stmt_key, type_casted_binds) + ActiveSupport::Dependencies.interlock.permit_concurrent_loads do + @connection.exec_prepared(stmt_key, type_casted_binds) + end end rescue ActiveRecord::StatementInvalid => e raise unless is_cached_plan_failure?(e) @@ -719,7 +733,7 @@ module ActiveRecord end # Returns the current ID of a table's sequence. - def last_insert_id_result(sequence_name) # :nodoc: + def last_insert_id_result(sequence_name) exec_query("SELECT currval('#{sequence_name}')", "SQL") end @@ -741,7 +755,7 @@ module ActiveRecord # Query implementation notes: # - format_type includes the column size constraint, e.g. varchar(50) # - ::regclass is a function that gives the id for a table name - def column_definitions(table_name) # :nodoc: + def column_definitions(table_name) query(<<-end_sql, "SCHEMA") SELECT a.attname, format_type(a.atttypid, a.atttypmod), pg_get_expr(d.adbin, d.adrelid), a.attnotnull, a.atttypid, a.atttypmod, @@ -750,18 +764,18 @@ module ActiveRecord col_description(a.attrelid, a.attnum) AS comment FROM pg_attribute a LEFT JOIN pg_attrdef d ON a.attrelid = d.adrelid AND a.attnum = d.adnum - WHERE a.attrelid = '#{quote_table_name(table_name)}'::regclass + WHERE a.attrelid = #{quote(quote_table_name(table_name))}::regclass AND a.attnum > 0 AND NOT a.attisdropped ORDER BY a.attnum end_sql end - def extract_table_ref_from_insert_sql(sql) # :nodoc: + def extract_table_ref_from_insert_sql(sql) sql[/into\s("[A-Za-z0-9_."\[\]\s]+"|[A-Za-z0-9_."\[\]]+)\s*/im] $1.strip if $1 end - def create_table_definition(*args) # :nodoc: + def create_table_definition(*args) PostgreSQL::TableDefinition.new(*args) end @@ -771,10 +785,14 @@ module ActiveRecord sql = <<-end_sql SELECT exists( SELECT * FROM pg_proc + WHERE proname = 'lower' + AND proargtypes = ARRAY[#{quote column.sql_type}::regtype]::oidvector + ) OR exists( + SELECT * FROM pg_proc INNER JOIN pg_cast - ON casttarget::text::oidvector = proargtypes + ON ARRAY[casttarget]::oidvector = proargtypes WHERE proname = 'lower' - AND castsource = '#{column.sql_type}'::regtype::oid + AND castsource = #{quote column.sql_type}::regtype ) end_sql execute_and_clear(sql, "SCHEMA", []) do |result| @@ -788,7 +806,6 @@ module ActiveRecord map[Integer] = PG::TextEncoder::Integer.new map[TrueClass] = PG::TextEncoder::Boolean.new map[FalseClass] = PG::TextEncoder::Boolean.new - map[Float] = PG::TextEncoder::Float.new @connection.type_map_for_queries = map end diff --git a/activerecord/lib/active_record/connection_adapters/schema_cache.rb b/activerecord/lib/active_record/connection_adapters/schema_cache.rb index 8219f132c3..4d339b0a8c 100644 --- a/activerecord/lib/active_record/connection_adapters/schema_cache.rb +++ b/activerecord/lib/active_record/connection_adapters/schema_cache.rb @@ -21,6 +21,22 @@ module ActiveRecord @data_sources = @data_sources.dup end + def encode_with(coder) + coder["columns"] = @columns + coder["columns_hash"] = @columns_hash + coder["primary_keys"] = @primary_keys + coder["data_sources"] = @data_sources + coder["version"] = ActiveRecord::Migrator.current_version + end + + def init_with(coder) + @columns = coder["columns"] + @columns_hash = coder["columns_hash"] + @primary_keys = coder["primary_keys"] + @data_sources = coder["data_sources"] + @version = coder["version"] + end + def primary_keys(table_name) @primary_keys[table_name] ||= data_source_exists?(table_name) ? connection.primary_key(table_name) : nil end @@ -32,8 +48,6 @@ module ActiveRecord @data_sources[name] = connection.data_source_exists?(name) end - alias table_exists? data_source_exists? - deprecate table_exists?: "use #data_source_exists? instead" # Add internal cache for table with +table_name+. def add(table_name) @@ -47,8 +61,6 @@ module ActiveRecord def data_sources(name) @data_sources[name] end - alias tables data_sources - deprecate tables: "use #data_sources instead" # Get the columns for a table def columns(table_name) @@ -83,8 +95,6 @@ module ActiveRecord @primary_keys.delete name @data_sources.delete name end - alias clear_table_cache! clear_data_source_cache! - deprecate clear_table_cache!: "use #clear_data_source_cache! instead" def marshal_dump # if we get current version during initialization, it happens stack over flow. diff --git a/activerecord/lib/active_record/connection_adapters/sqlite3/schema_definitions.rb b/activerecord/lib/active_record/connection_adapters/sqlite3/schema_definitions.rb new file mode 100644 index 0000000000..d0b38dff4c --- /dev/null +++ b/activerecord/lib/active_record/connection_adapters/sqlite3/schema_definitions.rb @@ -0,0 +1,23 @@ +module ActiveRecord + module ConnectionAdapters + module SQLite3 + module ColumnMethods + def primary_key(name, type = :primary_key, **options) + if options.delete(:auto_increment) == true && %i(integer bigint).include?(type) + type = :primary_key + end + + super + end + end + + class TableDefinition < ActiveRecord::ConnectionAdapters::TableDefinition + include ColumnMethods + end + + class Table < ActiveRecord::ConnectionAdapters::Table + include ColumnMethods + end + end + end +end diff --git a/activerecord/lib/active_record/connection_adapters/sqlite3/schema_dumper.rb b/activerecord/lib/active_record/connection_adapters/sqlite3/schema_dumper.rb new file mode 100644 index 0000000000..c027fef83c --- /dev/null +++ b/activerecord/lib/active_record/connection_adapters/sqlite3/schema_dumper.rb @@ -0,0 +1,13 @@ +module ActiveRecord + module ConnectionAdapters + module SQLite3 + module ColumnDumper + private + + def default_primary_key?(column) + schema_type(column) == :integer + end + end + end + end +end diff --git a/activerecord/lib/active_record/connection_adapters/sqlite3_adapter.rb b/activerecord/lib/active_record/connection_adapters/sqlite3_adapter.rb index e2b534b511..ec44d020c2 100644 --- a/activerecord/lib/active_record/connection_adapters/sqlite3_adapter.rb +++ b/activerecord/lib/active_record/connection_adapters/sqlite3_adapter.rb @@ -3,6 +3,8 @@ require "active_record/connection_adapters/statement_pool" require "active_record/connection_adapters/sqlite3/explain_pretty_printer" require "active_record/connection_adapters/sqlite3/quoting" require "active_record/connection_adapters/sqlite3/schema_creation" +require "active_record/connection_adapters/sqlite3/schema_definitions" +require "active_record/connection_adapters/sqlite3/schema_dumper" gem "sqlite3", "~> 1.3.6" require "sqlite3" @@ -52,6 +54,7 @@ module ActiveRecord ADAPTER_NAME = "SQLite".freeze include SQLite3::Quoting + include SQLite3::ColumnDumper NATIVE_DATABASE_TYPES = { primary_key: "INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL", @@ -75,6 +78,10 @@ module ActiveRecord end end + def update_table_definition(table_name, base) # :nodoc: + SQLite3::Table.new(table_name, base) + end + def schema_creation # :nodoc: SQLite3::SchemaCreation.new self end @@ -191,30 +198,32 @@ module ActiveRecord type_casted_binds = type_casted_binds(binds) log(sql, name, binds, type_casted_binds) do - # Don't cache statements if they are not prepared - unless prepare - stmt = @connection.prepare(sql) - begin - cols = stmt.columns - unless without_prepared_statement?(binds) - stmt.bind_params(type_casted_binds) + ActiveSupport::Dependencies.interlock.permit_concurrent_loads do + # Don't cache statements if they are not prepared + unless prepare + stmt = @connection.prepare(sql) + begin + cols = stmt.columns + unless without_prepared_statement?(binds) + stmt.bind_params(type_casted_binds) + end + records = stmt.to_a + ensure + stmt.close end + else + cache = @statements[sql] ||= { + stmt: @connection.prepare(sql) + } + stmt = cache[:stmt] + cols = cache[:cols] ||= stmt.columns + stmt.reset! + stmt.bind_params(type_casted_binds) records = stmt.to_a - ensure - stmt.close end - else - cache = @statements[sql] ||= { - stmt: @connection.prepare(sql) - } - stmt = cache[:stmt] - cols = cache[:cols] ||= stmt.columns - stmt.reset! - stmt.bind_params(type_casted_binds) - records = stmt.to_a - end - ActiveRecord::Result.new(cols, records) + ActiveRecord::Result.new(cols, records) + end end end @@ -229,64 +238,55 @@ module ActiveRecord end def execute(sql, name = nil) #:nodoc: - log(sql, name) { @connection.execute(sql) } + log(sql, name) do + ActiveSupport::Dependencies.interlock.permit_concurrent_loads do + @connection.execute(sql) + end + end end def begin_db_transaction #:nodoc: - log("begin transaction",nil) { @connection.transaction } + log("begin transaction", nil) { @connection.transaction } end def commit_db_transaction #:nodoc: - log("commit transaction",nil) { @connection.commit } + log("commit transaction", nil) { @connection.commit } end def exec_rollback_db_transaction #:nodoc: - log("rollback transaction",nil) { @connection.rollback } + log("rollback transaction", nil) { @connection.rollback } end # SCHEMA STATEMENTS ======================================== - def tables(name = nil) # :nodoc: - ActiveSupport::Deprecation.warn(<<-MSG.squish) - #tables currently returns both tables and views. - This behavior is deprecated and will be changed with Rails 5.1 to only return tables. - Use #data_sources instead. - MSG - - if name - ActiveSupport::Deprecation.warn(<<-MSG.squish) - Passing arguments to #tables is deprecated without replacement. - MSG - end - - data_sources + def tables # :nodoc: + select_values("SELECT name FROM sqlite_master WHERE type = 'table' AND name <> 'sqlite_sequence'", "SCHEMA") end - def data_sources + def data_sources # :nodoc: select_values("SELECT name FROM sqlite_master WHERE type IN ('table','view') AND name <> 'sqlite_sequence'", "SCHEMA") end - def table_exists?(table_name) - ActiveSupport::Deprecation.warn(<<-MSG.squish) - #table_exists? currently checks both tables and views. - This behavior is deprecated and will be changed with Rails 5.1 to only check tables. - Use #data_source_exists? instead. - MSG - - data_source_exists?(table_name) + def views # :nodoc: + select_values("SELECT name FROM sqlite_master WHERE type = 'view' AND name <> 'sqlite_sequence'", "SCHEMA") end - def data_source_exists?(table_name) + def table_exists?(table_name) # :nodoc: return false unless table_name.present? - sql = "SELECT name FROM sqlite_master WHERE type IN ('table','view') AND name <> 'sqlite_sequence'" + sql = "SELECT name FROM sqlite_master WHERE type = 'table' AND name <> 'sqlite_sequence'" sql << " AND name = #{quote(table_name)}" select_values(sql, "SCHEMA").any? end - def views # :nodoc: - select_values("SELECT name FROM sqlite_master WHERE type = 'view' AND name <> 'sqlite_sequence'", "SCHEMA") + def data_source_exists?(table_name) # :nodoc: + return false unless table_name.present? + + sql = "SELECT name FROM sqlite_master WHERE type IN ('table','view') AND name <> 'sqlite_sequence'" + sql << " AND name = #{quote(table_name)}" + + select_values(sql, "SCHEMA").any? end def view_exists?(view_name) # :nodoc: @@ -298,28 +298,30 @@ module ActiveRecord select_values(sql, "SCHEMA").any? end - # Returns an array of +Column+ objects for the table specified by +table_name+. - def columns(table_name) # :nodoc: - table_name = table_name.to_s - table_structure(table_name).map do |field| - case field["dflt_value"] - when /^null$/i - field["dflt_value"] = nil - when /^'(.*)'$/m - field["dflt_value"] = $1.gsub("''", "'") - when /^"(.*)"$/m - field["dflt_value"] = $1.gsub('""', '"') - end - - collation = field["collation"] - sql_type = field["type"] - type_metadata = fetch_type_metadata(sql_type) - new_column(field["name"], field["dflt_value"], type_metadata, field["notnull"].to_i == 0, table_name, nil, collation) + def new_column_from_field(table_name, field) # :nondoc: + case field["dflt_value"] + when /^null$/i + field["dflt_value"] = nil + when /^'(.*)'$/m + field["dflt_value"] = $1.gsub("''", "'") + when /^"(.*)"$/m + field["dflt_value"] = $1.gsub('""', '"') end + + collation = field["collation"] + sql_type = field["type"] + type_metadata = fetch_type_metadata(sql_type) + new_column(field["name"], field["dflt_value"], type_metadata, field["notnull"].to_i == 0, table_name, nil, collation) end # Returns an array of indexes for the given table. def indexes(table_name, name = nil) #:nodoc: + if name + ActiveSupport::Deprecation.warn(<<-MSG.squish) + Passing name to #indexes is deprecated without replacement. + MSG + end + exec_query("PRAGMA index_list(#{quote_table_name(table_name)})", "SCHEMA").map do |row| sql = <<-SQL SELECT sql @@ -410,7 +412,7 @@ module ActiveRecord self.default = options[:default] if include_default self.null = options[:null] if options.include?(:null) self.precision = options[:precision] if options.include?(:precision) - self.scale = options[:scale] if options.include?(:scale) + self.scale = options[:scale] if options.include?(:scale) self.collation = options[:collation] if options.include?(:collation) end end @@ -422,15 +424,16 @@ module ActiveRecord rename_column_indexes(table_name, column.name, new_column_name) end - protected + private def table_structure(table_name) structure = exec_query("PRAGMA table_info(#{quote_table_name(table_name)})", "SCHEMA") raise(ActiveRecord::StatementInvalid, "Could not find table '#{table_name}'") if structure.empty? table_structure_with_collation(table_name, structure) end + alias column_definitions table_structure - def alter_table(table_name, options = {}) #:nodoc: + def alter_table(table_name, options = {}) altered_table_name = "a#{table_name}" caller = lambda { |definition| yield definition if block_given? } @@ -441,12 +444,12 @@ module ActiveRecord end end - def move_table(from, to, options = {}, &block) #:nodoc: + def move_table(from, to, options = {}, &block) copy_table(from, to, options, &block) drop_table(from) end - def copy_table(from, to, options = {}) #:nodoc: + def copy_table(from, to, options = {}) from_primary_key = primary_key(from) options[:id] = false create_table(to, options) do |definition| @@ -472,7 +475,7 @@ module ActiveRecord options[:rename] || {}) end - def copy_table_indexes(from, to, rename = {}) #:nodoc: + def copy_table_indexes(from, to, rename = {}) indexes(from).each do |index| name = index.name if to == "a#{from}" @@ -495,7 +498,7 @@ module ActiveRecord end end - def copy_table_contents(from, to, columns, rename = {}) #:nodoc: + def copy_table_contents(from, to, columns, rename = {}) column_mappings = Hash[columns.map { |name| [name, name] }] rename.each { |a| column_mappings[a.last] = a.first } from_columns = columns(from).collect(&:name) @@ -520,20 +523,23 @@ module ActiveRecord # column *column_name* is not unique when /column(s)? .* (is|are) not unique/, /UNIQUE constraint failed: .*/ RecordNotUnique.new(message) + when /.* may not be NULL/, /NOT NULL constraint failed: .*/ + NotNullViolation.new(message) else super end end - private COLLATE_REGEX = /.*\"(\w+)\".*collate\s+\"(\w+)\".*/i.freeze def table_structure_with_collation(table_name, basic_structure) collation_hash = {} - sql = "SELECT sql FROM - (SELECT * FROM sqlite_master UNION ALL - SELECT * FROM sqlite_temp_master) - WHERE type='table' and name='#{ table_name }' \;" + sql = <<-SQL + SELECT sql FROM + (SELECT * FROM sqlite_master UNION ALL + SELECT * FROM sqlite_temp_master) + WHERE type = 'table' AND name = #{quote(table_name)} + SQL # Result will have following sample string # CREATE TABLE "users" ("id" INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, @@ -564,6 +570,10 @@ module ActiveRecord basic_structure.to_hash end end + + def create_table_definition(*args) + SQLite3::TableDefinition.new(*args) + end end end end diff --git a/activerecord/lib/active_record/connection_adapters/statement_pool.rb b/activerecord/lib/active_record/connection_adapters/statement_pool.rb index 273b1b0b5c..790db56185 100644 --- a/activerecord/lib/active_record/connection_adapters/statement_pool.rb +++ b/activerecord/lib/active_record/connection_adapters/statement_pool.rb @@ -6,7 +6,7 @@ module ActiveRecord DEFAULT_STATEMENT_LIMIT = 1000 def initialize(statement_limit = nil) - @cache = Hash.new { |h,pid| h[pid] = {} } + @cache = Hash.new { |h, pid| h[pid] = {} } @statement_limit = statement_limit || DEFAULT_STATEMENT_LIMIT end |