diff options
Diffstat (limited to 'activerecord/lib/active_record/connection_adapters')
72 files changed, 2375 insertions, 1489 deletions
diff --git a/activerecord/lib/active_record/connection_adapters/abstract/connection_pool.rb b/activerecord/lib/active_record/connection_adapters/abstract/connection_pool.rb index 3f2e86a98d..e977d36cb9 100644 --- a/activerecord/lib/active_record/connection_adapters/abstract/connection_pool.rb +++ b/activerecord/lib/active_record/connection_adapters/abstract/connection_pool.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require "thread" require "concurrent/map" require "monitor" @@ -61,15 +63,13 @@ module ActiveRecord # There are several connection-pooling-related options that you can add to # your database connection configuration: # - # * +pool+: number indicating size of connection pool (default 5) - # * +checkout_timeout+: number of seconds to block and wait for a connection - # before giving up and raising a timeout error (default 5 seconds). - # * +reaping_frequency+: frequency in seconds to periodically run the - # Reaper, which attempts to find and recover connections from dead - # threads, which can occur if a programmer forgets to close a - # connection at the end of a thread or a thread dies unexpectedly. - # Regardless of this setting, the Reaper will be invoked before every - # blocking wait. (Default +nil+, which means don't schedule the Reaper). + # * +pool+: maximum number of connections the pool may manage (default 5). + # * +idle_timeout+: number of seconds that a connection will be kept + # unused in the pool before it is automatically disconnected (default + # 300 seconds). Set this to zero to keep connections forever. + # * +checkout_timeout+: number of seconds to wait for a connection to + # become available before giving up and raising a timeout error (default + # 5 seconds). # #-- # Synchronization policy: @@ -80,11 +80,8 @@ module ActiveRecord # * private methods that require being called in a +synchronize+ blocks # are now explicitly documented class ConnectionPool - # Threadsafe, fair, FIFO queue. Meant to be used by ConnectionPool - # with which it shares a Monitor. But could be a generic Queue. - # - # The Queue in stdlib's 'thread' could replace this class except - # stdlib's doesn't support waiting with a timeout. + # Threadsafe, fair, LIFO queue. Meant to be used by ConnectionPool + # with which it shares a Monitor. class Queue def initialize(lock = Monitor.new) @lock = lock @@ -173,7 +170,7 @@ module ActiveRecord # Removes and returns the head of the queue if possible, or +nil+. def remove - @queue.shift + @queue.pop end # Remove and return the head the queue if the number of @@ -191,7 +188,9 @@ module ActiveRecord t0 = Time.now elapsed = 0 loop do - @cond.wait(timeout - elapsed) + ActiveSupport::Dependencies.interlock.permit_concurrent_loads do + @cond.wait(timeout - elapsed) + end return remove if any? @@ -268,7 +267,7 @@ module ActiveRecord # Connections must be leased while holding the main pool mutex. This is # an internal subclass that also +.leases+ returned connections while # still in queue's critical section (queue synchronizes with the same - # +@lock+ as the main pool) so that a returned connection is already + # <tt>@lock</tt> as the main pool) so that a returned connection is already # leased and there is no need to re-enter synchronized block. class ConnectionLeasingQueue < Queue # :nodoc: include BiasableQueue @@ -281,12 +280,12 @@ module ActiveRecord end end - # Every +frequency+ seconds, the reaper will call +reap+ on +pool+. - # A reaper instantiated with a +nil+ frequency will never reap the - # connection pool. + # Every +frequency+ seconds, the reaper will call +reap+ and +flush+ on + # +pool+. A reaper instantiated with a zero frequency will never reap + # the connection pool. # - # Configure the frequency by setting "reaping_frequency" in your - # database yaml file. + # Configure the frequency by setting +reaping_frequency+ in your database + # yaml file (default 60 seconds). class Reaper attr_reader :pool, :frequency @@ -296,11 +295,12 @@ module ActiveRecord end def run - return unless frequency + return unless frequency && frequency > 0 Thread.new(frequency, pool) { |t, p| loop do sleep t p.reap + p.flush end } end @@ -324,8 +324,10 @@ module ActiveRecord @spec = spec @checkout_timeout = (spec.config[:checkout_timeout] && spec.config[:checkout_timeout].to_f) || 5 - @reaper = Reaper.new(self, (spec.config[:reaping_frequency] && spec.config[:reaping_frequency].to_f)) - @reaper.run + if @idle_timeout = spec.config.fetch(:idle_timeout, 300) + @idle_timeout = @idle_timeout.to_f + @idle_timeout = nil if @idle_timeout <= 0 + end # default max pool size to 5 @size = (spec.config[:pool] && spec.config[:pool].to_i) || 5 @@ -338,7 +340,7 @@ module ActiveRecord # then that +thread+ does indeed own that +conn+. However, an absence of a such # mapping does not mean that the +thread+ doesn't own the said connection. In # that case +conn.owner+ attr should be consulted. - # Access and modification of +@thread_cached_conns+ does not require + # Access and modification of <tt>@thread_cached_conns</tt> does not require # synchronization. @thread_cached_conns = Concurrent::Map.new(initial_capacity: @size) @@ -355,6 +357,12 @@ module ActiveRecord @available = ConnectionLeasingQueue.new self @lock_thread = false + + # +reaping_frequency+ is configurable mostly for historical reasons, but it could + # also be useful if someone wants a very low +idle_timeout+. + reaping_frequency = spec.config.fetch(:reaping_frequency, 60) + @reaper = Reaper.new(self, reaping_frequency && reaping_frequency.to_f) + @reaper.run end def lock_thread=(lock_thread) @@ -447,6 +455,21 @@ module ActiveRecord disconnect(false) end + # Discards all connections in the pool (even if they're currently + # leased!), along with the pool itself. Any further interaction with the + # pool (except #spec and #schema_cache) is undefined. + # + # See AbstractAdapter#discard! + def discard! # :nodoc: + synchronize do + return if @connections.nil? # already discarded + @connections.each do |conn| + conn.discard! + end + @connections = @available = @thread_cached_conns = nil + end + end + # Clears the cache which maps classes and re-connects connections that # require reloading. # @@ -506,14 +529,16 @@ module ActiveRecord # +conn+: an AbstractAdapter object, which was obtained by earlier by # calling #checkout on this pool. def checkin(conn) - synchronize do - remove_connection_from_thread_cache conn + conn.lock.synchronize do + synchronize do + remove_connection_from_thread_cache conn - conn._run_checkin_callbacks do - conn.expire - end + conn._run_checkin_callbacks do + conn.expire + end - @available.add conn + @available.add conn + end end end @@ -570,6 +595,35 @@ module ActiveRecord end end + # Disconnect all connections that have been idle for at least + # +minimum_idle+ seconds. Connections currently checked out, or that were + # checked in less than +minimum_idle+ seconds ago, are unaffected. + def flush(minimum_idle = @idle_timeout) + return if minimum_idle.nil? + + idle_connections = synchronize do + @connections.select do |conn| + !conn.in_use? && conn.seconds_idle >= minimum_idle + end.each do |conn| + conn.lease + + @available.delete conn + @connections.delete conn + end + end + + idle_connections.each do |conn| + conn.disconnect! + end + end + + # Disconnect all currently idle connections. Connections currently checked + # out are unaffected. + def flush! + reap + flush(-1) + end + def num_waiting_in_queue # :nodoc: @available.num_waiting end @@ -677,7 +731,7 @@ module ActiveRecord # this block can't be easily moved into attempt_to_checkout_all_existing_connections's # rescue block, because doing so would put it outside of synchronize section, without # being in a critical section thread_report might become inaccurate - msg = "could not obtain ownership of all database connections in #{checkout_timeout} seconds" + msg = "could not obtain ownership of all database connections in #{checkout_timeout} seconds".dup thread_report = [] @connections.each do |conn| @@ -732,10 +786,10 @@ module ActiveRecord # Implementation detail: the connection returned by +acquire_connection+ # will already be "+connection.lease+ -ed" to the current thread. def acquire_connection(checkout_timeout) - # NOTE: we rely on +@available.poll+ and +try_to_checkout_new_connection+ to + # NOTE: we rely on <tt>@available.poll</tt> and +try_to_checkout_new_connection+ to # +conn.lease+ the returned connection (and to do this in a +synchronized+ # section). This is not the cleanest implementation, as ideally we would - # <tt>synchronize { conn.lease }</tt> in this method, but by leaving it to +@available.poll+ + # <tt>synchronize { conn.lease }</tt> in this method, but by leaving it to <tt>@available.poll</tt> # and +try_to_checkout_new_connection+ we can piggyback on +synchronize+ sections # of the said methods and avoid an additional +synchronize+ overhead. if conn = @available.poll || try_to_checkout_new_connection @@ -759,7 +813,7 @@ module ActiveRecord end end - # If the pool is not at a +@size+ limit, establish new connection. Connecting + # If the pool is not at a <tt>@size</tt> limit, establish new connection. Connecting # to the DB is done outside main synchronized section. #-- # Implementation constraint: a newly established connection returned by this @@ -825,7 +879,7 @@ module ActiveRecord # end # # class Book < ActiveRecord::Base - # establish_connection "library_db" + # establish_connection :library_db # end # # class ScaryBook < Book @@ -857,15 +911,35 @@ module ActiveRecord # All Active Record models use this handler to determine the connection pool that they # should use. # - # The ConnectionHandler class is not coupled with the Active models, as it has no knowlodge + # The ConnectionHandler class is not coupled with the Active models, as it has no knowledge # about the model. The model needs to pass a specification name to the handler, - # in order to lookup the correct connection pool. + # in order to look up the correct connection pool. class ConnectionHandler + def self.unowned_pool_finalizer(pid_map) # :nodoc: + lambda do |_| + discard_unowned_pools(pid_map) + end + end + + def self.discard_unowned_pools(pid_map) # :nodoc: + pid_map.each do |pid, pools| + pools.values.compact.each(&:discard!) unless pid == Process.pid + end + end + def initialize # These caches are keyed by spec.name (ConnectionSpecification#name). @owner_to_pool = Concurrent::Map.new(initial_capacity: 2) do |h, k| + # Discard the parent's connection pools immediately; we have no need + # of them + ConnectionHandler.discard_unowned_pools(h) + h[k] = Concurrent::Map.new(initial_capacity: 2) end + + # Backup finalizer: if the forked child never needed a pool, the above + # early discard has not occurred + ObjectSpace.define_finalizer self, ConnectionHandler.unowned_pool_finalizer(@owner_to_pool) end def connection_pool_list @@ -919,6 +993,13 @@ module ActiveRecord connection_pool_list.each(&:disconnect!) end + # Disconnects all currently idle connections. + # + # See ConnectionPool#flush! for details. + def flush_idle_connections! + connection_pool_list.each(&:flush!) + end + # Locate the connection of the nearest super class. This can be an # active or defined connection: if it is the latter, it will be # opened and set as the active connection for the class it was defined @@ -926,16 +1007,14 @@ module ActiveRecord def retrieve_connection(spec_name) #:nodoc: pool = retrieve_connection_pool(spec_name) raise ConnectionNotEstablished, "No connection pool with '#{spec_name}' found." unless pool - conn = pool.connection - raise ConnectionNotEstablished, "No connection for '#{spec_name}' in connection pool" unless conn - conn + pool.connection end # Returns true if a connection that's accessible to this class has # already been opened. def connected?(spec_name) - conn = retrieve_connection_pool(spec_name) - conn && conn.connected? + pool = retrieve_connection_pool(spec_name) + pool && pool.connected? end # Remove the connection for this class. This will close the active diff --git a/activerecord/lib/active_record/connection_adapters/abstract/database_limits.rb b/activerecord/lib/active_record/connection_adapters/abstract/database_limits.rb index 407e019326..ad148efcfe 100644 --- a/activerecord/lib/active_record/connection_adapters/abstract/database_limits.rb +++ b/activerecord/lib/active_record/connection_adapters/abstract/database_limits.rb @@ -1,3 +1,7 @@ +# frozen_string_literal: true + +require "active_support/deprecation" + module ActiveRecord module ConnectionAdapters # :nodoc: module DatabaseLimits @@ -10,11 +14,13 @@ module ActiveRecord def column_name_length 64 end + deprecate :column_name_length # Returns the maximum length of a table name. def table_name_length 64 end + deprecate :table_name_length # Returns the maximum allowed length for an index name. This # limit is enforced by \Rails and is less than or equal to @@ -34,16 +40,19 @@ module ActiveRecord def columns_per_table 1024 end + deprecate :columns_per_table # Returns the maximum number of indexes per table. def indexes_per_table 16 end + deprecate :indexes_per_table # Returns the maximum number of columns in a multicolumn index. def columns_per_multicolumn_index 16 end + deprecate :columns_per_multicolumn_index # Returns the maximum number of elements in an IN (x,y,z) clause. # +nil+ means no limit. @@ -55,11 +64,13 @@ module ActiveRecord def sql_query_length 1048575 end + deprecate :sql_query_length # Returns maximum number of joins in a single query. def joins_per_query 256 end + deprecate :joins_per_query end end end diff --git a/activerecord/lib/active_record/connection_adapters/abstract/database_statements.rb b/activerecord/lib/active_record/connection_adapters/abstract/database_statements.rb index 769f488469..fdc9ffa688 100644 --- a/activerecord/lib/active_record/connection_adapters/abstract/database_statements.rb +++ b/activerecord/lib/active_record/connection_adapters/abstract/database_statements.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module ActiveRecord module ConnectionAdapters # :nodoc: module DatabaseStatements @@ -7,30 +9,43 @@ module ActiveRecord end # Converts an arel AST to SQL - def to_sql(arel, binds = []) - if arel.respond_to?(:ast) - collected = visitor.accept(arel.ast, collector) - collected.compile(binds, self).freeze + def to_sql(arel_or_sql_string, binds = []) + sql, _ = to_sql_and_binds(arel_or_sql_string, binds) + sql + end + + def to_sql_and_binds(arel_or_sql_string, binds = []) # :nodoc: + if arel_or_sql_string.respond_to?(:ast) + unless binds.empty? + raise "Passing bind parameters with an arel AST is forbidden. " \ + "The values must be stored on the AST directly" + end + sql, binds = visitor.compile(arel_or_sql_string.ast, collector) + [sql.freeze, binds || []] else - arel.dup.freeze + [arel_or_sql_string.dup.freeze, binds] end end + private :to_sql_and_binds # This is used in the StatementCache object. It returns an object that # can be used to query the database repeatedly. def cacheable_query(klass, arel) # :nodoc: - collected = visitor.accept(arel.ast, collector) if prepared_statements - klass.query(collected.value) + sql, binds = visitor.compile(arel.ast, collector) + query = klass.query(sql) else - klass.partial_query(collected.value) + collector = PartialQueryCollector.new + parts, binds = visitor.compile(arel.ast, collector) + query = klass.partial_query(parts) end + [query, binds] end # Returns an ActiveRecord::Result instance. def select_all(arel, name = nil, binds = [], preparable: nil) - arel, binds = binds_from_relation arel, binds - sql = to_sql(arel, binds) + arel = arel_from_relation(arel) + sql, binds = to_sql_and_binds(arel, binds) if !prepared_statements || (arel.is_a?(String) && preparable.nil?) preparable = false else @@ -51,9 +66,7 @@ module ActiveRecord # Returns a single value from a record def select_value(arel, name = nil, binds = []) - if result = select_rows(arel, name, binds).first - result.first - end + single_value_from_rows(select_rows(arel, name, binds)) end # Returns an array of the values of the first column in a select: @@ -68,6 +81,18 @@ module ActiveRecord select_all(arel, name, binds).rows end + def query_value(sql, name = nil) # :nodoc: + single_value_from_rows(query(sql, name)) + end + + def query_values(sql, name = nil) # :nodoc: + query(sql, name).map(&:first) + end + + def query(sql, name = nil) # :nodoc: + exec_query(sql, name).rows + end + # Executes the SQL statement in the context of this connection and returns # the raw result from the connection adapter. # Note: depending on your database connector, the result returned by this @@ -120,26 +145,30 @@ module ActiveRecord # If the next id was calculated in advance (as in Oracle), it should be # passed in as +id_value+. def insert(arel, name = nil, pk = nil, id_value = nil, sequence_name = nil, binds = []) - value = exec_insert(to_sql(arel, binds), name, binds, pk, sequence_name) + sql, binds = to_sql_and_binds(arel, binds) + value = exec_insert(sql, name, binds, pk, sequence_name) id_value || last_inserted_id(value) end alias create insert # Executes the update statement and returns the number of rows affected. def update(arel, name = nil, binds = []) - exec_update(to_sql(arel, binds), name, binds) + sql, binds = to_sql_and_binds(arel, binds) + exec_update(sql, name, binds) end # Executes the delete statement and returns the number of rows affected. def delete(arel, name = nil, binds = []) - exec_delete(to_sql(arel, binds), name, binds) + sql, binds = to_sql_and_binds(arel, binds) + exec_delete(sql, name, binds) end # Returns +true+ when the connection adapter supports prepared statement # caching, otherwise returns +false+ - def supports_statement_cache? - false + def supports_statement_cache? # :nodoc: + true end + deprecate :supports_statement_cache? # Runs the given block in a database transaction, and returns the result # of the block. @@ -152,7 +181,7 @@ module ActiveRecord # # In order to get around this problem, #transaction will emulate the effect # of nested transactions, by using savepoints: - # http://dev.mysql.com/doc/refman/5.7/en/savepoint.html + # https://dev.mysql.com/doc/refman/5.7/en/savepoint.html # Savepoints are supported by MySQL and PostgreSQL. SQLite3 version >= '3.6.8' # supports savepoints. # @@ -204,7 +233,7 @@ module ActiveRecord # You should consult the documentation for your database to understand the # semantics of these different levels: # - # * http://www.postgresql.org/docs/current/static/transaction-iso.html + # * https://www.postgresql.org/docs/current/static/transaction-iso.html # * https://dev.mysql.com/doc/refman/5.7/en/set-transaction.html # # An ActiveRecord::TransactionIsolationError will be raised if: @@ -230,7 +259,9 @@ module ActiveRecord attr_reader :transaction_manager #:nodoc: - delegate :within_new_transaction, :open_transactions, :current_transaction, :begin_transaction, :commit_transaction, :rollback_transaction, to: :transaction_manager + delegate :within_new_transaction, :open_transactions, :current_transaction, :begin_transaction, + :commit_transaction, :rollback_transaction, :materialize_transactions, + :disable_lazy_transactions!, :enable_lazy_transactions!, to: :transaction_manager def transaction_open? current_transaction.open? @@ -295,6 +326,9 @@ module ActiveRecord # Inserts the given fixture into the table. Overridden in adapters that require # something beyond a simple insert (eg. Oracle). + # Most of adapters should implement `insert_fixtures` that leverages bulk SQL insert. + # We keep this method to provide fallback + # for databases like sqlite that do not support bulk inserts. def insert_fixture(fixture, table_name) fixture = fixture.stringify_keys @@ -307,19 +341,53 @@ module ActiveRecord raise Fixture::FixtureError, %(table "#{table_name}" has no column named #{name.inspect}.) end end - key_list = fixture.keys.map { |name| quote_column_name(name) } - value_list = binds.map(&:value_for_database).map do |value| - begin - quote(value) - rescue TypeError - quote(YAML.dump(value)) - end + + table = Arel::Table.new(table_name) + + values = binds.map do |bind| + value = with_yaml_fallback(bind.value_for_database) + [table[bind.name], value] end - execute "INSERT INTO #{quote_table_name(table_name)} (#{key_list.join(', ')}) VALUES (#{value_list.join(', ')})", "Fixture Insert" + manager = Arel::InsertManager.new + manager.into(table) + manager.insert(values) + execute manager.to_sql, "Fixture Insert" + end + + # Inserts a set of fixtures into the table. Overridden in adapters that require + # something beyond a simple insert (eg. Oracle). + def insert_fixtures(fixtures, table_name) + ActiveSupport::Deprecation.warn(<<-MSG.squish) + `insert_fixtures` is deprecated and will be removed in the next version of Rails. + Consider using `insert_fixtures_set` for performance improvement. + MSG + return if fixtures.empty? + + execute(build_fixture_sql(fixtures, table_name), "Fixtures Insert") + end + + def insert_fixtures_set(fixture_set, tables_to_delete = []) + fixture_inserts = fixture_set.map do |table_name, fixtures| + next if fixtures.empty? + + build_fixture_sql(fixtures, table_name) + end.compact + + table_deletes = tables_to_delete.map { |table| "DELETE FROM #{quote_table_name table}".dup } + total_sql = Array.wrap(combine_multi_statements(table_deletes + fixture_inserts)) + + disable_referential_integrity do + transaction(requires_new: true) do + total_sql.each do |sql| + execute sql, "Fixtures Load" + yield if block_given? + end + end + end end - def empty_insert_statement_value + def empty_insert_statement_value(primary_key = nil) "DEFAULT VALUES" end @@ -348,6 +416,44 @@ module ActiveRecord alias join_to_delete join_to_update private + def default_insert_value(column) + Arel.sql("DEFAULT") + end + + def build_fixture_sql(fixtures, table_name) + columns = schema_cache.columns_hash(table_name) + + values = fixtures.map do |fixture| + fixture = fixture.stringify_keys + + unknown_columns = fixture.keys - columns.keys + if unknown_columns.any? + raise Fixture::FixtureError, %(table "#{table_name}" has no columns named #{unknown_columns.map(&:inspect).join(', ')}.) + end + + columns.map do |name, column| + if fixture.key?(name) + type = lookup_cast_type_from_column(column) + bind = Relation::QueryAttribute.new(name, fixture[name], type) + with_yaml_fallback(bind.value_for_database) + else + default_insert_value(column) + end + end + end + + table = Arel::Table.new(table_name) + manager = Arel::InsertManager.new + manager.into(table) + columns.each_key { |column| manager.columns << table[column] } + manager.values = manager.create_values_list(values) + + manager.to_sql + end + + def combine_multi_statements(total_sql) + total_sql.join(";\n") + end # Returns a subquery for the given key using the join information. def subquery_for(key, select) @@ -370,15 +476,53 @@ module ActiveRecord end def last_inserted_id(result) - row = result.rows.first + single_value_from_rows(result.rows) + end + + def single_value_from_rows(rows) + row = rows.first row && row.first end - def binds_from_relation(relation, binds) - if relation.is_a?(Relation) && binds.empty? - relation, binds = relation.arel, relation.bound_attributes + def arel_from_relation(relation) + if relation.is_a?(Relation) + relation.arel + else + relation + end + end + + # Fixture value is quoted by Arel, however scalar values + # are not quotable. In this case we want to convert + # the column value to YAML. + def with_yaml_fallback(value) + if value.is_a?(Hash) || value.is_a?(Array) + YAML.dump(value) + else + value + end + end + + class PartialQueryCollector + def initialize + @parts = [] + @binds = [] + end + + def <<(str) + @parts << str + self + end + + def add_bind(obj) + @binds << obj + @parts << Arel::Nodes::BindParam.new(1) + self + end + + def value + [@parts, @binds] end - [relation, binds] end end end diff --git a/activerecord/lib/active_record/connection_adapters/abstract/query_cache.rb b/activerecord/lib/active_record/connection_adapters/abstract/query_cache.rb index e53ba4e666..8aeb934ec2 100644 --- a/activerecord/lib/active_record/connection_adapters/abstract/query_cache.rb +++ b/activerecord/lib/active_record/connection_adapters/abstract/query_cache.rb @@ -1,3 +1,7 @@ +# frozen_string_literal: true + +require "concurrent/map" + module ActiveRecord module ConnectionAdapters # :nodoc: module QueryCache @@ -90,8 +94,8 @@ module ActiveRecord def select_all(arel, name = nil, binds = [], preparable: nil) if @query_cache_enabled && !locked?(arel) - arel, binds = binds_from_relation arel, binds - sql = to_sql(arel, binds) + arel = arel_from_relation(arel) + sql, binds = to_sql_and_binds(arel, binds) cache_sql(sql, name, binds) { super(sql, name, binds, preparable: preparable) } else super @@ -106,11 +110,7 @@ module ActiveRecord if @query_cache[sql].key?(binds) ActiveSupport::Notifications.instrument( "sql.active_record", - sql: sql, - binds: binds, - name: name, - connection_id: object_id, - cached: true, + cache_notification_info(sql, name, binds) ) @query_cache[sql][binds] else @@ -120,9 +120,23 @@ module ActiveRecord end end + # Database adapters can override this method to + # provide custom cache information. + def cache_notification_info(sql, name, binds) + { + sql: sql, + binds: binds, + type_casted_binds: -> { type_casted_binds(binds) }, + name: name, + connection_id: object_id, + cached: true + } + end + # If arel is locked this is a SELECT ... FOR UPDATE or somesuch. Such # queries should not be cached. def locked?(arel) + arel = arel.arel if arel.is_a?(Relation) arel.respond_to?(:locked) && arel.locked end diff --git a/activerecord/lib/active_record/connection_adapters/abstract/quoting.rb b/activerecord/lib/active_record/connection_adapters/abstract/quoting.rb index e5a24b2aca..98b1348135 100644 --- a/activerecord/lib/active_record/connection_adapters/abstract/quoting.rb +++ b/activerecord/lib/active_record/connection_adapters/abstract/quoting.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require "active_support/core_ext/big_decimal/conversions" require "active_support/multibyte/chars" @@ -5,14 +7,12 @@ module ActiveRecord module ConnectionAdapters # :nodoc: module Quoting # Quotes the column value to help prevent - # {SQL injection attacks}[http://en.wikipedia.org/wiki/SQL_injection]. + # {SQL injection attacks}[https://en.wikipedia.org/wiki/SQL_injection]. def quote(value) value = id_value_for_database(value) if value.is_a?(Base) - if value.respond_to?(:quoted_id) - ActiveSupport::Deprecation.warn \ - "Using #quoted_id is deprecated and will be removed in Rails 5.2." - return value.quoted_id + if value.respond_to?(:value_for_database) + value = value.value_for_database end _quote(value) @@ -24,10 +24,6 @@ module ActiveRecord def type_cast(value, column = nil) value = id_value_for_database(value) if value.is_a?(Base) - if value.respond_to?(:quoted_id) && value.respond_to?(:id) - return value.id - end - if column value = type_cast_from_column(column, value) end @@ -61,17 +57,6 @@ module ActiveRecord lookup_cast_type(column.sql_type) end - def fetch_type_metadata(sql_type) - cast_type = lookup_cast_type(sql_type) - SqlTypeMetadata.new( - sql_type: sql_type, - type: cast_type.type, - limit: cast_type.limit, - precision: cast_type.precision, - scale: cast_type.scale, - ) - end - # Quotes a string, escaping any ' (single quote) and \ (backslash) # characters. def quote_string(s) @@ -110,19 +95,19 @@ module ActiveRecord end def quoted_true - "'t'".freeze + "TRUE".freeze end def unquoted_true - "t".freeze + true end def quoted_false - "'f'".freeze + "FALSE".freeze end def unquoted_false - "f".freeze + false end # Quote date/time values for use in SQL input. Includes microseconds @@ -145,18 +130,26 @@ module ActiveRecord end def quoted_time(value) # :nodoc: - quoted_date(value).sub(/\A2000-01-01 /, "") + value = value.change(year: 2000, month: 1, day: 1) + quoted_date(value).sub(/\A\d\d\d\d-\d\d-\d\d /, "") end def quoted_binary(value) # :nodoc: "'#{quote_string(value.to_s)}'" end - private - - def type_casted_binds(binds) + def type_casted_binds(binds) # :nodoc: + if binds.first.is_a?(Array) + binds.map { |column, value| type_cast(value, column) } + else binds.map { |attr| type_cast(attr.value_for_database) } end + end + + private + def lookup_cast_type(sql_type) + type_map.lookup(sql_type) + end def id_value_for_database(value) if primary_key = value.class.primary_key diff --git a/activerecord/lib/active_record/connection_adapters/abstract/savepoints.rb b/activerecord/lib/active_record/connection_adapters/abstract/savepoints.rb index 3a06f75292..52a796b926 100644 --- a/activerecord/lib/active_record/connection_adapters/abstract/savepoints.rb +++ b/activerecord/lib/active_record/connection_adapters/abstract/savepoints.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module ActiveRecord module ConnectionAdapters module Savepoints diff --git a/activerecord/lib/active_record/connection_adapters/abstract/schema_creation.rb b/activerecord/lib/active_record/connection_adapters/abstract/schema_creation.rb index c48a4acff8..529c9d8ca6 100644 --- a/activerecord/lib/active_record/connection_adapters/abstract/schema_creation.rb +++ b/activerecord/lib/active_record/connection_adapters/abstract/schema_creation.rb @@ -1,4 +1,4 @@ -require "active_support/core_ext/string/strip" +# frozen_string_literal: true module ActiveRecord module ConnectionAdapters @@ -15,14 +15,13 @@ module ActiveRecord end delegate :quote_column_name, :quote_table_name, :quote_default_expression, :type_to_sql, - :options_include_default?, :supports_indexes_in_create?, :supports_foreign_keys_in_create?, :foreign_key_options, to: :@conn - private :quote_column_name, :quote_table_name, :quote_default_expression, :type_to_sql, - :options_include_default?, :supports_indexes_in_create?, :supports_foreign_keys_in_create?, :foreign_key_options + :options_include_default?, :supports_indexes_in_create?, :supports_foreign_keys_in_create?, :foreign_key_options, + to: :@conn, private: true private def visit_AlterTable(o) - sql = "ALTER TABLE #{quote_table_name(o.name)} " + sql = "ALTER TABLE #{quote_table_name(o.name)} ".dup sql << o.adds.map { |col| accept col }.join(" ") sql << o.foreign_key_adds.map { |fk| visit_AddForeignKey fk }.join(" ") sql << o.foreign_key_drops.map { |fk| visit_DropForeignKey fk }.join(" ") @@ -30,17 +29,17 @@ module ActiveRecord def visit_ColumnDefinition(o) o.sql_type = type_to_sql(o.type, o.options) - column_sql = "#{quote_column_name(o.name)} #{o.sql_type}" + column_sql = "#{quote_column_name(o.name)} #{o.sql_type}".dup add_column_options!(column_sql, column_options(o)) unless o.type == :primary_key column_sql end def visit_AddColumnDefinition(o) - "ADD #{accept(o.column)}" + "ADD #{accept(o.column)}".dup end def visit_TableDefinition(o) - create_sql = "CREATE#{' TEMPORARY' if o.temporary} TABLE #{quote_table_name(o.name)} " + create_sql = "CREATE#{' TEMPORARY' if o.temporary} TABLE #{quote_table_name(o.name)} ".dup statements = o.columns.map { |c| accept c } statements << accept(o.primary_keys) if o.primary_keys @@ -55,16 +54,16 @@ module ActiveRecord create_sql << "(#{statements.join(', ')})" if statements.present? add_table_options!(create_sql, table_options(o)) - create_sql << " AS #{@conn.to_sql(o.as)}" if o.as + create_sql << " AS #{to_sql(o.as)}" if o.as create_sql end def visit_PrimaryKeyDefinition(o) - "PRIMARY KEY (#{o.name.join(', ')})" + "PRIMARY KEY (#{o.name.map { |name| quote_column_name(name) }.join(', ')})" end def visit_ForeignKeyDefinition(o) - sql = <<-SQL.strip_heredoc + sql = +<<~SQL CONSTRAINT #{quote_column_name(o.name)} FOREIGN KEY (#{quote_column_name(o.column)}) REFERENCES #{quote_table_name(o.to_table)} (#{quote_column_name(o.primary_key)}) @@ -93,6 +92,7 @@ module ActiveRecord if options_sql = options[:options] create_sql << " #{options_sql}" end + create_sql end def column_options(o) @@ -114,6 +114,11 @@ module ActiveRecord sql end + def to_sql(sql) + sql = sql.to_sql if sql.respond_to?(:to_sql) + sql + end + def foreign_key_in_create(from_table, to_table, options) options = foreign_key_options(from_table, to_table, options) accept ForeignKeyDefinition.new(from_table, to_table, options) @@ -125,7 +130,7 @@ module ActiveRecord when :cascade then "ON #{action} CASCADE" when :restrict then "ON #{action} RESTRICT" else - raise ArgumentError, <<-MSG.strip_heredoc + raise ArgumentError, <<~MSG '#{dependency}' is not supported for :on_update or :on_delete. Supported values are: :nullify, :cascade, :restrict MSG @@ -133,5 +138,6 @@ module ActiveRecord end end end + SchemaCreation = AbstractAdapter::SchemaCreation # :nodoc: end end diff --git a/activerecord/lib/active_record/connection_adapters/abstract/schema_definitions.rb b/activerecord/lib/active_record/connection_adapters/abstract/schema_definitions.rb index 5eb7787226..582ac516c7 100644 --- a/activerecord/lib/active_record/connection_adapters/abstract/schema_definitions.rb +++ b/activerecord/lib/active_record/connection_adapters/abstract/schema_definitions.rb @@ -1,9 +1,47 @@ +# frozen_string_literal: true + module ActiveRecord module ConnectionAdapters #:nodoc: # Abstract representation of an index definition on a table. Instances of # this type are typically created and returned by methods in database - # adapters. e.g. ActiveRecord::ConnectionAdapters::AbstractMysqlAdapter#indexes - IndexDefinition = Struct.new(:table, :name, :unique, :columns, :lengths, :orders, :where, :type, :using, :comment) #:nodoc: + # adapters. e.g. ActiveRecord::ConnectionAdapters::MySQL::SchemaStatements#indexes + class IndexDefinition # :nodoc: + attr_reader :table, :name, :unique, :columns, :lengths, :orders, :opclasses, :where, :type, :using, :comment + + def initialize( + table, name, + unique = false, + columns = [], + lengths: {}, + orders: {}, + opclasses: {}, + where: nil, + type: nil, + using: nil, + comment: nil + ) + @table = table + @name = name + @unique = unique + @columns = columns + @lengths = concise_options(lengths) + @orders = concise_options(orders) + @opclasses = concise_options(opclasses) + @where = where + @type = type + @using = using + @comment = comment + end + + private + def concise_options(options) + if columns.size == options.size && options.values.uniq.size == 1 + options.values.first + else + options + end + end + end # Abstract representation of a column definition. Instances of this type # are typically created by methods in TableDefinition, and added to the @@ -58,6 +96,15 @@ module ActiveRecord options[:primary_key] != default_primary_key end + def validate? + options.fetch(:validate, true) + end + alias validated? validate? + + def export_name_on_schema_dump? + name !~ ActiveRecord::SchemaDumper.fk_ignore_pattern + end + def defined_for?(to_table_ord = nil, to_table: nil, **options) if to_table_ord self.to_table == to_table_ord.to_s @@ -108,24 +155,15 @@ module ActiveRecord end end - # TODO Change this to private once we've dropped Ruby 2.2 support. - # Workaround for Ruby 2.2 "private attribute?" warning. - protected - - attr_reader :name, :polymorphic, :index, :foreign_key, :type, :options - private + attr_reader :name, :polymorphic, :index, :foreign_key, :type, :options - def as_options(value, default = {}) - if value.is_a?(Hash) - value - else - default - end + def as_options(value) + value.is_a?(Hash) ? value : {} end def polymorphic_options - as_options(polymorphic, options) + as_options(polymorphic).merge(options.slice(:null, :first, :after)) end def index_options @@ -181,6 +219,7 @@ module ActiveRecord :decimal, :float, :integer, + :json, :string, :text, :time, @@ -317,8 +356,12 @@ module ActiveRecord type = type.to_sym if type options = options.dup - if @columns_hash[name] && @columns_hash[name].primary_key? - raise ArgumentError, "you can't redefine the primary key column '#{name}'. To define a custom primary key, pass { id: false } to create_table." + if @columns_hash[name] + if @columns_hash[name].primary_key? + raise ArgumentError, "you can't redefine the primary key column '#{name}'. To define a custom primary key, pass { id: false } to create_table." + else + raise ArgumentError, "you can't define an already defined column '#{name}'." + end end index_options = options.delete(:index) @@ -373,6 +416,9 @@ module ActiveRecord alias :belongs_to :references def new_column_definition(name, type, **options) # :nodoc: + if integer_like_primary_key?(type, options) + type = integer_like_primary_key_type(type, options) + end type = aliased_types(type.to_s, type) options[:primary_key] ||= type == :primary_key options[:null] = false if options[:primary_key] @@ -387,6 +433,14 @@ module ActiveRecord def aliased_types(name, fallback) "timestamp" == name ? :datetime : fallback end + + def integer_like_primary_key?(type, options) + options[:primary_key] && [:integer, :bigint].include?(type) && !options.key?(:default) + end + + def integer_like_primary_key_type(type, options) + type + end end class AlterTable # :nodoc: @@ -447,6 +501,9 @@ module ActiveRecord # t.date # t.binary # t.boolean + # t.foreign_key + # t.json + # t.virtual # t.remove # t.remove_references # t.remove_belongs_to @@ -611,19 +668,19 @@ module ActiveRecord # Adds a foreign key. # - # t.foreign_key(:authors) + # t.foreign_key(:authors) # # See {connection.add_foreign_key}[rdoc-ref:SchemaStatements#add_foreign_key] - def foreign_key(*args) # :nodoc: + def foreign_key(*args) @base.add_foreign_key(name, *args) end # Checks to see if a foreign key exists. # - # t.foreign_key(:authors) unless t.foreign_key_exists?(:authors) + # t.foreign_key(:authors) unless t.foreign_key_exists?(:authors) # # See {connection.foreign_key_exists?}[rdoc-ref:SchemaStatements#foreign_key_exists?] - def foreign_key_exists?(*args) # :nodoc: + def foreign_key_exists?(*args) @base.foreign_key_exists?(name, *args) end end diff --git a/activerecord/lib/active_record/connection_adapters/abstract/schema_dumper.rb b/activerecord/lib/active_record/connection_adapters/abstract/schema_dumper.rb index 34036d8a1d..622e00fffb 100644 --- a/activerecord/lib/active_record/connection_adapters/abstract/schema_dumper.rb +++ b/activerecord/lib/active_record/connection_adapters/abstract/schema_dumper.rb @@ -1,63 +1,38 @@ +# frozen_string_literal: true + module ActiveRecord module ConnectionAdapters # :nodoc: - # The goal of this module is to move Adapter specific column - # definitions to the Adapter instead of having it in the schema - # dumper itself. This code represents the normal case. - # We can then redefine how certain data types may be handled in the schema dumper on the - # Adapter level by over-writing this code inside the database specific adapters - module ColumnDumper - def column_spec(column) - [schema_type_with_virtual(column), prepare_column_options(column)] - end - - def column_spec_for_primary_key(column) - return {} if default_primary_key?(column) - spec = { id: schema_type(column).inspect } - spec.merge!(prepare_column_options(column).except!(:null)) - spec[:default] ||= "nil" if explicit_primary_key_default?(column) - spec + class SchemaDumper < SchemaDumper # :nodoc: + def self.create(connection, options) + new(connection, options) end - # This can be overridden on an Adapter level basis to support other - # extended datatypes (Example: Adding an array option in the - # PostgreSQL::ColumnDumper) - def prepare_column_options(column) - spec = {} - - if limit = schema_limit(column) - spec[:limit] = limit - end - - if precision = schema_precision(column) - spec[:precision] = precision + private + def column_spec(column) + [schema_type_with_virtual(column), prepare_column_options(column)] end - if scale = schema_scale(column) - spec[:scale] = scale + def column_spec_for_primary_key(column) + return {} if default_primary_key?(column) + spec = { id: schema_type(column).inspect } + spec.merge!(prepare_column_options(column).except!(:null)) + spec[:default] ||= "nil" if explicit_primary_key_default?(column) + spec end - default = schema_default(column) if column.has_default? - spec[:default] = default unless default.nil? - - spec[:null] = "false" unless column.null - - if collation = schema_collation(column) - spec[:collation] = collation + def prepare_column_options(column) + spec = {} + spec[:limit] = schema_limit(column) + spec[:precision] = schema_precision(column) + spec[:scale] = schema_scale(column) + spec[:default] = schema_default(column) + spec[:null] = "false" unless column.null + spec[:collation] = schema_collation(column) + spec[:comment] = column.comment.inspect if column.comment.present? + spec.compact! + spec end - spec[:comment] = column.comment.inspect if column.comment.present? - - spec - end - - # Lists the valid migration options - def migration_keys # :nodoc: - column_options_keys - end - deprecate :migration_keys - - private - def default_primary_key?(column) schema_type(column) == :bigint end @@ -67,7 +42,7 @@ module ActiveRecord end def schema_type_with_virtual(column) - if supports_virtual_columns? && column.virtual? + if @connection.supports_virtual_columns? && column.virtual? :virtual else schema_type(column) @@ -84,7 +59,7 @@ module ActiveRecord def schema_limit(column) limit = column.limit unless column.bigint? - limit.inspect if limit && limit != native_database_types[column.type][:limit] + limit.inspect if limit && limit != @connection.native_database_types[column.type][:limit] end def schema_precision(column) @@ -96,7 +71,8 @@ module ActiveRecord end def schema_default(column) - type = lookup_cast_type_from_column(column) + return unless column.has_default? + type = @connection.lookup_cast_type_from_column(column) default = type.deserialize(column.default) if default.nil? schema_expression(column) diff --git a/activerecord/lib/active_record/connection_adapters/abstract/schema_statements.rb b/activerecord/lib/active_record/connection_adapters/abstract/schema_statements.rb index c44215cd43..4a6e915b66 100644 --- a/activerecord/lib/active_record/connection_adapters/abstract/schema_statements.rb +++ b/activerecord/lib/active_record/connection_adapters/abstract/schema_statements.rb @@ -1,6 +1,8 @@ +# frozen_string_literal: true + require "active_record/migration/join_table" require "active_support/core_ext/string/access" -require "digest" +require "digest/sha2" module ActiveRecord module ConnectionAdapters # :nodoc: @@ -31,6 +33,8 @@ module ActiveRecord # Returns the relation names useable to back Active Record models. # For most adapters this means all #tables and #views. def data_sources + query_values(data_source_sql, "SCHEMA") + rescue NotImplementedError tables | views end @@ -39,12 +43,14 @@ module ActiveRecord # data_source_exists?(:ebooks) # def data_source_exists?(name) + query_values(data_source_sql(name), "SCHEMA").any? if name.present? + rescue NotImplementedError data_sources.include?(name.to_s) end # Returns an array of table names defined in the database. def tables - raise NotImplementedError, "#tables is not implemented" + query_values(data_source_sql(type: "BASE TABLE"), "SCHEMA") end # Checks to see if the table +table_name+ exists on the database. @@ -52,12 +58,14 @@ module ActiveRecord # table_exists?(:developers) # def table_exists?(table_name) + query_values(data_source_sql(table_name, type: "BASE TABLE"), "SCHEMA").any? if table_name.present? + rescue NotImplementedError tables.include?(table_name.to_s) end # Returns an array of view names defined in the database. def views - raise NotImplementedError, "#views is not implemented" + query_values(data_source_sql(type: "VIEW"), "SCHEMA") end # Checks to see if the view +view_name+ exists on the database. @@ -65,11 +73,13 @@ module ActiveRecord # view_exists?(:ebooks) # def view_exists?(view_name) + query_values(data_source_sql(view_name, type: "VIEW"), "SCHEMA").any? if view_name.present? + rescue NotImplementedError views.include?(view_name.to_s) end # Returns an array of indexes for the given table. - def indexes(table_name, name = nil) + def indexes(table_name) raise NotImplementedError, "#indexes is not implemented" end @@ -97,10 +107,12 @@ module ActiveRecord indexes(table_name).any? { |i| checks.all? { |check| check[i] } } end - # Returns an array of Column objects for the table specified by +table_name+. - # See the concrete implementation for details on the expected parameter values. + # Returns an array of +Column+ objects for the table specified by +table_name+. def columns(table_name) - raise NotImplementedError, "#columns is not implemented" + table_name = table_name.to_s + column_definitions(table_name).map do |field| + new_column_from_field(table_name, field) + end end # Checks to see if a column exists in a given table. @@ -178,6 +190,8 @@ module ActiveRecord # The name of the primary key, if one is to be added automatically. # Defaults to +id+. If <tt>:id</tt> is false, then this option is ignored. # + # If an array is passed, a composite primary key will be created. + # # Note that Active Record models will automatically detect their # primary key. This can be avoided by using # {self.primary_key=}[rdoc-ref:AttributeMethods::PrimaryKey::ClassMethods#primary_key=] on the model @@ -202,7 +216,7 @@ module ActiveRecord # generates: # # CREATE TABLE suppliers ( - # id int auto_increment PRIMARY KEY + # id bigint auto_increment PRIMARY KEY # ) ENGINE=InnoDB DEFAULT CHARSET=utf8 # # ====== Rename the primary key column @@ -214,7 +228,7 @@ module ActiveRecord # generates: # # CREATE TABLE objects ( - # guid int auto_increment PRIMARY KEY, + # guid bigint auto_increment PRIMARY KEY, # name varchar(80) # ) # @@ -231,18 +245,35 @@ module ActiveRecord # label varchar # ) # + # ====== Create a composite primary key + # + # create_table(:orders, primary_key: [:product_id, :client_id]) do |t| + # t.belongs_to :product + # t.belongs_to :client + # end + # + # generates: + # + # CREATE TABLE order ( + # product_id bigint NOT NULL, + # client_id bigint NOT NULL + # ); + # + # ALTER TABLE ONLY "orders" + # ADD CONSTRAINT orders_pkey PRIMARY KEY (product_id, client_id); + # # ====== Do not add a primary key column # # create_table(:categories_suppliers, id: false) do |t| - # t.column :category_id, :integer - # t.column :supplier_id, :integer + # t.column :category_id, :bigint + # t.column :supplier_id, :bigint # end # # generates: # # CREATE TABLE categories_suppliers ( - # category_id int, - # supplier_id int + # category_id bigint, + # supplier_id bigint # ) # # ====== Create a temporary table based on a query @@ -274,7 +305,7 @@ module ActiveRecord yield td if block_given? if options[:force] - drop_table(table_name, **options, if_exists: true) + drop_table(table_name, options.merge(if_exists: true)) end result = execute schema_creation.accept td @@ -330,22 +361,20 @@ module ActiveRecord # generates: # # CREATE TABLE assemblies_parts ( - # assembly_id int NOT NULL, - # part_id int NOT NULL, + # assembly_id bigint NOT NULL, + # part_id bigint NOT NULL, # ) ENGINE=InnoDB DEFAULT CHARSET=utf8 # - def create_join_table(table_1, table_2, options = {}) + def create_join_table(table_1, table_2, column_options: {}, **options) join_table_name = find_join_table_name(table_1, table_2, options) - column_options = options.delete(:column_options) || {} - column_options.reverse_merge!(null: false) - type = column_options.delete(:type) || :integer + column_options.reverse_merge!(null: false, index: false) - t1_column, t2_column = [table_1, table_2].map { |t| t.to_s.singularize.foreign_key } + t1_ref, t2_ref = [table_1, table_2].map { |t| t.to_s.singularize } create_table(join_table_name, options.merge!(id: false)) do |td| - td.send type, t1_column, column_options - td.send type, t2_column, column_options + td.references t1_ref, column_options + td.references t2_ref, column_options yield td if block_given? end end @@ -377,6 +406,8 @@ module ActiveRecord # # Defaults to false. # + # Only supported on the MySQL and PostgreSQL adapter, ignored elsewhere. + # # ====== Add a column # # change_table(:suppliers) do |t| @@ -401,7 +432,7 @@ module ActiveRecord # t.references :company # end # - # Creates a <tt>company_id(integer)</tt> column. + # Creates a <tt>company_id(bigint)</tt> column. # # ====== Add a polymorphic foreign key column # @@ -409,7 +440,7 @@ module ActiveRecord # t.belongs_to :company, polymorphic: true # end # - # Creates <tt>company_type(varchar)</tt> and <tt>company_id(integer)</tt> columns. + # Creates <tt>company_type(varchar)</tt> and <tt>company_id(bigint)</tt> columns. # # ====== Remove a column # @@ -482,15 +513,20 @@ module ActiveRecord # * <tt>:limit</tt> - # Requests a maximum column length. This is the number of characters for a <tt>:string</tt> column # and number of bytes for <tt>:text</tt>, <tt>:binary</tt> and <tt>:integer</tt> columns. + # This option is ignored by some backends. # * <tt>:default</tt> - # The column's default value. Use +nil+ for +NULL+. # * <tt>:null</tt> - - # Allows or disallows +NULL+ values in the column. This option could - # have been named <tt>:null_allowed</tt>. + # Allows or disallows +NULL+ values in the column. # * <tt>:precision</tt> - # Specifies the precision for the <tt>:decimal</tt> and <tt>:numeric</tt> columns. # * <tt>:scale</tt> - # Specifies the scale for the <tt>:decimal</tt> and <tt>:numeric</tt> columns. + # * <tt>:collation</tt> - + # Specifies the collation for a <tt>:string</tt> or <tt>:text</tt> column. If not specified, the + # column will have the same collation as the table. + # * <tt>:comment</tt> - + # Specifies the comment for the column. This option is ignored by some backends. # # Note: The precision is the total number of significant digits, # and the scale is the number of digits that can be stored following @@ -566,8 +602,9 @@ module ActiveRecord # The +type+ and +options+ parameters will be ignored if present. It can be helpful # to provide these in a migration's +change+ method so it can be reverted. # In that case, +type+ and +options+ will be used by #add_column. + # Indexes on the column are automatically removed. def remove_column(table_name, column_name, type = nil, options = {}) - execute "ALTER TABLE #{quote_table_name(table_name)} DROP #{quote_column_name(column_name)}" + execute "ALTER TABLE #{quote_table_name(table_name)} #{remove_column_for_alter(table_name, column_name, type, options)}" end # Changes the column's definition according to the new options. @@ -682,7 +719,7 @@ module ActiveRecord # # CREATE INDEX by_branch_desc_party ON accounts(branch_id DESC, party_id ASC, surname) # - # Note: MySQL doesn't yet support index order (it accepts the syntax but ignores it). + # Note: MySQL only supports index order from 8.0.1 onwards (earlier versions accepted the syntax but ignored it). # # ====== Creating a partial index # @@ -705,6 +742,19 @@ module ActiveRecord # # Note: only supported by PostgreSQL and MySQL # + # ====== Creating an index with a specific operator class + # + # add_index(:developers, :name, using: 'gist', opclass: :gist_trgm_ops) + # # CREATE INDEX developers_on_name ON developers USING gist (name gist_trgm_ops) -- PostgreSQL + # + # add_index(:developers, [:name, :city], using: 'gist', opclass: { city: :gist_trgm_ops }) + # # CREATE INDEX developers_on_name_and_city ON developers USING gist (name, city gist_trgm_ops) -- PostgreSQL + # + # add_index(:developers, [:name, :city], using: 'gist', opclass: :gist_trgm_ops) + # # CREATE INDEX developers_on_name_and_city ON developers USING gist (name gist_trgm_ops, city gist_trgm_ops) -- PostgreSQL + # + # Note: only supported by PostgreSQL + # # ====== Creating an index with a specific type # # add_index(:developers, :name, type: :fulltext) @@ -751,7 +801,7 @@ module ActiveRecord def rename_index(table_name, old_name, new_name) validate_index_length!(table_name, new_name) - # this is a naive implementation; some DBs may support this more efficiently (Postgres, for instance) + # this is a naive implementation; some DBs may support this more efficiently (PostgreSQL, for instance) old_index_def = indexes(table_name).detect { |i| i.name == old_name } return unless old_index_def add_index(table_name, old_index_def.columns, name: new_name, unique: old_index_def.unique) @@ -773,24 +823,19 @@ module ActiveRecord end # Verifies the existence of an index with a given name. - def index_name_exists?(table_name, index_name, default = nil) - unless default.nil? - ActiveSupport::Deprecation.warn(<<-MSG.squish) - Passing default to #index_name_exists? is deprecated without replacement. - MSG - end + def index_name_exists?(table_name, index_name) index_name = index_name.to_s indexes(table_name).detect { |i| i.name == index_name } end - # Adds a reference. The reference column is an integer by default, + # Adds a reference. The reference column is a bigint by default, # the <tt>:type</tt> option can be used to specify a different type. # Optionally adds a +_type+ column, if <tt>:polymorphic</tt> option is provided. # #add_reference and #add_belongs_to are acceptable. # # The +options+ hash can include the following keys: # [<tt>:type</tt>] - # The reference column type. Defaults to +:integer+. + # The reference column type. Defaults to +:bigint+. # [<tt>:index</tt>] # Add an appropriate index. Defaults to true. # See #add_index for usage of this option. @@ -801,7 +846,7 @@ module ActiveRecord # [<tt>:null</tt>] # Whether the column allows nulls. Defaults to true. # - # ====== Create a user_id integer column + # ====== Create a user_id bigint column # # add_reference(:products, :user) # @@ -858,7 +903,7 @@ module ActiveRecord foreign_key_options = { to_table: reference_name } end foreign_key_options[:column] ||= "#{ref_name}_id" - remove_foreign_key(table_name, **foreign_key_options) + remove_foreign_key(table_name, foreign_key_options) end remove_column(table_name, "#{ref_name}_id") @@ -914,6 +959,8 @@ module ActiveRecord # Action that happens <tt>ON DELETE</tt>. Valid values are +:nullify+, +:cascade+ and +:restrict+ # [<tt>:on_update</tt>] # Action that happens <tt>ON UPDATE</tt>. Valid values are +:nullify+, +:cascade+ and +:restrict+ + # [<tt>:validate</tt>] + # (Postgres only) Specify whether or not the constraint should be validated. Defaults to +true+. def add_foreign_key(from_table, to_table, options = {}) return unless supports_foreign_keys? @@ -937,11 +984,18 @@ module ActiveRecord # # remove_foreign_key :accounts, column: :owner_id # + # Removes the foreign key on +accounts.owner_id+. + # + # remove_foreign_key :accounts, to_table: :owners + # # Removes the foreign key named +special_fk_name+ on the +accounts+ table. # # remove_foreign_key :accounts, name: :special_fk_name # - # The +options+ hash accepts the same keys as SchemaStatements#add_foreign_key. + # The +options+ hash accepts the same keys as SchemaStatements#add_foreign_key + # with an addition of + # [<tt>:to_table</tt>] + # The name of the table that contains the referenced primary key. def remove_foreign_key(from_table, options_or_to_table = {}) return unless supports_foreign_keys? @@ -968,16 +1022,6 @@ module ActiveRecord foreign_key_for(from_table, options_or_to_table).present? end - def foreign_key_for(from_table, options_or_to_table = {}) # :nodoc: - return unless supports_foreign_keys? - foreign_keys(from_table).detect { |fk| fk.defined_for? options_or_to_table } - end - - def foreign_key_for!(from_table, options_or_to_table = {}) # :nodoc: - foreign_key_for(from_table, options_or_to_table) || \ - raise(ArgumentError, "Table '#{from_table}' has no foreign key for #{options_or_to_table}") - end - def foreign_key_column_for(table_name) # :nodoc: prefix = Base.table_name_prefix suffix = Base.table_name_suffix @@ -993,32 +1037,9 @@ module ActiveRecord end def dump_schema_information #:nodoc: - versions = ActiveRecord::SchemaMigration.order("version").pluck(:version) - insert_versions_sql(versions) - end - - def insert_versions_sql(versions) # :nodoc: - sm_table = quote_table_name(ActiveRecord::Migrator.schema_migrations_table_name) - - if versions.is_a?(Array) - sql = "INSERT INTO #{sm_table} (version) VALUES\n" - sql << versions.map { |v| "(#{quote(v)})" }.join(",\n") - sql << ";\n\n" - sql - else - "INSERT INTO #{sm_table} (version) VALUES (#{quote(versions)});" - end - end - - def initialize_schema_migrations_table # :nodoc: - ActiveRecord::SchemaMigration.create_table + versions = ActiveRecord::SchemaMigration.all_versions + insert_versions_sql(versions) if versions.any? end - deprecate :initialize_schema_migrations_table - - def initialize_internal_metadata_table # :nodoc: - ActiveRecord::InternalMetadata.create_table - end - deprecate :initialize_internal_metadata_table def internal_string_options_for_primary_key # :nodoc: { primary_key: true } @@ -1027,12 +1048,11 @@ module ActiveRecord def assume_migrated_upto_version(version, migrations_paths) migrations_paths = Array(migrations_paths) version = version.to_i - sm_table = quote_table_name(ActiveRecord::Migrator.schema_migrations_table_name) + sm_table = quote_table_name(ActiveRecord::SchemaMigration.table_name) - migrated = select_values("SELECT version FROM #{sm_table}").map(&:to_i) - paths = migrations_paths.map { |p| "#{p}/[0-9]*_*.rb" } - versions = Dir[*paths].map do |filename| - filename.split("/").last.split("_").first.to_i + migrated = ActiveRecord::SchemaMigration.all_versions.map(&:to_i) + versions = migration_context.migration_files.map do |file| + migration_context.parse_migration_filename(file).first.to_i end unless migrated.include?(version) @@ -1044,13 +1064,7 @@ module ActiveRecord if (duplicate = inserting.detect { |v| inserting.count(v) > 1 }) raise "Duplicate migration #{duplicate}. Please renumber your migrations to resolve the conflict." end - if supports_multi_insert? - execute insert_versions_sql(inserting) - else - inserting.each do |v| - execute insert_versions_sql(v) - end - end + execute insert_versions_sql(inserting) end end @@ -1126,7 +1140,7 @@ module ActiveRecord def add_index_options(table_name, column_name, comment: nil, **options) # :nodoc: column_names = index_column_names(column_name) - options.assert_valid_keys(:unique, :order, :name, :where, :length, :internal, :using, :algorithm, :type) + options.assert_valid_keys(:unique, :order, :name, :where, :length, :internal, :using, :algorithm, :type, :opclass) index_type = options[:type].to_s if options.key?(:type) index_type ||= options[:unique] ? "UNIQUE" : "" @@ -1165,30 +1179,36 @@ module ActiveRecord end # Changes the comment for a column or removes it if +nil+. - def change_column_comment(table_name, column_name, comment) #:nodoc: + def change_column_comment(table_name, column_name, comment) raise NotImplementedError, "#{self.class} does not support changing column comments" end + def create_schema_dumper(options) # :nodoc: + SchemaDumper.create(self, options) + end + private def column_options_keys [:limit, :precision, :scale, :default, :null, :collation, :comment] end def add_index_sort_order(quoted_columns, **options) - if order = options[:order] - case order - when Hash - order = order.symbolize_keys - quoted_columns.each { |name, column| column << " #{order[name].upcase}" if order[name].present? } - when String - quoted_columns.each { |name, column| column << " #{order.upcase}" if order.present? } - end + orders = options_for_index_columns(options[:order]) + quoted_columns.each do |name, column| + column << " #{orders[name].upcase}" if orders[name].present? end + end - quoted_columns + def options_for_index_columns(options) + if options.is_a?(Hash) + options.symbolize_keys + else + Hash.new { |hash, column| hash[column] = options } + end end - # Overridden by the MySQL adapter for supporting index lengths + # Overridden by the MySQL adapter for supporting index lengths and by + # the PostgreSQL adapter for supporting operator classes. def add_options_for_index_columns(quoted_columns, **options) if supports_index_sort_order? quoted_columns = add_index_sort_order(quoted_columns, options) @@ -1256,6 +1276,10 @@ module ActiveRecord end end + def schema_creation + SchemaCreation.new(self) + end + def create_table_definition(*args) TableDefinition.new(*args) end @@ -1264,6 +1288,17 @@ module ActiveRecord AlterTable.new create_table_definition(name) end + def fetch_type_metadata(sql_type) + cast_type = lookup_cast_type(sql_type) + SqlTypeMetadata.new( + sql_type: sql_type, + type: cast_type.type, + limit: cast_type.limit, + precision: cast_type.precision, + scale: cast_type.scale, + ) + end + def index_column_names(column_names) if column_names.is_a?(String) && /\W/.match?(column_names) column_names @@ -1281,13 +1316,32 @@ module ActiveRecord end def foreign_key_name(table_name, options) - identifier = "#{table_name}_#{options.fetch(:column)}_fk" - hashed_identifier = Digest::SHA256.hexdigest(identifier).first(10) options.fetch(:name) do + identifier = "#{table_name}_#{options.fetch(:column)}_fk" + hashed_identifier = Digest::SHA256.hexdigest(identifier).first(10) + "fk_rails_#{hashed_identifier}" end end + def foreign_key_for(from_table, options_or_to_table = {}) + return unless supports_foreign_keys? + foreign_keys(from_table).detect { |fk| fk.defined_for? options_or_to_table } + end + + def foreign_key_for!(from_table, options_or_to_table = {}) + foreign_key_for(from_table, options_or_to_table) || \ + raise(ArgumentError, "Table '#{from_table}' has no foreign key for #{options_or_to_table}") + end + + def extract_foreign_key_action(specifier) + case specifier + when "CASCADE"; :cascade + when "SET NULL"; :nullify + when "RESTRICT"; :restrict + end + end + def validate_index_length!(table_name, new_name, internal = false) max_index_length = internal ? index_name_length : allowed_index_name_length @@ -1307,6 +1361,41 @@ module ActiveRecord def can_remove_index_by_name?(options) options.is_a?(Hash) && options.key?(:name) && options.except(:name, :algorithm).empty? end + + def add_column_for_alter(table_name, column_name, type, options = {}) + td = create_table_definition(table_name) + cd = td.new_column_definition(column_name, type, options) + schema_creation.accept(AddColumnDefinition.new(cd)) + end + + def remove_column_for_alter(table_name, column_name, type = nil, options = {}) + "DROP COLUMN #{quote_column_name(column_name)}" + end + + def remove_columns_for_alter(table_name, *column_names) + column_names.map { |column_name| remove_column_for_alter(table_name, column_name) } + end + + def insert_versions_sql(versions) + sm_table = quote_table_name(ActiveRecord::SchemaMigration.table_name) + + if versions.is_a?(Array) + sql = "INSERT INTO #{sm_table} (version) VALUES\n".dup + sql << versions.map { |v| "(#{quote(v)})" }.join(",\n") + sql << ";\n\n" + sql + else + "INSERT INTO #{sm_table} (version) VALUES (#{quote(versions)});" + end + end + + def data_source_sql(name = nil, type: nil) + raise NotImplementedError + end + + def quoted_scope(name = nil, type: nil) + raise NotImplementedError + end end end end diff --git a/activerecord/lib/active_record/connection_adapters/abstract/transaction.rb b/activerecord/lib/active_record/connection_adapters/abstract/transaction.rb index 6bb072dd73..564b226b39 100644 --- a/activerecord/lib/active_record/connection_adapters/abstract/transaction.rb +++ b/activerecord/lib/active_record/connection_adapters/abstract/transaction.rb @@ -1,10 +1,15 @@ +# frozen_string_literal: true + module ActiveRecord module ConnectionAdapters class TransactionState - VALID_STATES = Set.new([:committed, :rolledback, nil]) - def initialize(state = nil) @state = state + @children = [] + end + + def add_child(state) + @children << state end def finalized? @@ -12,11 +17,23 @@ module ActiveRecord end def committed? - @state == :committed + @state == :committed || @state == :fully_committed + end + + def fully_committed? + @state == :fully_committed end def rolledback? - @state == :rolledback + @state == :rolledback || @state == :fully_rolledback + end + + def fully_rolledback? + @state == :fully_rolledback + end + + def fully_completed? + completed? end def completed? @@ -24,10 +41,43 @@ module ActiveRecord end def set_state(state) - unless VALID_STATES.include?(state) + ActiveSupport::Deprecation.warn(<<-MSG.squish) + The set_state method is deprecated and will be removed in + Rails 6.0. Please use rollback! or commit! to set transaction + state directly. + MSG + case state + when :rolledback + rollback! + when :committed + commit! + when nil + nullify! + else raise ArgumentError, "Invalid transaction state: #{state}" end - @state = state + end + + def rollback! + @children.each { |c| c.rollback! } + @state = :rolledback + end + + def full_rollback! + @children.each { |c| c.rollback! } + @state = :fully_rolledback + end + + def commit! + @state = :committed + end + + def full_commit! + @state = :fully_committed + end + + def nullify! + @state = nil end end @@ -41,13 +91,14 @@ module ActiveRecord end class Transaction #:nodoc: - attr_reader :connection, :state, :records, :savepoint_name - attr_writer :joinable + attr_reader :connection, :state, :records, :savepoint_name, :isolation_level def initialize(connection, options, run_commit_callbacks: false) @connection = connection @state = TransactionState.new @records = [] + @isolation_level = options[:isolation] + @materialized = false @joinable = options.fetch(:joinable, true) @run_commit_callbacks = run_commit_callbacks end @@ -56,8 +107,12 @@ module ActiveRecord records << record end - def rollback - @state.set_state(:rolledback) + def materialize! + @materialized = true + end + + def materialized? + @materialized end def rollback_records @@ -71,10 +126,6 @@ module ActiveRecord end end - def commit - @state.set_state(:committed) - end - def before_commit_records records.uniq.each(&:before_committed!) if @run_commit_callbacks end @@ -100,45 +151,55 @@ module ActiveRecord end class SavepointTransaction < Transaction - def initialize(connection, savepoint_name, options, *args) - super(connection, options, *args) - if options[:isolation] + def initialize(connection, savepoint_name, parent_transaction, *args) + super(connection, *args) + + parent_transaction.state.add_child(@state) + + if isolation_level raise ActiveRecord::TransactionIsolationError, "cannot set transaction isolation in a nested transaction" end - connection.create_savepoint(@savepoint_name = savepoint_name) + + @savepoint_name = savepoint_name end - def rollback - connection.rollback_to_savepoint(savepoint_name) + def materialize! + connection.create_savepoint(savepoint_name) super end + def rollback + connection.rollback_to_savepoint(savepoint_name) if materialized? + @state.rollback! + end + def commit - connection.release_savepoint(savepoint_name) - super + connection.release_savepoint(savepoint_name) if materialized? + @state.commit! end def full_rollback?; false; end end class RealTransaction < Transaction - def initialize(connection, options, *args) - super - if options[:isolation] - connection.begin_isolated_db_transaction(options[:isolation]) + def materialize! + if isolation_level + connection.begin_isolated_db_transaction(isolation_level) else connection.begin_db_transaction end + + super end def rollback - connection.rollback_db_transaction - super + connection.rollback_db_transaction if materialized? + @state.full_rollback! end def commit - connection.commit_db_transaction - super + connection.commit_db_transaction if materialized? + @state.full_commit! end end @@ -146,60 +207,103 @@ module ActiveRecord def initialize(connection) @stack = [] @connection = connection + @has_unmaterialized_transactions = false + @materializing_transactions = false + @lazy_transactions_enabled = true end def begin_transaction(options = {}) - run_commit_callbacks = !current_transaction.joinable? - transaction = - if @stack.empty? - RealTransaction.new(@connection, options, run_commit_callbacks: run_commit_callbacks) - else - SavepointTransaction.new(@connection, "active_record_#{@stack.size}", options, - run_commit_callbacks: run_commit_callbacks) - end + @connection.lock.synchronize do + run_commit_callbacks = !current_transaction.joinable? + transaction = + if @stack.empty? + RealTransaction.new(@connection, options, run_commit_callbacks: run_commit_callbacks) + else + SavepointTransaction.new(@connection, "active_record_#{@stack.size}", @stack.last, options, + run_commit_callbacks: run_commit_callbacks) + end - @stack.push(transaction) - transaction + transaction.materialize! unless @connection.supports_lazy_transactions? && lazy_transactions_enabled? + @stack.push(transaction) + @has_unmaterialized_transactions = true if @connection.supports_lazy_transactions? + transaction + end end - def commit_transaction - transaction = @stack.last + def disable_lazy_transactions! + materialize_transactions + @lazy_transactions_enabled = false + end + + def enable_lazy_transactions! + @lazy_transactions_enabled = true + end + + def lazy_transactions_enabled? + @lazy_transactions_enabled + end + + def materialize_transactions + return if @materializing_transactions + return unless @has_unmaterialized_transactions - begin - transaction.before_commit_records - ensure - @stack.pop + @connection.lock.synchronize do + begin + @materializing_transactions = true + @stack.each { |t| t.materialize! unless t.materialized? } + ensure + @materializing_transactions = false + end + @has_unmaterialized_transactions = false end + end + + def commit_transaction + @connection.lock.synchronize do + transaction = @stack.last - transaction.commit - transaction.commit_records + begin + transaction.before_commit_records + ensure + @stack.pop + end + + transaction.commit + transaction.commit_records + end end def rollback_transaction(transaction = nil) - transaction ||= @stack.pop - transaction.rollback - transaction.rollback_records + @connection.lock.synchronize do + transaction ||= @stack.pop + transaction.rollback + transaction.rollback_records + end end def within_new_transaction(options = {}) - transaction = begin_transaction options - yield - rescue Exception => error - if transaction - rollback_transaction - after_failure_actions(transaction, error) - end - raise - ensure - unless error - if Thread.current.status == "aborting" - rollback_transaction if transaction - else - begin - commit_transaction - rescue Exception - rollback_transaction(transaction) unless transaction.state.completed? - raise + @connection.lock.synchronize do + begin + transaction = begin_transaction options + yield + rescue Exception => error + if transaction + rollback_transaction + after_failure_actions(transaction, error) + end + raise + ensure + unless error + if Thread.current.status == "aborting" + rollback_transaction if transaction + else + begin + commit_transaction if transaction + rescue Exception + rollback_transaction(transaction) unless transaction.state.completed? + raise + end + end end end end diff --git a/activerecord/lib/active_record/connection_adapters/abstract_adapter.rb b/activerecord/lib/active_record/connection_adapters/abstract_adapter.rb index ef1d9f81a9..42ed9ce82d 100644 --- a/activerecord/lib/active_record/connection_adapters/abstract_adapter.rb +++ b/activerecord/lib/active_record/connection_adapters/abstract_adapter.rb @@ -1,11 +1,15 @@ -require "active_record/type" +# frozen_string_literal: true + require "active_record/connection_adapters/determine_if_preparable_visitor" require "active_record/connection_adapters/schema_cache" require "active_record/connection_adapters/sql_type_metadata" require "active_record/connection_adapters/abstract/schema_dumper" require "active_record/connection_adapters/abstract/schema_creation" +require "active_support/concurrency/load_interlock_aware_monitor" require "arel/collectors/bind" +require "arel/collectors/composite" require "arel/collectors/sql_string" +require "arel/collectors/substitute_binds" module ActiveRecord module ConnectionAdapters # :nodoc: @@ -68,17 +72,20 @@ module ActiveRecord include Quoting, DatabaseStatements, SchemaStatements include DatabaseLimits include QueryCache - include ColumnDumper include Savepoints SIMPLE_INT = /\A\d+\z/ attr_accessor :visitor, :pool - attr_reader :schema_cache, :owner, :logger + attr_reader :schema_cache, :owner, :logger, :prepared_statements, :lock alias :in_use? :owner + set_callback :checkin, :after, :enable_lazy_transactions! + def self.type_cast_config_to_integer(config) - if config =~ SIMPLE_INT + if config.is_a?(Integer) + config + elsif SIMPLE_INT.match?(config) config.to_i else config @@ -93,8 +100,6 @@ module ActiveRecord end end - attr_reader :prepared_statements - def initialize(connection, logger = nil, config = {}) # :nodoc: super() @@ -104,10 +109,11 @@ module ActiveRecord @logger = logger @config = config @pool = nil + @idle_since = Concurrent.monotonic_time @schema_cache = SchemaCache.new self @quoted_column_names, @quoted_table_names = {}, {} @visitor = arel_visitor - @lock = Monitor.new + @lock = ActiveSupport::Concurrency::LoadInterlockAwareMonitor.new if self.class.type_cast_config_to_boolean(config.fetch(:prepared_statements) { true }) @prepared_statements = true @@ -115,6 +121,22 @@ module ActiveRecord else @prepared_statements = false end + + @advisory_locks_enabled = self.class.type_cast_config_to_boolean( + config.fetch(:advisory_locks, true) + ) + end + + def replica? + @config[:replica] || false + end + + def migrations_paths # :nodoc: + @config[:migrations_paths] || Migrator.migrations_paths + end + + def migration_context # :nodoc: + MigrationContext.new(migrations_paths) end class Version @@ -127,53 +149,20 @@ module ActiveRecord def <=>(version_string) @version <=> version_string.split(".").map(&:to_i) end - end - class BindCollector < Arel::Collectors::Bind - def compile(bvs, conn) - casted_binds = bvs.map(&:value_for_database) - super(casted_binds.map { |value| conn.quote(value) }) + def to_s + @version.join(".") end end - class SQLString < Arel::Collectors::SQLString - def compile(bvs, conn) - super(bvs) - end - end - - def collector - if prepared_statements - SQLString.new - else - BindCollector.new - end - end - - def arel_visitor # :nodoc: - Arel::Visitors::ToSql.new(self) - end - def valid_type?(type) # :nodoc: !native_database_types[type].nil? end - def schema_creation - SchemaCreation.new self - end - - # Returns an array of +Column+ objects for the table specified by +table_name+. - def columns(table_name) # :nodoc: - table_name = table_name.to_s - column_definitions(table_name).map do |field| - new_column_from_field(table_name, field) - end - end - # this method must only be called while holding connection pool's mutex def lease if in_use? - msg = "Cannot lease connection, " + msg = "Cannot lease connection, ".dup if @owner == Thread.current msg << "it is already leased by the current thread." else @@ -200,6 +189,7 @@ module ActiveRecord "Current thread: #{Thread.current}." end + @idle_since = Concurrent.monotonic_time @owner = nil else raise ActiveRecordError, "Cannot expire connection, it is not currently leased." @@ -219,6 +209,12 @@ module ActiveRecord end end + # Seconds since this connection was returned to the pool + def seconds_idle # :nodoc: + return 0 if in_use? + Concurrent.monotonic_time - @idle_since + end + def unprepared_statement old_prepared_statements, @prepared_statements = @prepared_statements, false yield @@ -232,16 +228,6 @@ module ActiveRecord self.class::ADAPTER_NAME end - def supports_migrations? # :nodoc: - true - end - deprecate :supports_migrations? - - def supports_primary_key? # :nodoc: - true - end - deprecate :supports_primary_key? - # Does this adapter support DDL rollbacks in transactions? That is, would # CREATE TABLE or ALTER TABLE get rolled back by a transaction? def supports_ddl_transactions? @@ -310,6 +296,11 @@ module ActiveRecord false end + # Does this adapter support creating invalid constraints? + def supports_validate_constraints? + false + end + # Does this adapter support creating foreign key constraints # in the same statement as creating the table? def supports_foreign_keys_in_create? @@ -345,12 +336,22 @@ module ActiveRecord def supports_multi_insert? true end + deprecate :supports_multi_insert? # Does this adapter support virtual columns? def supports_virtual_columns? false end + # Does this adapter support foreign/external tables? + def supports_foreign_tables? + false + end + + def supports_lazy_transactions? + false + end + # This is meant to be implemented by the adapters that support extensions def disable_extension(name) end @@ -359,6 +360,10 @@ module ActiveRecord def enable_extension(name) end + def advisory_locks_enabled? # :nodoc: + supports_advisory_locks? && @advisory_locks_enabled + end + # This is meant to be implemented by the adapters that support advisory # locks # @@ -413,6 +418,19 @@ module ActiveRecord reset_transaction end + # Immediately forget this connection ever existed. Unlike disconnect!, + # this will not communicate with the server. + # + # After calling this method, the behavior of all other methods becomes + # undefined. This is called internally just before a forked process gets + # rid of a connection that belonged to its parent. + def discard! + # This should be overridden by concrete adapters. + # + # Prevent @connection's finalizer from touching the socket, or + # otherwise communicating with its server, when it is collected. + end + # Reset the state of this connection, directing the DBMS to clear # transactions and other connection-related server-side state. Usually a # database-dependent operation. @@ -438,20 +456,18 @@ module ActiveRecord # Checks whether the connection to the database is still active (i.e. not stale). # This is done under the hood by calling #active?. If the connection # is no longer active, then this method will reconnect to the database. - def verify!(*ignored) - if ignored.size > 0 - ActiveSupport::Deprecation.warn("Passing arguments to #verify method of the connection has no effect and has been deprecated. Please remove all arguments from the #verify method call.") - end + def verify! reconnect! unless active? end # Provides access to the underlying database driver for this adapter. For # example, this method returns a Mysql2::Client object in case of Mysql2Adapter, - # and a PGconn object in case of PostgreSQLAdapter. + # and a PG::Connection object in case of PostgreSQLAdapter. # # This is useful for when you need to call a proprietary method such as # PostgreSQL's lo_* methods. def raw_connection + disable_lazy_transactions! @connection end @@ -477,40 +493,8 @@ module ActiveRecord pool.checkin self end - def type_map # :nodoc: - @type_map ||= Type::TypeMap.new.tap do |mapping| - initialize_type_map(mapping) - end - end - - def new_column(name, default, sql_type_metadata, null, table_name, default_function = nil, collation = nil) # :nodoc: - Column.new(name, default, sql_type_metadata, null, table_name, default_function, collation) - end - - def lookup_cast_type(sql_type) # :nodoc: - type_map.lookup(sql_type) - end - def column_name_for_operation(operation, node) # :nodoc: - visitor.accept(node, collector).value - end - - def combine_bind_parameters( - from_clause: [], - join_clause: [], - where_clause: [], - having_clause: [], - limit: nil, - offset: nil - ) # :nodoc: - result = from_clause + join_clause + where_clause + having_clause - if limit - result << limit - end - if offset - result << offset - end - result + visitor.compile(node) end def default_index_type?(index) # :nodoc: @@ -518,8 +502,13 @@ module ActiveRecord end private + def type_map + @type_map ||= Type::TypeMap.new.tap do |mapping| + initialize_type_map(mapping) + end + end - def initialize_type_map(m) + def initialize_type_map(m = type_map) register_class_with_limit m, %r(boolean)i, Type::Boolean register_class_with_limit m, %r(char)i, Type::String register_class_with_limit m, %r(binary)i, Type::Binary @@ -537,6 +526,8 @@ module ActiveRecord m.alias_type %r(number)i, "decimal" m.alias_type %r(double)i, "float" + m.register_type %r(^json)i, Type::Json.new + m.register_type(%r(decimal)i) do |sql_type| scale = extract_scale(sql_type) precision = extract_precision(sql_type) @@ -552,7 +543,7 @@ module ActiveRecord def reload_type_map type_map.clear - initialize_type_map(type_map) + initialize_type_map end def register_class_with_limit(mapping, key, klass) @@ -581,12 +572,7 @@ module ActiveRecord end def extract_limit(sql_type) - case sql_type - when /^bigint/i - 8 - when /\((.*)\)/ - $1.to_i - end + $1.to_i if sql_type =~ /\((.*)\)/ end def translate_exception_class(e, sql) @@ -610,12 +596,14 @@ module ActiveRecord type_casted_binds: type_casted_binds, statement_name: statement_name, connection_id: object_id) do + begin @lock.synchronize do yield end + rescue => e + raise translate_exception_class(e, sql) end - rescue => e - raise translate_exception_class(e, sql) + end end def translate_exception(exception, message) @@ -637,6 +625,24 @@ module ActiveRecord columns(table_name).detect { |c| c.name == column_name } || raise(ActiveRecordError, "No such column: #{table_name}.#{column_name}") end + + def collector + if prepared_statements + Arel::Collectors::Composite.new( + Arel::Collectors::SQLString.new, + Arel::Collectors::Bind.new, + ) + else + Arel::Collectors::SubstituteBinds.new( + self, + Arel::Collectors::SQLString.new, + ) + end + end + + def arel_visitor + Arel::Visitors::ToSql.new(self) + end end end end diff --git a/activerecord/lib/active_record/connection_adapters/abstract_mysql_adapter.rb b/activerecord/lib/active_record/connection_adapters/abstract_mysql_adapter.rb index e3b6327dd8..2d287d56e8 100644 --- a/activerecord/lib/active_record/connection_adapters/abstract_mysql_adapter.rb +++ b/activerecord/lib/active_record/connection_adapters/abstract_mysql_adapter.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require "active_record/connection_adapters/abstract_adapter" require "active_record/connection_adapters/statement_pool" require "active_record/connection_adapters/mysql/column" @@ -6,27 +8,14 @@ require "active_record/connection_adapters/mysql/quoting" require "active_record/connection_adapters/mysql/schema_creation" require "active_record/connection_adapters/mysql/schema_definitions" require "active_record/connection_adapters/mysql/schema_dumper" +require "active_record/connection_adapters/mysql/schema_statements" require "active_record/connection_adapters/mysql/type_metadata" -require "active_support/core_ext/string/strip" - module ActiveRecord module ConnectionAdapters class AbstractMysqlAdapter < AbstractAdapter include MySQL::Quoting - include MySQL::ColumnDumper - - def update_table_definition(table_name, base) # :nodoc: - MySQL::Table.new(table_name, base) - end - - def schema_creation # :nodoc: - MySQL::SchemaCreation.new(self) - end - - def arel_visitor # :nodoc: - Arel::Visitors::MySQL.new(self) - end + include MySQL::SchemaStatements ## # :singleton-method: @@ -35,15 +24,14 @@ module ActiveRecord # to your application.rb file: # # ActiveRecord::ConnectionAdapters::Mysql2Adapter.emulate_booleans = false - class_attribute :emulate_booleans - self.emulate_booleans = true + class_attribute :emulate_booleans, default: true NATIVE_DATABASE_TYPES = { primary_key: "bigint auto_increment PRIMARY KEY", string: { name: "varchar", limit: 255 }, text: { name: "text", limit: 65535 }, integer: { name: "int", limit: 4 }, - float: { name: "float" }, + float: { name: "float", limit: 24 }, decimal: { name: "decimal" }, datetime: { name: "datetime" }, timestamp: { name: "timestamp" }, @@ -54,12 +42,9 @@ module ActiveRecord json: { name: "json" }, } - INDEX_TYPES = [:fulltext, :spatial] - INDEX_USINGS = [:btree, :hash] - - class StatementPool < ConnectionAdapters::StatementPool + class StatementPool < ConnectionAdapters::StatementPool # :nodoc: private def dealloc(stmt) - stmt[:stmt].close + stmt.close end end @@ -69,20 +54,12 @@ module ActiveRecord @statements = StatementPool.new(self.class.type_cast_config_to_integer(config[:statement_limit])) if version < "5.1.10" - raise "Your version of MySQL (#{full_version.match(/^\d+\.\d+\.\d+/)[0]}) is too old. Active Record supports MySQL >= 5.1.10." + raise "Your version of MySQL (#{version_string}) is too old. Active Record supports MySQL >= 5.1.10." end end - CHARSETS_OF_4BYTES_MAXLEN = ["utf8mb4", "utf16", "utf16le", "utf32"] - - def internal_string_options_for_primary_key # :nodoc: - super.tap { |options| - options[:collation] = collation.sub(/\A[^_]+/, "utf8") if CHARSETS_OF_4BYTES_MAXLEN.include?(charset) - } - end - def version #:nodoc: - @version ||= Version.new(full_version.match(/^\d+\.\d+\.\d+/)[0]) + @version ||= Version.new(version_string) end def mariadb? # :nodoc: @@ -93,16 +70,8 @@ module ActiveRecord true end - # Returns true, since this connection adapter supports prepared statement - # caching. - def supports_statement_cache? - true - end - - # Technically MySQL allows to create indexes with the sort order syntax - # but at the moment (5.5) it doesn't yet implement them def supports_index_sort_order? - true + !mariadb? && version >= "8.0.1" end def supports_transaction_isolation? @@ -146,11 +115,11 @@ module ActiveRecord end def get_advisory_lock(lock_name, timeout = 0) # :nodoc: - select_value("SELECT GET_LOCK(#{quote(lock_name)}, #{timeout})") == 1 + query_value("SELECT GET_LOCK(#{quote(lock_name.to_s)}, #{timeout})") == 1 end def release_advisory_lock(lock_name) # :nodoc: - select_value("SELECT RELEASE_LOCK(#{quote(lock_name)})") == 1 + query_value("SELECT RELEASE_LOCK(#{quote(lock_name.to_s)})") == 1 end def native_database_types @@ -158,7 +127,7 @@ module ActiveRecord end def index_algorithms - { default: "ALGORITHM = DEFAULT", copy: "ALGORITHM = COPY", inplace: "ALGORITHM = INPLACE" } + { default: "ALGORITHM = DEFAULT".dup, copy: "ALGORITHM = COPY".dup, inplace: "ALGORITHM = INPLACE".dup } end # HELPER METHODS =========================================== @@ -169,10 +138,6 @@ module ActiveRecord raise NotImplementedError end - def new_column(*args) #:nodoc: - MySQL::Column.new(*args) - end - # Must return the MySQL error number from the exception, if the exception has an # error number. def error_number(exception) # :nodoc: @@ -182,7 +147,7 @@ module ActiveRecord # REFERENTIAL INTEGRITY ==================================== def disable_referential_integrity #:nodoc: - old = select_value("SELECT @@FOREIGN_KEY_CHECKS") + old = query_value("SELECT @@FOREIGN_KEY_CHECKS") begin update("SET FOREIGN_KEY_CHECKS = 0") @@ -215,6 +180,8 @@ module ActiveRecord # Executes the SQL statement in the context of this connection. def execute(sql, name = nil) + materialize_transactions + log(sql, name) do ActiveSupport::Dependencies.interlock.permit_concurrent_loads do @connection.query(sql) @@ -258,7 +225,7 @@ module ActiveRecord end end - def empty_insert_statement_value + def empty_insert_statement_value(primary_key = nil) "VALUES ()" end @@ -274,7 +241,7 @@ module ActiveRecord end # Create a new MySQL database with optional <tt>:charset</tt> and <tt>:collation</tt>. - # Charset defaults to utf8. + # Charset defaults to utf8mb4. # # Example: # create_database 'charset_test', charset: 'latin1', collation: 'latin1_bin' @@ -282,9 +249,9 @@ module ActiveRecord # create_database 'matt_development', charset: :big5 def create_database(name, options = {}) if options[:collation] - execute "CREATE DATABASE #{quote_table_name(name)} DEFAULT CHARACTER SET #{quote_table_name(options[:charset] || 'utf8')} COLLATE #{quote_table_name(options[:collation])}" + execute "CREATE DATABASE #{quote_table_name(name)} DEFAULT COLLATE #{quote_table_name(options[:collation])}" else - execute "CREATE DATABASE #{quote_table_name(name)} DEFAULT CHARACTER SET #{quote_table_name(options[:charset] || 'utf8')}" + execute "CREATE DATABASE #{quote_table_name(name)} DEFAULT CHARACTER SET #{quote_table_name(options[:charset] || 'utf8mb4')}" end end @@ -297,7 +264,7 @@ module ActiveRecord end def current_database - select_value "SELECT DATABASE() as db" + query_value("SELECT database()", "SCHEMA") end # Returns the database character set. @@ -310,120 +277,25 @@ module ActiveRecord show_variable "collation_database" end - def tables # :nodoc: - sql = "SELECT table_name FROM information_schema.tables WHERE table_type = 'BASE TABLE'" - sql << " AND table_schema = #{quote(@config[:database])}" - - select_values(sql, "SCHEMA") - end - - def views # :nodoc: - select_values("SHOW FULL TABLES WHERE table_type = 'VIEW'", "SCHEMA") - end - - def data_sources # :nodoc: - sql = "SELECT table_name FROM information_schema.tables " - sql << "WHERE table_schema = #{quote(@config[:database])}" - - select_values(sql, "SCHEMA") - end - - def table_exists?(table_name) # :nodoc: - return false unless table_name.present? - - schema, name = extract_schema_qualified_name(table_name) - - sql = "SELECT table_name FROM information_schema.tables WHERE table_type = 'BASE TABLE'" - sql << " AND table_schema = #{quote(schema)} AND table_name = #{quote(name)}" - - select_values(sql, "SCHEMA").any? - end - - def data_source_exists?(table_name) # :nodoc: - return false unless table_name.present? - - schema, name = extract_schema_qualified_name(table_name) - - sql = "SELECT table_name FROM information_schema.tables " - sql << "WHERE table_schema = #{quote(schema)} AND table_name = #{quote(name)}" - - select_values(sql, "SCHEMA").any? - end - - def view_exists?(view_name) # :nodoc: - return false unless view_name.present? - - schema, name = extract_schema_qualified_name(view_name) - - sql = "SELECT table_name FROM information_schema.tables WHERE table_type = 'VIEW'" - sql << " AND table_schema = #{quote(schema)} AND table_name = #{quote(name)}" - - select_values(sql, "SCHEMA").any? - end - def truncate(table_name, name = nil) execute "TRUNCATE TABLE #{quote_table_name(table_name)}", name end - # Returns an array of indexes for the given table. - def indexes(table_name, name = nil) #:nodoc: - if name - ActiveSupport::Deprecation.warn(<<-MSG.squish) - Passing name to #indexes is deprecated without replacement. - MSG - end - - indexes = [] - current_index = nil - execute_and_free("SHOW KEYS FROM #{quote_table_name(table_name)}", "SCHEMA") do |result| - each_hash(result) do |row| - if current_index != row[:Key_name] - next if row[:Key_name] == "PRIMARY" # skip the primary key - current_index = row[:Key_name] - - mysql_index_type = row[:Index_type].downcase.to_sym - index_type = INDEX_TYPES.include?(mysql_index_type) ? mysql_index_type : nil - index_using = INDEX_USINGS.include?(mysql_index_type) ? mysql_index_type : nil - indexes << IndexDefinition.new(row[:Table], row[:Key_name], row[:Non_unique].to_i == 0, [], {}, nil, nil, index_type, index_using, row[:Index_comment].presence) - end - - indexes.last.columns << row[:Column_name] - indexes.last.lengths.merge!(row[:Column_name] => row[:Sub_part].to_i) if row[:Sub_part] - end - end - - indexes - end - - def new_column_from_field(table_name, field) # :nodoc: - type_metadata = fetch_type_metadata(field[:Type], field[:Extra]) - if type_metadata.type == :datetime && field[:Default] == "CURRENT_TIMESTAMP" - default, default_function = nil, field[:Default] - else - default, default_function = field[:Default], nil - end - new_column(field[:Field], default, type_metadata, field[:Null] == "YES", table_name, default_function, field[:Collation], comment: field[:Comment].presence) - end - def table_comment(table_name) # :nodoc: - schema, name = extract_schema_qualified_name(table_name) + scope = quoted_scope(table_name) - select_value(<<-SQL.strip_heredoc, "SCHEMA") + query_value(<<~SQL, "SCHEMA").presence SELECT table_comment FROM information_schema.tables - WHERE table_schema = #{quote(schema)} - AND table_name = #{quote(name)} + WHERE table_schema = #{scope[:schema]} + AND table_name = #{scope[:name]} SQL end - def create_table(table_name, **options) #:nodoc: - super(table_name, options: "ENGINE=InnoDB", **options) - end - def bulk_change_table(table_name, operations) #:nodoc: sqls = operations.flat_map do |command, args| table, arguments = args.shift, args - method = :"#{command}_sql" + method = :"#{command}_for_alter" if respond_to?(method, true) send(method, table, *arguments) @@ -435,6 +307,11 @@ module ActiveRecord execute("ALTER TABLE #{quote_table_name(table_name)} #{sqls}") end + def change_table_comment(table_name, comment) #:nodoc: + comment = "" if comment.nil? + execute("ALTER TABLE #{quote_table_name(table_name)} COMMENT #{quote(comment)}") + end + # Renames a table. # # Example: @@ -475,32 +352,33 @@ module ActiveRecord def change_column_default(table_name, column_name, default_or_changes) #:nodoc: default = extract_new_default_value(default_or_changes) - column = column_for(table_name, column_name) - change_column table_name, column_name, column.sql_type, default: default + change_column table_name, column_name, nil, default: default end def change_column_null(table_name, column_name, null, default = nil) #:nodoc: - column = column_for(table_name, column_name) - unless null || default.nil? execute("UPDATE #{quote_table_name(table_name)} SET #{quote_column_name(column_name)}=#{quote(default)} WHERE #{quote_column_name(column_name)} IS NULL") end - change_column table_name, column_name, column.sql_type, null: null + change_column table_name, column_name, nil, null: null + end + + def change_column_comment(table_name, column_name, comment) #:nodoc: + change_column table_name, column_name, nil, comment: comment end def change_column(table_name, column_name, type, options = {}) #:nodoc: - execute("ALTER TABLE #{quote_table_name(table_name)} #{change_column_sql(table_name, column_name, type, options)}") + execute("ALTER TABLE #{quote_table_name(table_name)} #{change_column_for_alter(table_name, column_name, type, options)}") end def rename_column(table_name, column_name, new_column_name) #:nodoc: - execute("ALTER TABLE #{quote_table_name(table_name)} #{rename_column_sql(table_name, column_name, new_column_name)}") + execute("ALTER TABLE #{quote_table_name(table_name)} #{rename_column_for_alter(table_name, column_name, new_column_name)}") rename_column_indexes(table_name, column_name, new_column_name) end def add_index(table_name, column_name, options = {}) #:nodoc: index_name, index_type, index_columns, _, index_algorithm, index_using, comment = add_index_options(table_name, column_name, options) - sql = "CREATE #{index_type} INDEX #{quote_column_name(index_name)} #{index_using} ON #{quote_table_name(table_name)} (#{index_columns}) #{index_algorithm}" + sql = "CREATE #{index_type} INDEX #{quote_column_name(index_name)} #{index_using} ON #{quote_table_name(table_name)} (#{index_columns}) #{index_algorithm}".dup execute add_sql_comment!(sql, comment) end @@ -512,22 +390,23 @@ module ActiveRecord def foreign_keys(table_name) raise ArgumentError unless table_name.present? - schema, name = extract_schema_qualified_name(table_name) + scope = quoted_scope(table_name) - fk_info = select_all(<<-SQL.strip_heredoc, "SCHEMA") + fk_info = exec_query(<<~SQL, "SCHEMA") SELECT fk.referenced_table_name AS 'to_table', fk.referenced_column_name AS 'primary_key', fk.column_name AS 'column', fk.constraint_name AS 'name', rc.update_rule AS 'on_update', rc.delete_rule AS 'on_delete' - FROM information_schema.key_column_usage fk - JOIN information_schema.referential_constraints rc + FROM information_schema.referential_constraints rc + JOIN information_schema.key_column_usage fk USING (constraint_schema, constraint_name) WHERE fk.referenced_column_name IS NOT NULL - AND fk.table_schema = #{quote(schema)} - AND fk.table_name = #{quote(name)} - AND rc.table_name = #{quote(name)} + AND fk.table_schema = #{scope[:schema]} + AND fk.table_name = #{scope[:name]} + AND rc.constraint_schema = #{scope[:schema]} + AND rc.table_name = #{scope[:name]} SQL fk_info.map do |row| @@ -585,13 +464,13 @@ module ActiveRecord super end - sql << " unsigned" if unsigned && type != :primary_key + sql = "#{sql} unsigned" if unsigned && type != :primary_key sql end # SHOW VARIABLES LIKE 'name' def show_variable(name) - select_value("SELECT @@#{name}", "SCHEMA") + query_value("SELECT @@#{name}", "SCHEMA") rescue ActiveRecord::StatementInvalid nil end @@ -599,14 +478,14 @@ module ActiveRecord def primary_keys(table_name) # :nodoc: raise ArgumentError unless table_name.present? - schema, name = extract_schema_qualified_name(table_name) + scope = quoted_scope(table_name) - select_values(<<-SQL.strip_heredoc, "SCHEMA") + query_values(<<~SQL, "SCHEMA") SELECT column_name FROM information_schema.key_column_usage WHERE constraint_name = 'PRIMARY' - AND table_schema = #{quote(schema)} - AND table_name = #{quote(name)} + AND table_schema = #{scope[:schema]} + AND table_name = #{scope[:name]} ORDER BY ordinal_position SQL end @@ -636,7 +515,7 @@ module ActiveRecord s.gsub(/\s+(?:ASC|DESC)\b/i, "") }.reject(&:blank?).map.with_index { |column, i| "#{column} AS alias_#{i}" } - [super, *order_columns].join(", ") + (order_columns << super).join(", ") end def strict_mode? @@ -647,9 +526,41 @@ module ActiveRecord index.using == :btree || super end + def insert_fixtures_set(fixture_set, tables_to_delete = []) + with_multi_statements do + super { discard_remaining_results } + end + end + private + def combine_multi_statements(total_sql) + total_sql.each_with_object([]) do |sql, total_sql_chunks| + previous_packet = total_sql_chunks.last + sql << ";\n" + if max_allowed_packet_reached?(sql, previous_packet) || total_sql_chunks.empty? + total_sql_chunks << sql + else + previous_packet << sql + end + end + end + + def max_allowed_packet_reached?(current_packet, previous_packet) + if current_packet.bytesize > max_allowed_packet + raise ActiveRecordError, "Fixtures set is too large #{current_packet.bytesize}. Consider increasing the max_allowed_packet variable." + elsif previous_packet.nil? + false + else + (current_packet.bytesize + previous_packet.bytesize) > max_allowed_packet + end + end - def initialize_type_map(m) + def max_allowed_packet + bytes_margin = 2 + @max_allowed_packet ||= (show_variable("max_allowed_packet") - bytes_margin) + end + + def initialize_type_map(m = type_map) super register_class_with_limit m, %r(char)i, MysqlString @@ -664,7 +575,6 @@ module ActiveRecord m.register_type %r(longblob)i, Type::Binary.new(limit: 2**32 - 1) m.register_type %r(^float)i, Type::Float.new(limit: 24) m.register_type %r(^double)i, Type::Float.new(limit: 53) - m.register_type %r(^json)i, MysqlJson.new register_integer_type m, %r(^bigint)i, limit: 8 register_integer_type m, %r(^int)i, limit: 4 @@ -707,45 +617,26 @@ module ActiveRecord end end - def fetch_type_metadata(sql_type, extra = "") - MySQL::TypeMetadata.new(super(sql_type), extra: extra) - end - - def add_index_length(quoted_columns, **options) - if length = options[:length] - case length - when Hash - length = length.symbolize_keys - quoted_columns.each { |name, column| column << "(#{length[name]})" if length[name].present? } - when Integer - quoted_columns.each { |name, column| column << "(#{length})" } - end - end - - quoted_columns - end - - def add_options_for_index_columns(quoted_columns, **options) - quoted_columns = add_index_length(quoted_columns, options) - super - end - # See https://dev.mysql.com/doc/refman/5.7/en/error-messages-server.html ER_DUP_ENTRY = 1062 ER_NOT_NULL_VIOLATION = 1048 ER_DO_NOT_HAVE_DEFAULT = 1364 + ER_ROW_IS_REFERENCED_2 = 1451 ER_NO_REFERENCED_ROW_2 = 1452 ER_DATA_TOO_LONG = 1406 ER_OUT_OF_RANGE = 1264 ER_LOCK_DEADLOCK = 1213 ER_CANNOT_ADD_FOREIGN = 1215 ER_CANNOT_CREATE_TABLE = 1005 + ER_LOCK_WAIT_TIMEOUT = 1205 + ER_QUERY_INTERRUPTED = 1317 + ER_QUERY_TIMEOUT = 3024 def translate_exception(exception, message) case error_number(exception) when ER_DUP_ENTRY RecordNotUnique.new(message) - when ER_NO_REFERENCED_ROW_2 + when ER_ROW_IS_REFERENCED_2, ER_NO_REFERENCED_ROW_2 InvalidForeignKey.new(message) when ER_CANNOT_ADD_FOREIGN mismatched_foreign_key(message) @@ -763,19 +654,20 @@ module ActiveRecord NotNullViolation.new(message) when ER_LOCK_DEADLOCK Deadlocked.new(message) + when ER_LOCK_WAIT_TIMEOUT + LockWaitTimeout.new(message) + when ER_QUERY_TIMEOUT + StatementTimeout.new(message) + when ER_QUERY_INTERRUPTED + QueryCanceled.new(message) else super end end - def add_column_sql(table_name, column_name, type, options = {}) - td = create_table_definition(table_name) - cd = td.new_column_definition(column_name, type, options) - schema_creation.accept(AddColumnDefinition.new(cd)) - end - - def change_column_sql(table_name, column_name, type, options = {}) + def change_column_for_alter(table_name, column_name, type, options = {}) column = column_for(table_name, column_name) + type ||= column.sql_type unless options.key?(:default) options[:default] = column.default @@ -794,7 +686,7 @@ module ActiveRecord schema_creation.accept(ChangeColumnDefinition.new(cd, column.name)) end - def rename_column_sql(table_name, column_name, new_column_name) + def rename_column_for_alter(table_name, column_name, new_column_name) column = column_for(table_name, column_name) options = { default: column.default, @@ -802,52 +694,43 @@ module ActiveRecord auto_increment: column.auto_increment? } - current_type = select_one("SHOW COLUMNS FROM #{quote_table_name(table_name)} LIKE '#{column_name}'", "SCHEMA")["Type"] + current_type = exec_query("SHOW COLUMNS FROM #{quote_table_name(table_name)} LIKE #{quote(column_name)}", "SCHEMA").first["Type"] td = create_table_definition(table_name) cd = td.new_column_definition(new_column_name, current_type, options) schema_creation.accept(ChangeColumnDefinition.new(cd, column.name)) end - def remove_column_sql(table_name, column_name, type = nil, options = {}) - "DROP #{quote_column_name(column_name)}" - end - - def remove_columns_sql(table_name, *column_names) - column_names.map { |column_name| remove_column_sql(table_name, column_name) } - end - - def add_index_sql(table_name, column_name, options = {}) + def add_index_for_alter(table_name, column_name, options = {}) index_name, index_type, index_columns, _, index_algorithm, index_using = add_index_options(table_name, column_name, options) index_algorithm[0, 0] = ", " if index_algorithm.present? "ADD #{index_type} INDEX #{quote_column_name(index_name)} #{index_using} (#{index_columns})#{index_algorithm}" end - def remove_index_sql(table_name, options = {}) + def remove_index_for_alter(table_name, options = {}) index_name = index_name_for_remove(table_name, options) - "DROP INDEX #{index_name}" + "DROP INDEX #{quote_column_name(index_name)}" end - def add_timestamps_sql(table_name, options = {}) - [add_column_sql(table_name, :created_at, :datetime, options), add_column_sql(table_name, :updated_at, :datetime, options)] + def add_timestamps_for_alter(table_name, options = {}) + [add_column_for_alter(table_name, :created_at, :datetime, options), add_column_for_alter(table_name, :updated_at, :datetime, options)] end - def remove_timestamps_sql(table_name, options = {}) - [remove_column_sql(table_name, :updated_at), remove_column_sql(table_name, :created_at)] + def remove_timestamps_for_alter(table_name, options = {}) + [remove_column_for_alter(table_name, :updated_at), remove_column_for_alter(table_name, :created_at)] end # MySQL is too stupid to create a temporary table for use subquery, so we have # to give it some prompting in the form of a subsubquery. Ugh! def subquery_for(key, select) - subsubselect = select.clone - subsubselect.projections = [key] + subselect = select.clone + subselect.projections = [key] # Materialize subquery by adding distinct # to work with MySQL 5.7.6 which sets optimizer_switch='derived_merge=on' - subsubselect.distinct unless select.limit || select.offset || select.orders.any? + subselect.distinct unless select.limit || select.offset || select.orders.any? - subselect = Arel::SelectManager.new(select.engine) - subselect.project Arel.sql(key.name) - subselect.from subsubselect.as("__active_record_temp") + key_name = quote_column_name(key.name) + Arel::SelectManager.new(subselect.as("__active_record_temp")).project(Arel.sql(key_name)) end def supports_rename_index? @@ -868,7 +751,7 @@ module ActiveRecord defaults = [":default", :default].to_set # Make MySQL reject illegal values rather than truncating or blanking them, see - # http://dev.mysql.com/doc/refman/5.7/en/sql-mode.html#sqlmode_strict_all_tables + # https://dev.mysql.com/doc/refman/5.7/en/sql-mode.html#sqlmode_strict_all_tables # If the user has provided another value for sql_mode, don't replace it. if sql_mode = variables.delete("sql_mode") sql_mode = quote(sql_mode) @@ -885,10 +768,10 @@ module ActiveRecord sql_mode_assignment = "@@SESSION.sql_mode = #{sql_mode}, " if sql_mode # NAMES does not have an equals sign, see - # http://dev.mysql.com/doc/refman/5.7/en/set-statement.html#id944430 + # https://dev.mysql.com/doc/refman/5.7/en/set-names.html # (trailing comma because variable_assignments will always have content) if @config[:encoding] - encoding = "NAMES #{@config[:encoding]}" + encoding = "NAMES #{@config[:encoding]}".dup encoding << " COLLATE #{@config[:collation]}" if @config[:collation] encoding << ", " end @@ -913,19 +796,12 @@ module ActiveRecord end end - def extract_foreign_key_action(specifier) # :nodoc: - case specifier - when "CASCADE"; :cascade - when "SET NULL"; :nullify - end - end - def create_table_info(table_name) # :nodoc: - select_one("SHOW CREATE TABLE #{quote_table_name(table_name)}")["Create Table"] + exec_query("SHOW CREATE TABLE #{quote_table_name(table_name)}", "SCHEMA").first["Create Table"] end - def create_table_definition(*args) # :nodoc: - MySQL::TableDefinition.new(*args) + def arel_visitor + Arel::Visitors::MySQL.new(self) end def mismatched_foreign_key(message) @@ -940,12 +816,6 @@ module ActiveRecord ) end - def extract_schema_qualified_name(string) # :nodoc: - schema, name = string.to_s.scan(/[^`.\s]+|`[^`]*`/) - schema, name = @config[:database], schema unless name - [schema, name] - end - def integer_to_sql(limit) # :nodoc: case limit when 1; "tinyint" @@ -977,19 +847,15 @@ module ActiveRecord end end - class MysqlJson < Type::Internal::AbstractJson # :nodoc: - def changed_in_place?(raw_old_value, new_value) - # Normalization is required because MySQL JSON data format includes - # the space between the elements. - super(serialize(deserialize(raw_old_value)), new_value) - end + def version_string + full_version.match(/^(?:5\.5\.5-)?(\d+\.\d+\.\d+)/)[1] end class MysqlString < Type::String # :nodoc: def serialize(value) case value - when true then MySQL::Quoting::QUOTED_TRUE - when false then MySQL::Quoting::QUOTED_FALSE + when true then "1" + when false then "0" else super end end @@ -998,14 +864,13 @@ module ActiveRecord def cast_value(value) case value - when true then MySQL::Quoting::QUOTED_TRUE - when false then MySQL::Quoting::QUOTED_FALSE + when true then "1" + when false then "0" else super end end end - ActiveRecord::Type.register(:json, MysqlJson, adapter: :mysql2) ActiveRecord::Type.register(:string, MysqlString, adapter: :mysql2) ActiveRecord::Type.register(:unsigned_integer, Type::UnsignedInteger, adapter: :mysql2) end diff --git a/activerecord/lib/active_record/connection_adapters/column.rb b/activerecord/lib/active_record/connection_adapters/column.rb index 61cd7ae4cc..5d81de9fe1 100644 --- a/activerecord/lib/active_record/connection_adapters/column.rb +++ b/activerecord/lib/active_record/connection_adapters/column.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module ActiveRecord # :stopdoc: module ConnectionAdapters @@ -9,11 +11,11 @@ module ActiveRecord # Instantiates a new column in the table. # - # +name+ is the column's name, such as <tt>supplier_id</tt> in <tt>supplier_id int</tt>. + # +name+ is the column's name, such as <tt>supplier_id</tt> in <tt>supplier_id bigint</tt>. # +default+ is the type-casted default value, such as +new+ in <tt>sales_stage varchar(20) default 'new'</tt>. # +sql_type_metadata+ is various information about the type of the column # +null+ determines if this column allows +NULL+ values. - def initialize(name, default, sql_type_metadata = nil, null = true, table_name = nil, default_function = nil, collation = nil, comment: nil) + def initialize(name, default, sql_type_metadata = nil, null = true, table_name = nil, default_function = nil, collation = nil, comment: nil, **) @name = name.freeze @table_name = table_name @sql_type_metadata = sql_type_metadata diff --git a/activerecord/lib/active_record/connection_adapters/connection_specification.rb b/activerecord/lib/active_record/connection_adapters/connection_specification.rb index 3e4ea28f63..2e7a78215a 100644 --- a/activerecord/lib/active_record/connection_adapters/connection_specification.rb +++ b/activerecord/lib/active_record/connection_adapters/connection_specification.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require "uri" module ActiveRecord @@ -55,9 +57,7 @@ module ActiveRecord private - def uri - @uri - end + attr_reader :uri def uri_parser @uri_parser ||= URI::Parser.new @@ -114,8 +114,7 @@ module ActiveRecord class Resolver # :nodoc: attr_reader :configurations - # Accepts a hash two layers deep, keys on the first layer represent - # environments such as "production". Keys must be strings. + # Accepts a list of db config objects. def initialize(configurations) @configurations = configurations end @@ -136,34 +135,14 @@ module ActiveRecord # Resolver.new(configurations).resolve(:production) # # => { "host" => "localhost", "database" => "foo", "adapter" => "postgresql" } # - def resolve(config) - if config - resolve_connection config - elsif env = ActiveRecord::ConnectionHandling::RAILS_ENV.call - resolve_symbol_connection env.to_sym + def resolve(config_or_env, pool_name = nil) + if config_or_env + resolve_connection config_or_env, pool_name else raise AdapterNotSpecified end end - # Expands each key in @configurations hash into fully resolved hash - def resolve_all - config = configurations.dup - - if env = ActiveRecord::ConnectionHandling::DEFAULT_ENV.call - env_config = config[env] if config[env].is_a?(Hash) && !(config[env].key?("adapter") || config[env].key?("url")) - end - - config.reject! { |k, v| v.is_a?(Hash) && !(v.key?("adapter") || v.key?("url")) } - config.merge! env_config if env_config - - config.each do |key, value| - config[key] = resolve(value) if value - end - - config - end - # Returns an instance of ConnectionSpecification for a given adapter. # Accepts a hash one layer deep that contains all connection information. # @@ -177,17 +156,31 @@ module ActiveRecord # # => { "host" => "localhost", "database" => "foo", "adapter" => "sqlite3" } # def spec(config) - spec = resolve(config).symbolize_keys + pool_name = config if config.is_a?(Symbol) + + spec = resolve(config, pool_name).symbolize_keys raise(AdapterNotSpecified, "database configuration does not specify adapter") unless spec.key?(:adapter) + # Require the adapter itself and give useful feedback about + # 1. Missing adapter gems and + # 2. Adapter gems' missing dependencies. path_to_adapter = "active_record/connection_adapters/#{spec[:adapter]}_adapter" begin require path_to_adapter - rescue Gem::LoadError => e - raise Gem::LoadError, "Specified '#{spec[:adapter]}' for database adapter, but the gem is not loaded. Add `gem '#{e.name}'` to your Gemfile (and ensure its version is at the minimum required by ActiveRecord)." rescue LoadError => e - raise LoadError, "Could not load '#{path_to_adapter}'. Make sure that the adapter in config/database.yml is valid. If you use an adapter other than 'mysql2', 'postgresql' or 'sqlite3' add the necessary adapter gem to the Gemfile.", e.backtrace + # We couldn't require the adapter itself. Raise an exception that + # points out config typos and missing gems. + if e.path == path_to_adapter + # We can assume that a non-builtin adapter was specified, so it's + # either misspelled or missing from Gemfile. + raise e.class, "Could not load the '#{spec[:adapter]}' Active Record adapter. Ensure that the adapter is spelled correctly in config/database.yml and that you've added the necessary adapter gem to your Gemfile.", e.backtrace + + # Bubbled up from the adapter require. Prefix the exception message + # with some guidance about how to address it and reraise. + else + raise e.class, "Error loading the '#{spec[:adapter]}' Active Record adapter. Missing a gem it depends on? #{e.message}", e.backtrace + end end adapter_method = "#{spec[:adapter]}_connection" @@ -200,7 +193,6 @@ module ActiveRecord end private - # Returns fully resolved connection, accepts hash, string or symbol. # Always returns a hash. # @@ -221,29 +213,42 @@ module ActiveRecord # Resolver.new({}).resolve_connection("postgresql://localhost/foo") # # => { "host" => "localhost", "database" => "foo", "adapter" => "postgresql" } # - def resolve_connection(spec) - case spec + def resolve_connection(config_or_env, pool_name = nil) + case config_or_env when Symbol - resolve_symbol_connection spec + resolve_symbol_connection config_or_env, pool_name when String - resolve_url_connection spec + resolve_url_connection config_or_env when Hash - resolve_hash_connection spec + resolve_hash_connection config_or_env + else + resolve_connection config_or_env end end - # Takes the environment such as +:production+ or +:development+. + # Takes the environment such as +:production+ or +:development+ and a + # pool name the corresponds to the name given by the connection pool + # to the connection. That pool name is merged into the hash with the + # name key. + # # This requires that the @configurations was initialized with a key that # matches. # - # Resolver.new("production" => {}).resolve_symbol_connection(:production) - # # => {} + # configurations = #<ActiveRecord::DatabaseConfigurations:0x00007fd9fdace3e0 + # @configurations=[ + # #<ActiveRecord::DatabaseConfigurations::HashConfig:0x00007fd9fdace250 + # @env_name="production", @spec_name="primary", @config={"database"=>"my_db"}> + # ]> # - def resolve_symbol_connection(spec) - if config = configurations[spec.to_s] - resolve_connection(config).merge("name" => spec.to_s) + # Resolver.new(configurations).resolve_symbol_connection(:production, "primary") + # # => { "database" => "my_db" } + def resolve_symbol_connection(env_name, pool_name) + db_config = configurations.find_db_config(env_name) + + if db_config + resolve_connection(db_config.config).merge("name" => pool_name.to_s) else - raise(AdapterNotSpecified, "'#{spec}' database is not configured. Available: #{configurations.keys.inspect}") + raise(AdapterNotSpecified, "'#{env_name}' database is not configured. Available: #{configurations.configurations.map(&:env_name).join(", ")}") end end diff --git a/activerecord/lib/active_record/connection_adapters/determine_if_preparable_visitor.rb b/activerecord/lib/active_record/connection_adapters/determine_if_preparable_visitor.rb index 0fdc185c45..3dcb916d99 100644 --- a/activerecord/lib/active_record/connection_adapters/determine_if_preparable_visitor.rb +++ b/activerecord/lib/active_record/connection_adapters/determine_if_preparable_visitor.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module ActiveRecord module ConnectionAdapters module DetermineIfPreparableVisitor diff --git a/activerecord/lib/active_record/connection_adapters/mysql/column.rb b/activerecord/lib/active_record/connection_adapters/mysql/column.rb index c9ad47c035..fa1541019d 100644 --- a/activerecord/lib/active_record/connection_adapters/mysql/column.rb +++ b/activerecord/lib/active_record/connection_adapters/mysql/column.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module ActiveRecord module ConnectionAdapters module MySQL diff --git a/activerecord/lib/active_record/connection_adapters/mysql/database_statements.rb b/activerecord/lib/active_record/connection_adapters/mysql/database_statements.rb index 8c67a7a80b..684c7042a7 100644 --- a/activerecord/lib/active_record/connection_adapters/mysql/database_statements.rb +++ b/activerecord/lib/active_record/connection_adapters/mysql/database_statements.rb @@ -1,25 +1,22 @@ +# frozen_string_literal: true + module ActiveRecord module ConnectionAdapters module MySQL module DatabaseStatements # Returns an ActiveRecord::Result instance. - def select_all(arel, name = nil, binds = [], preparable: nil) # :nodoc: + def select_all(*) # :nodoc: result = if ExplainRegistry.collect? && prepared_statements unprepared_statement { super } else super end - @connection.next_result while @connection.more_results? + discard_remaining_results result end - # Returns an array of arrays containing the field values. - # Order is the same as that returned by +columns+. - def select_rows(arel, name = nil, binds = []) # :nodoc: - select_result(arel, name, binds) do |result| - @connection.next_result while @connection.more_results? - result.to_a - end + def query(sql, name = nil) # :nodoc: + execute(sql, name).to_a end # Executes the SQL statement in the context of this connection. @@ -32,6 +29,8 @@ module ActiveRecord end def exec_query(sql, name = "SQL", binds = [], prepare: false) + materialize_transactions + if without_prepared_statement?(binds) execute_and_free(sql, name) do |result| ActiveRecord::Result.new(result.fields, result.to_a) if result @@ -44,6 +43,8 @@ module ActiveRecord end def exec_delete(sql, name = nil, binds = []) + materialize_transactions + if without_prepared_statement?(binds) execute_and_free(sql, name) { @connection.affected_rows } else @@ -53,18 +54,51 @@ module ActiveRecord alias :exec_update :exec_delete private + def default_insert_value(column) + Arel.sql("DEFAULT") unless column.auto_increment? + end def last_inserted_id(result) @connection.last_id end - def select_result(arel, name, binds) - arel, binds = binds_from_relation(arel, binds) - sql = to_sql(arel, binds) - if without_prepared_statement?(binds) - execute_and_free(sql, name) { |result| yield result } + def discard_remaining_results + @connection.abandon_results! + end + + def supports_set_server_option? + @connection.respond_to?(:set_server_option) + end + + def multi_statements_enabled?(flags) + if flags.is_a?(Array) + flags.include?("MULTI_STATEMENTS") else - exec_stmt_and_free(sql, name, binds, cache_stmt: true) { |_, result| yield result } + (flags & Mysql2::Client::MULTI_STATEMENTS) != 0 + end + end + + def with_multi_statements + previous_flags = @config[:flags] + + unless multi_statements_enabled?(previous_flags) + if supports_set_server_option? + @connection.set_server_option(Mysql2::Client::OPTION_MULTI_STATEMENTS_ON) + else + @config[:flags] = Mysql2::Client::MULTI_STATEMENTS + reconnect! + end + end + + yield + ensure + unless multi_statements_enabled?(previous_flags) + if supports_set_server_option? + @connection.set_server_option(Mysql2::Client::OPTION_MULTI_STATEMENTS_OFF) + else + @config[:flags] = previous_flags + reconnect! + end end end @@ -77,10 +111,7 @@ module ActiveRecord log(sql, name, binds, type_casted_binds) do if cache_stmt - cache = @statements[sql] ||= { - stmt: @connection.prepare(sql) - } - stmt = cache[:stmt] + stmt = @statements[sql] ||= @connection.prepare(sql) else stmt = @connection.prepare(sql) end diff --git a/activerecord/lib/active_record/connection_adapters/mysql/explain_pretty_printer.rb b/activerecord/lib/active_record/connection_adapters/mysql/explain_pretty_printer.rb index 9691060cd3..20c3c83664 100644 --- a/activerecord/lib/active_record/connection_adapters/mysql/explain_pretty_printer.rb +++ b/activerecord/lib/active_record/connection_adapters/mysql/explain_pretty_printer.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module ActiveRecord module ConnectionAdapters module MySQL diff --git a/activerecord/lib/active_record/connection_adapters/mysql/quoting.rb b/activerecord/lib/active_record/connection_adapters/mysql/quoting.rb index d4f5906b33..be038403b8 100644 --- a/activerecord/lib/active_record/connection_adapters/mysql/quoting.rb +++ b/activerecord/lib/active_record/connection_adapters/mysql/quoting.rb @@ -1,9 +1,9 @@ +# frozen_string_literal: true + module ActiveRecord module ConnectionAdapters module MySQL module Quoting # :nodoc: - QUOTED_TRUE, QUOTED_FALSE = "1".freeze, "0".freeze - def quote_column_name(name) @quoted_column_names[name] ||= "`#{super.gsub('`', '``')}`".freeze end @@ -12,18 +12,10 @@ module ActiveRecord @quoted_table_names[name] ||= super.gsub(".", "`.`").freeze end - def quoted_true - QUOTED_TRUE - end - def unquoted_true 1 end - def quoted_false - QUOTED_FALSE - end - def unquoted_false 0 end @@ -39,6 +31,13 @@ module ActiveRecord def quoted_binary(value) "x'#{value.hex}'" end + + def _type_cast(value) + case value + when Date, Time then value + else super + end + end end end end diff --git a/activerecord/lib/active_record/connection_adapters/mysql/schema_creation.rb b/activerecord/lib/active_record/connection_adapters/mysql/schema_creation.rb index 083cd6340f..c9ea653b77 100644 --- a/activerecord/lib/active_record/connection_adapters/mysql/schema_creation.rb +++ b/activerecord/lib/active_record/connection_adapters/mysql/schema_creation.rb @@ -1,9 +1,10 @@ +# frozen_string_literal: true + module ActiveRecord module ConnectionAdapters module MySQL class SchemaCreation < AbstractAdapter::SchemaCreation # :nodoc: - delegate :add_sql_comment!, :mariadb?, to: :@conn - private :add_sql_comment!, :mariadb? + delegate :add_sql_comment!, :mariadb?, to: :@conn, private: true private @@ -16,7 +17,7 @@ module ActiveRecord end def visit_ChangeColumnDefinition(o) - change_column_sql = "CHANGE #{quote_column_name(o.name)} #{accept(o.column)}" + change_column_sql = "CHANGE #{quote_column_name(o.name)} #{accept(o.column)}".dup add_column_position!(change_column_sql, column_options(o.column)) end @@ -28,7 +29,7 @@ module ActiveRecord # By default, TIMESTAMP columns are NOT NULL, cannot contain NULL values, # and assigning NULL assigns the current timestamp. To permit a TIMESTAMP # column to contain NULL, explicitly declare it with the NULL attribute. - # See http://dev.mysql.com/doc/refman/5.7/en/timestamp-initialization.html + # See https://dev.mysql.com/doc/refman/5.7/en/timestamp-initialization.html if /\Atimestamp\b/.match?(options[:column].sql_type) && !options[:primary_key] sql << " NULL" unless options[:null] == false || options_include_default?(options) end @@ -63,7 +64,7 @@ module ActiveRecord def index_in_create(table_name, column_name, options) index_name, index_type, index_columns, _, _, index_using, comment = @conn.add_index_options(table_name, column_name, options) - add_sql_comment!("#{index_type} INDEX #{quote_column_name(index_name)} #{index_using} (#{index_columns})", comment) + add_sql_comment!("#{index_type} INDEX #{quote_column_name(index_name)} #{index_using} (#{index_columns})".dup, comment) end end end diff --git a/activerecord/lib/active_record/connection_adapters/mysql/schema_definitions.rb b/activerecord/lib/active_record/connection_adapters/mysql/schema_definitions.rb index 6d88c14d50..2ed4ad16ae 100644 --- a/activerecord/lib/active_record/connection_adapters/mysql/schema_definitions.rb +++ b/activerecord/lib/active_record/connection_adapters/mysql/schema_definitions.rb @@ -1,12 +1,9 @@ +# frozen_string_literal: true + module ActiveRecord module ConnectionAdapters module MySQL module ColumnMethods - def primary_key(name, type = :primary_key, **options) - options[:auto_increment] = true if [:integer, :bigint].include?(type) && !options.key?(:default) - super - end - def blob(*args, **options) args.each { |name| column(name, :blob, options) } end @@ -35,10 +32,6 @@ module ActiveRecord args.each { |name| column(name, :longtext, options) } end - def json(*args, **options) - args.each { |name| column(name, :json, options) } - end - def unsigned_integer(*args, **options) args.each { |name| column(name, :unsigned_integer, options) } end @@ -66,7 +59,6 @@ module ActiveRecord when :primary_key type = :integer options[:limit] ||= 8 - options[:auto_increment] = true options[:primary_key] = true when /\Aunsigned_(?<type>.+)\z/ type = $~[:type].to_sym @@ -80,6 +72,11 @@ module ActiveRecord def aliased_types(name, fallback) fallback end + + def integer_like_primary_key_type(type, options) + options[:auto_increment] = true + type + end end class Table < ActiveRecord::ConnectionAdapters::Table diff --git a/activerecord/lib/active_record/connection_adapters/mysql/schema_dumper.rb b/activerecord/lib/active_record/connection_adapters/mysql/schema_dumper.rb index 3e0afd9761..d23178e43c 100644 --- a/activerecord/lib/active_record/connection_adapters/mysql/schema_dumper.rb +++ b/activerecord/lib/active_record/connection_adapters/mysql/schema_dumper.rb @@ -1,25 +1,29 @@ +# frozen_string_literal: true + module ActiveRecord module ConnectionAdapters module MySQL - module ColumnDumper # :nodoc: - def prepare_column_options(column) - spec = super - spec[:unsigned] = "true" if column.unsigned? - - if supports_virtual_columns? && column.virtual? - spec[:as] = extract_expression_for_virtual_column(column) - spec[:stored] = "true" if /\b(?:STORED|PERSISTENT)\b/.match?(column.extra) - spec = { type: schema_type(column).inspect }.merge!(spec) - end + class SchemaDumper < ConnectionAdapters::SchemaDumper # :nodoc: + private + def prepare_column_options(column) + spec = super + spec[:unsigned] = "true" if column.unsigned? + spec[:auto_increment] = "true" if column.auto_increment? - spec - end + if @connection.supports_virtual_columns? && column.virtual? + spec[:as] = extract_expression_for_virtual_column(column) + spec[:stored] = "true" if /\b(?:STORED|PERSISTENT)\b/.match?(column.extra) + spec = { type: schema_type(column).inspect }.merge!(spec) + end - def migration_keys - super + [:unsigned] - end + spec + end - private + def column_spec_for_primary_key(column) + spec = super + spec.delete(:auto_increment) if column.type == :integer && column.auto_increment? + spec + end def default_primary_key?(column) super && column.auto_increment? && !column.unsigned? @@ -47,23 +51,27 @@ module ActiveRecord def schema_collation(column) if column.collation && table_name = column.table_name @table_collation_cache ||= {} - @table_collation_cache[table_name] ||= select_one("SHOW TABLE STATUS LIKE '#{table_name}'")["Collation"] + @table_collation_cache[table_name] ||= + @connection.exec_query("SHOW TABLE STATUS LIKE #{@connection.quote(table_name)}", "SCHEMA").first["Collation"] column.collation.inspect if column.collation != @table_collation_cache[table_name] end end def extract_expression_for_virtual_column(column) - if mariadb? - create_table_info = create_table_info(column.table_name) - if %r/#{quote_column_name(column.name)} #{Regexp.quote(column.sql_type)} AS \((?<expression>.+?)\) #{column.extra}/m =~ create_table_info + if @connection.mariadb? && @connection.version < "10.2.5" + create_table_info = @connection.send(:create_table_info, column.table_name) + column_name = @connection.quote_column_name(column.name) + if %r/#{column_name} #{Regexp.quote(column.sql_type)}(?: COLLATE \w+)? AS \((?<expression>.+?)\) #{column.extra}/ =~ create_table_info $~[:expression].inspect end else + scope = @connection.send(:quoted_scope, column.table_name) + column_name = @connection.quote(column.name) sql = "SELECT generation_expression FROM information_schema.columns" \ - " WHERE table_schema = #{quote(@config[:database])}" \ - " AND table_name = #{quote(column.table_name)}" \ - " AND column_name = #{quote(column.name)}" - select_value(sql, "SCHEMA").inspect + " WHERE table_schema = #{scope[:schema]}" \ + " AND table_name = #{scope[:name]}" \ + " AND column_name = #{column_name}" + @connection.query_value(sql, "SCHEMA").inspect end end end diff --git a/activerecord/lib/active_record/connection_adapters/mysql/schema_statements.rb b/activerecord/lib/active_record/connection_adapters/mysql/schema_statements.rb new file mode 100644 index 0000000000..1cf210d85b --- /dev/null +++ b/activerecord/lib/active_record/connection_adapters/mysql/schema_statements.rb @@ -0,0 +1,148 @@ +# frozen_string_literal: true + +module ActiveRecord + module ConnectionAdapters + module MySQL + module SchemaStatements # :nodoc: + # Returns an array of indexes for the given table. + def indexes(table_name) + indexes = [] + current_index = nil + execute_and_free("SHOW KEYS FROM #{quote_table_name(table_name)}", "SCHEMA") do |result| + each_hash(result) do |row| + if current_index != row[:Key_name] + next if row[:Key_name] == "PRIMARY" # skip the primary key + current_index = row[:Key_name] + + mysql_index_type = row[:Index_type].downcase.to_sym + case mysql_index_type + when :fulltext, :spatial + index_type = mysql_index_type + when :btree, :hash + index_using = mysql_index_type + end + + indexes << [ + row[:Table], + row[:Key_name], + row[:Non_unique].to_i == 0, + [], + lengths: {}, + orders: {}, + type: index_type, + using: index_using, + comment: row[:Index_comment].presence + ] + end + + indexes.last[-2] << row[:Column_name] + indexes.last[-1][:lengths][row[:Column_name]] = row[:Sub_part].to_i if row[:Sub_part] + indexes.last[-1][:orders].merge!(row[:Column_name] => :desc) if row[:Collation] == "D" + end + end + + indexes.map { |index| IndexDefinition.new(*index) } + end + + def remove_column(table_name, column_name, type = nil, options = {}) + if foreign_key_exists?(table_name, column: column_name) + remove_foreign_key(table_name, column: column_name) + end + super + end + + def internal_string_options_for_primary_key + super.tap do |options| + if CHARSETS_OF_4BYTES_MAXLEN.include?(charset) && (mariadb? || version < "8.0.0") + options[:collation] = collation.sub(/\A[^_]+/, "utf8") + end + end + end + + def update_table_definition(table_name, base) + MySQL::Table.new(table_name, base) + end + + def create_schema_dumper(options) + MySQL::SchemaDumper.create(self, options) + end + + private + CHARSETS_OF_4BYTES_MAXLEN = ["utf8mb4", "utf16", "utf16le", "utf32"] + + def schema_creation + MySQL::SchemaCreation.new(self) + end + + def create_table_definition(*args) + MySQL::TableDefinition.new(*args) + end + + def new_column_from_field(table_name, field) + type_metadata = fetch_type_metadata(field[:Type], field[:Extra]) + if type_metadata.type == :datetime && /\ACURRENT_TIMESTAMP(?:\([0-6]?\))?\z/i.match?(field[:Default]) + default, default_function = nil, field[:Default] + else + default, default_function = field[:Default], nil + end + + MySQL::Column.new( + field[:Field], + default, + type_metadata, + field[:Null] == "YES", + table_name, + default_function, + field[:Collation], + comment: field[:Comment].presence + ) + end + + def fetch_type_metadata(sql_type, extra = "") + MySQL::TypeMetadata.new(super(sql_type), extra: extra) + end + + def extract_foreign_key_action(specifier) + super unless specifier == "RESTRICT" + end + + def add_index_length(quoted_columns, **options) + lengths = options_for_index_columns(options[:length]) + quoted_columns.each do |name, column| + column << "(#{lengths[name]})" if lengths[name].present? + end + end + + def add_options_for_index_columns(quoted_columns, **options) + quoted_columns = add_index_length(quoted_columns, options) + super + end + + def data_source_sql(name = nil, type: nil) + scope = quoted_scope(name, type: type) + + sql = "SELECT table_name FROM information_schema.tables".dup + sql << " WHERE table_schema = #{scope[:schema]}" + sql << " AND table_name = #{scope[:name]}" if scope[:name] + sql << " AND table_type = #{scope[:type]}" if scope[:type] + sql + end + + def quoted_scope(name = nil, type: nil) + schema, name = extract_schema_qualified_name(name) + scope = {} + scope[:schema] = schema ? quote(schema) : "database()" + scope[:name] = quote(name) if name + scope[:type] = quote(type) if type + scope + end + + def extract_schema_qualified_name(string) + schema, name = string.to_s.scan(/[^`.\s]+|`[^`]*`/) + schema, name = nil, schema unless name + [schema, name] + end + end + end + end +end diff --git a/activerecord/lib/active_record/connection_adapters/mysql/type_metadata.rb b/activerecord/lib/active_record/connection_adapters/mysql/type_metadata.rb index 24dcf852e1..7ad0944d51 100644 --- a/activerecord/lib/active_record/connection_adapters/mysql/type_metadata.rb +++ b/activerecord/lib/active_record/connection_adapters/mysql/type_metadata.rb @@ -1,7 +1,11 @@ +# frozen_string_literal: true + module ActiveRecord module ConnectionAdapters module MySQL class TypeMetadata < DelegateClass(SqlTypeMetadata) # :nodoc: + undef to_yaml if method_defined?(:to_yaml) + attr_reader :extra def initialize(type_metadata, extra: "") diff --git a/activerecord/lib/active_record/connection_adapters/mysql2_adapter.rb b/activerecord/lib/active_record/connection_adapters/mysql2_adapter.rb index 45e400b75b..92f15de219 100644 --- a/activerecord/lib/active_record/connection_adapters/mysql2_adapter.rb +++ b/activerecord/lib/active_record/connection_adapters/mysql2_adapter.rb @@ -1,17 +1,16 @@ +# frozen_string_literal: true + require "active_record/connection_adapters/abstract_mysql_adapter" require "active_record/connection_adapters/mysql/database_statements" -gem "mysql2", ">= 0.3.18", "< 0.5" +gem "mysql2", ">= 0.4.4", "< 0.6.0" require "mysql2" -raise "mysql2 0.4.3 is not supported. Please upgrade to 0.4.4+" if Mysql2::VERSION == "0.4.3" module ActiveRecord module ConnectionHandling # :nodoc: # Establishes a connection to the database that's used by all Active Record objects. def mysql2_connection(config) config = config.symbolize_keys - - config[:username] = "root" if config[:username].nil? config[:flags] ||= 0 if config[:flags].kind_of? Array @@ -59,6 +58,10 @@ module ActiveRecord true end + def supports_lazy_transactions? + true + end + # HELPER METHODS =========================================== def each_hash(result) # :nodoc: @@ -105,6 +108,11 @@ module ActiveRecord @connection.close end + def discard! # :nodoc: + @connection.automatic_close = false + @connection = nil + end + private def connect @@ -113,7 +121,7 @@ module ActiveRecord end def configure_connection - @connection.query_options.merge!(as: :array) + @connection.query_options[:as] = :array super end diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/column.rb b/activerecord/lib/active_record/connection_adapters/postgresql/column.rb index 3ad1911a28..3ccc7271ab 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql/column.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql/column.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module ActiveRecord module ConnectionAdapters # PostgreSQL-specific extensions to column definitions in a table. @@ -5,11 +7,37 @@ module ActiveRecord delegate :array, :oid, :fmod, to: :sql_type_metadata alias :array? :array + def initialize(*, max_identifier_length: 63, **) + super + @max_identifier_length = max_identifier_length + end + def serial? return unless default_function - %r{\Anextval\('"?#{table_name}_#{name}_seq"?'::regclass\)\z} === default_function + if %r{\Anextval\('"?(?<sequence_name>.+_(?<suffix>seq\d*))"?'::regclass\)\z} =~ default_function + sequence_name_from_parts(table_name, name, suffix) == sequence_name + end end + + private + attr_reader :max_identifier_length + + def sequence_name_from_parts(table_name, column_name, suffix) + over_length = [table_name, column_name, suffix].map(&:length).sum + 2 - max_identifier_length + + if over_length > 0 + column_name_length = [(max_identifier_length - suffix.length - 2) / 2, column_name.length].min + over_length -= column_name.length - column_name_length + column_name = column_name[0, column_name_length - [over_length, 0].min] + end + + if over_length > 0 + table_name = table_name[0, table_name.length - over_length] + end + + "#{table_name}_#{column_name}_#{suffix}" + end end end end diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/database_statements.rb b/activerecord/lib/active_record/connection_adapters/postgresql/database_statements.rb index 705e6063dc..6bd6b67165 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql/database_statements.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql/database_statements.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module ActiveRecord module ConnectionAdapters module PostgreSQL @@ -7,30 +9,6 @@ module ActiveRecord PostgreSQL::ExplainPrettyPrinter.new.pp(exec_query(sql, "EXPLAIN", binds)) end - def select_value(arel, name = nil, binds = []) # :nodoc: - select_result(arel, name, binds) do |result| - result.getvalue(0, 0) if result.ntuples > 0 && result.nfields > 0 - end - end - - def select_values(arel, name = nil, binds = []) # :nodoc: - select_result(arel, name, binds) do |result| - if result.nfields > 0 - result.column_values(0) - else - [] - end - end - end - - # Executes a SELECT query and returns an array of rows. Each row is an - # array of field values. - def select_rows(arel, name = nil, binds = []) # :nodoc: - select_result(arel, name, binds) do |result| - result.values - end - end - # The internal PostgreSQL identifier of the money data type. MONEY_COLUMN_TYPE_OID = 790 #:nodoc: # The internal PostgreSQL identifier of the BYTEA data type. @@ -80,6 +58,8 @@ module ActiveRecord # Queries the database and returns the results in an Array-like object def query(sql, name = nil) #:nodoc: + materialize_transactions + log(sql, name) do ActiveSupport::Dependencies.interlock.permit_concurrent_loads do result_as_array @connection.async_exec(sql) @@ -92,6 +72,8 @@ module ActiveRecord # Note: the PG::Result object is manually memory managed; if you don't # need it specifically, you may want consider the <tt>exec_query</tt> wrapper. def execute(sql, name = nil) + materialize_transactions + log(sql, name) do ActiveSupport::Dependencies.interlock.permit_concurrent_loads do @connection.async_exec(sql) @@ -171,18 +153,14 @@ module ActiveRecord end private + # Returns the current ID of a table's sequence. + def last_insert_id_result(sequence_name) + exec_query("SELECT currval(#{quote(sequence_name)})", "SQL") + end def suppress_composite_primary_key(pk) pk unless pk.is_a?(Array) end - - def select_result(arel, name, binds) - arel, binds = binds_from_relation(arel, binds) - sql = to_sql(arel, binds) - execute_and_clear(sql, name, binds) do |result| - yield result - end - end end end end diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/explain_pretty_printer.rb b/activerecord/lib/active_record/connection_adapters/postgresql/explain_pretty_printer.rb index 99f3a5bbdf..086a5dcc15 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql/explain_pretty_printer.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql/explain_pretty_printer.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module ActiveRecord module ConnectionAdapters module PostgreSQL diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/oid.rb b/activerecord/lib/active_record/connection_adapters/postgresql/oid.rb index 4098250f3e..247a25054e 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql/oid.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql/oid.rb @@ -1,14 +1,16 @@ +# frozen_string_literal: true + require "active_record/connection_adapters/postgresql/oid/array" require "active_record/connection_adapters/postgresql/oid/bit" require "active_record/connection_adapters/postgresql/oid/bit_varying" require "active_record/connection_adapters/postgresql/oid/bytea" require "active_record/connection_adapters/postgresql/oid/cidr" +require "active_record/connection_adapters/postgresql/oid/date" require "active_record/connection_adapters/postgresql/oid/date_time" require "active_record/connection_adapters/postgresql/oid/decimal" require "active_record/connection_adapters/postgresql/oid/enum" require "active_record/connection_adapters/postgresql/oid/hstore" require "active_record/connection_adapters/postgresql/oid/inet" -require "active_record/connection_adapters/postgresql/oid/json" require "active_record/connection_adapters/postgresql/oid/jsonb" require "active_record/connection_adapters/postgresql/oid/money" require "active_record/connection_adapters/postgresql/oid/oid" diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/oid/array.rb b/activerecord/lib/active_record/connection_adapters/postgresql/oid/array.rb index e1a75f8e5e..26abeea7ed 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql/oid/array.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql/oid/array.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module ActiveRecord module ConnectionAdapters module PostgreSQL @@ -23,7 +25,7 @@ module ActiveRecord when ::String type_cast_array(@pg_decoder.decode(value), :deserialize) when Data - deserialize(value.values) + type_cast_array(value.values, :deserialize) else super end @@ -64,6 +66,10 @@ module ActiveRecord deserialize(raw_old_value) != new_value end + def force_equality?(value) + value.is_a?(::Array) + end + private def type_cast_array(value, method) diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/oid/bit.rb b/activerecord/lib/active_record/connection_adapters/postgresql/oid/bit.rb index 0a505f46a7..e9a79526f9 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql/oid/bit.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql/oid/bit.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module ActiveRecord module ConnectionAdapters module PostgreSQL @@ -41,10 +43,7 @@ module ActiveRecord /\A[0-9A-F]*\Z/i.match?(value) end - # TODO Change this to private once we've dropped Ruby 2.2 support. - # Workaround for Ruby 2.2 "private attribute?" warning. - protected - + private attr_reader :value end end diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/oid/bit_varying.rb b/activerecord/lib/active_record/connection_adapters/postgresql/oid/bit_varying.rb index 4c21097d48..dc7079dda2 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql/oid/bit_varying.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql/oid/bit_varying.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module ActiveRecord module ConnectionAdapters module PostgreSQL diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/oid/bytea.rb b/activerecord/lib/active_record/connection_adapters/postgresql/oid/bytea.rb index 8f9d6e7f9b..a3c60ecef6 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql/oid/bytea.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql/oid/bytea.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module ActiveRecord module ConnectionAdapters module PostgreSQL @@ -6,7 +8,7 @@ module ActiveRecord def deserialize(value) return if value.nil? return value.to_s if value.is_a?(Type::Binary::Data) - PGconn.unescape_bytea(super) + PG::Connection.unescape_bytea(super) end end end diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/oid/cidr.rb b/activerecord/lib/active_record/connection_adapters/postgresql/oid/cidr.rb index 5225609e37..66e99d9404 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql/oid/cidr.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql/oid/cidr.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require "ipaddr" module ActiveRecord diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/oid/date.rb b/activerecord/lib/active_record/connection_adapters/postgresql/oid/date.rb new file mode 100644 index 0000000000..24a1daa95a --- /dev/null +++ b/activerecord/lib/active_record/connection_adapters/postgresql/oid/date.rb @@ -0,0 +1,23 @@ +# frozen_string_literal: true + +module ActiveRecord + module ConnectionAdapters + module PostgreSQL + module OID # :nodoc: + class Date < Type::Date # :nodoc: + def cast_value(value) + case value + when "infinity" then ::Float::INFINITY + when "-infinity" then -::Float::INFINITY + when / BC$/ + astronomical_year = format("%04d", -value[/^\d+/].to_i + 1) + super(value.sub(/ BC$/, "").sub(/^\d+/, astronomical_year)) + else + super + end + end + end + end + end + end +end diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/oid/date_time.rb b/activerecord/lib/active_record/connection_adapters/postgresql/oid/date_time.rb index b7acbf7178..cd667422f5 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql/oid/date_time.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql/oid/date_time.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module ActiveRecord module ConnectionAdapters module PostgreSQL diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/oid/decimal.rb b/activerecord/lib/active_record/connection_adapters/postgresql/oid/decimal.rb index 43d22c8daf..e7d33855c4 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql/oid/decimal.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql/oid/decimal.rb @@ -1,10 +1,12 @@ +# frozen_string_literal: true + module ActiveRecord module ConnectionAdapters module PostgreSQL module OID # :nodoc: class Decimal < Type::Decimal # :nodoc: def infinity(options = {}) - BigDecimal.new("Infinity") * (options[:negative] ? -1 : 1) + BigDecimal("Infinity") * (options[:negative] ? -1 : 1) end end end diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/oid/enum.rb b/activerecord/lib/active_record/connection_adapters/postgresql/oid/enum.rb index 950d23d516..f70f09ad95 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql/oid/enum.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql/oid/enum.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module ActiveRecord module ConnectionAdapters module PostgreSQL diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/oid/hstore.rb b/activerecord/lib/active_record/connection_adapters/postgresql/oid/hstore.rb index 49dd4fc73f..aabe83b85d 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql/oid/hstore.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql/oid/hstore.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module ActiveRecord module ConnectionAdapters module PostgreSQL diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/oid/inet.rb b/activerecord/lib/active_record/connection_adapters/postgresql/oid/inet.rb index 96486fa65b..55be71fd26 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql/oid/inet.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql/oid/inet.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module ActiveRecord module ConnectionAdapters module PostgreSQL diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/oid/json.rb b/activerecord/lib/active_record/connection_adapters/postgresql/oid/json.rb deleted file mode 100644 index dbc879ffd4..0000000000 --- a/activerecord/lib/active_record/connection_adapters/postgresql/oid/json.rb +++ /dev/null @@ -1,10 +0,0 @@ -module ActiveRecord - module ConnectionAdapters - module PostgreSQL - module OID # :nodoc: - class Json < Type::Internal::AbstractJson - end - end - end - end -end diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/oid/jsonb.rb b/activerecord/lib/active_record/connection_adapters/postgresql/oid/jsonb.rb index 87391b5dc7..e0216f1089 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql/oid/jsonb.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql/oid/jsonb.rb @@ -1,21 +1,13 @@ +# frozen_string_literal: true + module ActiveRecord module ConnectionAdapters module PostgreSQL module OID # :nodoc: - class Jsonb < Json # :nodoc: + class Jsonb < Type::Json # :nodoc: def type :jsonb end - - def changed_in_place?(raw_old_value, new_value) - # Postgres does not preserve insignificant whitespaces when - # round-tripping jsonb columns. This causes some false positives for - # the comparison here. Therefore, we need to parse and re-dump the - # raw value here to ensure the insignificant whitespaces are - # consistent with our encoder's output. - raw_old_value = serialize(deserialize(raw_old_value)) - super(raw_old_value, new_value) - end end end end diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/oid/legacy_point.rb b/activerecord/lib/active_record/connection_adapters/postgresql/oid/legacy_point.rb index 775eecaf85..7b057a8452 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql/oid/legacy_point.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql/oid/legacy_point.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module ActiveRecord module ConnectionAdapters module PostgreSQL diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/oid/money.rb b/activerecord/lib/active_record/connection_adapters/postgresql/oid/money.rb index 7a91272d1c..6434377b57 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql/oid/money.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql/oid/money.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module ActiveRecord module ConnectionAdapters module PostgreSQL @@ -22,7 +24,7 @@ module ActiveRecord # (3) -$2.55 # (4) ($2.55) - value.sub!(/^\((.+)\)$/, '-\1') # (4) + value = value.sub(/^\((.+)\)$/, '-\1') # (4) case value when /^-?\D+[\d,]+\.\d{2}$/ # (1) value.gsub!(/[^-\d.]/, "") diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/oid/oid.rb b/activerecord/lib/active_record/connection_adapters/postgresql/oid/oid.rb index 9c2ac08b30..d8c044320d 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql/oid/oid.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql/oid/oid.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module ActiveRecord module ConnectionAdapters module PostgreSQL diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/oid/point.rb b/activerecord/lib/active_record/connection_adapters/postgresql/oid/point.rb index 7c764e7287..02a9c506f6 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql/oid/point.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql/oid/point.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module ActiveRecord Point = Struct.new(:x, :y) diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/oid/range.rb b/activerecord/lib/active_record/connection_adapters/postgresql/oid/range.rb index 54d5d0902e..d85f9ab3ef 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql/oid/range.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql/oid/range.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module ActiveRecord module ConnectionAdapters module PostgreSQL @@ -33,7 +35,7 @@ module ActiveRecord if value.is_a?(::Range) from = type_cast_single_for_database(value.begin) to = type_cast_single_for_database(value.end) - "[#{from},#{to}#{value.exclude_end? ? ')' : ']'}" + ::Range.new(from, to, value.exclude_end?) else super end @@ -51,6 +53,10 @@ module ActiveRecord ::Range.new(new_begin, new_end, value.exclude_end?) end + def force_equality?(value) + value.is_a?(::Range) + end + private def type_cast_single(value) @@ -58,7 +64,7 @@ module ActiveRecord end def type_cast_single_for_database(value) - infinity?(value) ? "" : @subtype.serialize(value) + infinity?(value) ? value : @subtype.serialize(value) end def extract_bounds(value) diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/oid/specialized_string.rb b/activerecord/lib/active_record/connection_adapters/postgresql/oid/specialized_string.rb index 564e82a4ac..4ad1344f05 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql/oid/specialized_string.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql/oid/specialized_string.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module ActiveRecord module ConnectionAdapters module PostgreSQL diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/oid/type_map_initializer.rb b/activerecord/lib/active_record/connection_adapters/postgresql/oid/type_map_initializer.rb index d9ae1aa7a2..79351bc3a4 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql/oid/type_map_initializer.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql/oid/type_map_initializer.rb @@ -1,3 +1,7 @@ +# frozen_string_literal: true + +require "active_support/core_ext/array/extract" + module ActiveRecord module ConnectionAdapters module PostgreSQL @@ -14,12 +18,12 @@ module ActiveRecord def run(records) nodes = records.reject { |row| @store.key? row["oid"].to_i } - mapped, nodes = nodes.partition { |row| @store.key? row["typname"] } - ranges, nodes = nodes.partition { |row| row["typtype"] == "r".freeze } - enums, nodes = nodes.partition { |row| row["typtype"] == "e".freeze } - domains, nodes = nodes.partition { |row| row["typtype"] == "d".freeze } - arrays, nodes = nodes.partition { |row| row["typinput"] == "array_in".freeze } - composites, nodes = nodes.partition { |row| row["typelem"].to_i != 0 } + mapped = nodes.extract! { |row| @store.key? row["typname"] } + ranges = nodes.extract! { |row| row["typtype"] == "r".freeze } + enums = nodes.extract! { |row| row["typtype"] == "e".freeze } + domains = nodes.extract! { |row| row["typtype"] == "d".freeze } + arrays = nodes.extract! { |row| row["typinput"] == "array_in".freeze } + composites = nodes.extract! { |row| row["typelem"].to_i != 0 } mapped.each { |row| register_mapped_type(row) } enums.each { |row| register_enum_type(row) } @@ -29,8 +33,8 @@ module ActiveRecord composites.each { |row| register_composite_type(row) } end - def query_conditions_for_initial_load(type_map) - known_type_names = type_map.keys.map { |n| "'#{n}'" } + def query_conditions_for_initial_load + known_type_names = @store.keys.map { |n| "'#{n}'" } known_type_types = %w('r' 'e' 'd') <<-SQL % [known_type_names.join(", "), known_type_types.join(", ")] WHERE diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/oid/uuid.rb b/activerecord/lib/active_record/connection_adapters/postgresql/oid/uuid.rb index 5e839228e9..bc9b8dbfcf 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql/oid/uuid.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql/oid/uuid.rb @@ -1,9 +1,11 @@ +# frozen_string_literal: true + module ActiveRecord module ConnectionAdapters module PostgreSQL module OID # :nodoc: class Uuid < Type::Value # :nodoc: - ACCEPTABLE_UUID = %r{\A\{?([a-fA-F0-9]{4}-?){8}\}?\z}x + ACCEPTABLE_UUID = %r{\A(\{)?([a-fA-F0-9]{4}-?){8}(?(1)\}|)\z} alias_method :serialize, :deserialize diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/oid/vector.rb b/activerecord/lib/active_record/connection_adapters/postgresql/oid/vector.rb index b26e876b54..88ef626a16 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql/oid/vector.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql/oid/vector.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module ActiveRecord module ConnectionAdapters module PostgreSQL diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/oid/xml.rb b/activerecord/lib/active_record/connection_adapters/postgresql/oid/xml.rb index d40d837cee..042f32fdc3 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql/oid/xml.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql/oid/xml.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module ActiveRecord module ConnectionAdapters module PostgreSQL diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/quoting.rb b/activerecord/lib/active_record/connection_adapters/postgresql/quoting.rb index 6663448a99..e75202b0be 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql/quoting.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql/quoting.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module ActiveRecord module ConnectionAdapters module PostgreSQL @@ -33,7 +35,7 @@ module ActiveRecord # Quotes schema names for use in SQL queries. def quote_schema_name(name) - PGconn.quote_ident(name) + PG::Connection.quote_ident(name) end def quote_table_name_for_assignment(table, attr) @@ -42,7 +44,7 @@ module ActiveRecord # Quotes column names for use in SQL queries. def quote_column_name(name) # :nodoc: - @quoted_column_names[name] ||= PGconn.quote_ident(super).freeze + @quoted_column_names[name] ||= PG::Connection.quote_ident(super).freeze end # Quote date/time values for use in SQL input. @@ -62,7 +64,7 @@ module ActiveRecord def quote_default_expression(value, column) # :nodoc: if value.is_a?(Proc) value.call - elsif column.type == :uuid && value.include?("()") + elsif column.type == :uuid && value.is_a?(String) && /\(\)/.match?(value) value # Does not quote function default values for UUID columns elsif column.respond_to?(:array?) value = type_cast_from_column(column, value) @@ -77,6 +79,9 @@ module ActiveRecord end private + def lookup_cast_type(sql_type) + super(query_value("SELECT #{quote(sql_type)}::regtype::oid", "SCHEMA").to_i) + end def _quote(value) case value @@ -96,6 +101,8 @@ module ActiveRecord end when OID::Array::Data _quote(encode_array(value)) + when Range + _quote(encode_range(value)) else super end @@ -105,13 +112,15 @@ module ActiveRecord case value when Type::Binary::Data # Return a bind param hash with format as binary. - # See http://deveiate.org/code/pg/PGconn.html#method-i-exec_prepared-doc + # See https://deveiate.org/code/pg/PG/Connection.html#method-i-exec_prepared-doc # for more information { value: value.to_s, format: 1 } when OID::Xml::Data, OID::Bit::Data value.to_s when OID::Array::Data encode_array(value) + when Range + encode_range(value) else super end @@ -128,6 +137,10 @@ module ActiveRecord result end + def encode_range(range) + "[#{type_cast_range_value(range.first)},#{type_cast_range_value(range.last)}#{range.exclude_end? ? ')' : ']'}" + end + def determine_encoding_of_strings_in_array(value) case value when ::Array then determine_encoding_of_strings_in_array(value.first) @@ -141,6 +154,14 @@ module ActiveRecord else _type_cast(values) end end + + def type_cast_range_value(value) + infinity?(value) ? "" : type_cast(value) + end + + def infinity?(value) + value.respond_to?(:infinite?) && value.infinite? + end end end end diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/referential_integrity.rb b/activerecord/lib/active_record/connection_adapters/postgresql/referential_integrity.rb index 44a7338bf5..8df91c988b 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql/referential_integrity.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql/referential_integrity.rb @@ -1,27 +1,24 @@ +# frozen_string_literal: true + module ActiveRecord module ConnectionAdapters module PostgreSQL module ReferentialIntegrity # :nodoc: - def supports_disable_referential_integrity? # :nodoc: - true - end - def disable_referential_integrity # :nodoc: - if supports_disable_referential_integrity? - original_exception = nil + original_exception = nil - begin - transaction(requires_new: true) do - execute(tables.collect { |name| "ALTER TABLE #{quote_table_name(name)} DISABLE TRIGGER ALL" }.join(";")) - end - rescue ActiveRecord::ActiveRecordError => e - original_exception = e + begin + transaction(requires_new: true) do + execute(tables.collect { |name| "ALTER TABLE #{quote_table_name(name)} DISABLE TRIGGER ALL" }.join(";")) end + rescue ActiveRecord::ActiveRecordError => e + original_exception = e + end - begin - yield - rescue ActiveRecord::InvalidForeignKey => e - warn <<-WARNING + begin + yield + rescue ActiveRecord::InvalidForeignKey => e + warn <<-WARNING WARNING: Rails was not able to disable referential integrity. This is most likely caused due to missing permissions. @@ -30,17 +27,14 @@ Rails needs superuser privileges to disable referential integrity. cause: #{original_exception.try(:message)} WARNING - raise e - end + raise e + end - begin - transaction(requires_new: true) do - execute(tables.collect { |name| "ALTER TABLE #{quote_table_name(name)} ENABLE TRIGGER ALL" }.join(";")) - end - rescue ActiveRecord::ActiveRecordError + begin + transaction(requires_new: true) do + execute(tables.collect { |name| "ALTER TABLE #{quote_table_name(name)} ENABLE TRIGGER ALL" }.join(";")) end - else - yield + rescue ActiveRecord::ActiveRecordError end end end diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/schema_creation.rb b/activerecord/lib/active_record/connection_adapters/postgresql/schema_creation.rb index e1d5089115..8e381a92cf 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql/schema_creation.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql/schema_creation.rb @@ -1,8 +1,22 @@ +# frozen_string_literal: true + module ActiveRecord module ConnectionAdapters module PostgreSQL class SchemaCreation < AbstractAdapter::SchemaCreation # :nodoc: private + def visit_AlterTable(o) + super << o.constraint_validations.map { |fk| visit_ValidateConstraint fk }.join(" ") + end + + def visit_AddForeignKey(o) + super.dup.tap { |sql| sql << " NOT VALID" unless o.validate? } + end + + def visit_ValidateConstraint(name) + "VALIDATE CONSTRAINT #{quote_column_name(name)}" + end + def add_column_options!(sql, options) if options[:collation] sql << " COLLATE \"#{options[:collation]}\"" diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/schema_definitions.rb b/activerecord/lib/active_record/connection_adapters/postgresql/schema_definitions.rb index 11ea1e5144..206b855a18 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql/schema_definitions.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql/schema_definitions.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module ActiveRecord module ConnectionAdapters module PostgreSQL @@ -11,10 +13,10 @@ module ActiveRecord # t.timestamps # end # - # By default, this will use the +gen_random_uuid()+ function from the + # By default, this will use the <tt>gen_random_uuid()</tt> function from the # +pgcrypto+ extension. As that extension is only available in # PostgreSQL 9.4+, for earlier versions an explicit default can be set - # to use +uuid_generate_v4()+ from the +uuid-ossp+ extension instead: + # to use <tt>uuid_generate_v4()</tt> from the +uuid-ossp+ extension instead: # # create_table :stuffs, id: false do |t| # t.primary_key :id, :uuid, default: "uuid_generate_v4()" @@ -42,15 +44,8 @@ module ActiveRecord # a record (as primary keys cannot be +nil+). This might be done via the # +SecureRandom.uuid+ method and a +before_save+ callback, for instance. def primary_key(name, type = :primary_key, **options) - options[:auto_increment] = true if [:integer, :bigint].include?(type) && !options.key?(:default) if type == :uuid options[:default] = options.fetch(:default, "gen_random_uuid()") - elsif options.delete(:auto_increment) == true && %i(integer bigint).include?(type) - type = if type == :bigint || options[:limit] == 8 - :bigserial - else - :serial - end end super @@ -100,10 +95,6 @@ module ActiveRecord args.each { |name| column(name, :int8range, options) } end - def json(*args, **options) - args.each { |name| column(name, :json, options) } - end - def jsonb(*args, **options) args.each { |name| column(name, :jsonb, options) } end @@ -183,11 +174,33 @@ module ActiveRecord class TableDefinition < ActiveRecord::ConnectionAdapters::TableDefinition include ColumnMethods + + private + def integer_like_primary_key_type(type, options) + if type == :bigint || options[:limit] == 8 + :bigserial + else + :serial + end + end end class Table < ActiveRecord::ConnectionAdapters::Table include ColumnMethods end + + class AlterTable < ActiveRecord::ConnectionAdapters::AlterTable + attr_reader :constraint_validations + + def initialize(td) + super + @constraint_validations = [] + end + + def validate_constraint(name) + @constraint_validations << name + end + end end end end diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/schema_dumper.rb b/activerecord/lib/active_record/connection_adapters/postgresql/schema_dumper.rb index 5555a46b6b..84643d20da 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql/schema_dumper.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql/schema_dumper.rb @@ -1,21 +1,28 @@ +# frozen_string_literal: true + module ActiveRecord module ConnectionAdapters module PostgreSQL - module ColumnDumper # :nodoc: - # Adds +:array+ option to the default set - def prepare_column_options(column) - spec = super - spec[:array] = "true" if column.array? - spec - end - - # Adds +:array+ as a valid migration key - def migration_keys - super + [:array] - end - + class SchemaDumper < ConnectionAdapters::SchemaDumper # :nodoc: private + def extensions(stream) + extensions = @connection.extensions + if extensions.any? + stream.puts " # These are extensions that must be enabled in order to support this database" + extensions.sort.each do |extension| + stream.puts " enable_extension #{extension.inspect}" + end + stream.puts + end + end + + def prepare_column_options(column) + spec = super + spec[:array] = "true" if column.array? + spec + end + def default_primary_key?(column) schema_type(column) == :bigserial end diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/schema_statements.rb b/activerecord/lib/active_record/connection_adapters/postgresql/schema_statements.rb index afef0da5c7..00da7690a2 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql/schema_statements.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql/schema_statements.rb @@ -1,4 +1,4 @@ -require "active_support/core_ext/string/strip" +# frozen_string_literal: true module ActiveRecord module ConnectionAdapters @@ -38,7 +38,7 @@ module ActiveRecord " TABLESPACE = \"#{value}\"" when :connection_limit " CONNECTION LIMIT = #{value}" - else + else "" end end @@ -54,130 +54,48 @@ module ActiveRecord execute "DROP DATABASE IF EXISTS #{quote_table_name(name)}" end - # Returns the list of all tables in the schema search path. - def tables - select_values("SELECT tablename FROM pg_tables WHERE schemaname = ANY(current_schemas(false))", "SCHEMA") - end - - def data_sources # :nodoc - select_values(<<-SQL, "SCHEMA") - SELECT c.relname - FROM pg_class c - LEFT JOIN pg_namespace n ON n.oid = c.relnamespace - WHERE c.relkind IN ('r','v','m') -- (r)elation/table, (v)iew, (m)aterialized view - AND n.nspname = ANY (current_schemas(false)) - SQL - end - - def views # :nodoc: - select_values(<<-SQL, "SCHEMA") - SELECT c.relname - FROM pg_class c - LEFT JOIN pg_namespace n ON n.oid = c.relnamespace - WHERE c.relkind IN ('v','m') -- (v)iew, (m)aterialized view - AND n.nspname = ANY (current_schemas(false)) - SQL - end - - # Returns true if table exists. - # If the schema is not specified as part of +name+ then it will only find tables within - # the current schema search path (regardless of permissions to access tables in other schemas) - def table_exists?(name) - name = Utils.extract_schema_qualified_name(name.to_s) - return false unless name.identifier - - select_values(<<-SQL, "SCHEMA").any? - SELECT tablename - FROM pg_tables - WHERE tablename = #{quote(name.identifier)} - AND schemaname = #{name.schema ? quote(name.schema) : "ANY (current_schemas(false))"} - SQL - end - - def data_source_exists?(name) # :nodoc: - name = Utils.extract_schema_qualified_name(name.to_s) - return false unless name.identifier - - select_values(<<-SQL, "SCHEMA").any? - SELECT c.relname - FROM pg_class c - LEFT JOIN pg_namespace n ON n.oid = c.relnamespace - WHERE c.relkind IN ('r','v','m') -- (r)elation/table, (v)iew, (m)aterialized view - AND c.relname = #{quote(name.identifier)} - AND n.nspname = #{name.schema ? quote(name.schema) : "ANY (current_schemas(false))"} - SQL - end - - def view_exists?(view_name) # :nodoc: - name = Utils.extract_schema_qualified_name(view_name.to_s) - return false unless name.identifier - - select_values(<<-SQL, "SCHEMA").any? - SELECT c.relname - FROM pg_class c - LEFT JOIN pg_namespace n ON n.oid = c.relnamespace - WHERE c.relkind IN ('v','m') -- (v)iew, (m)aterialized view - AND c.relname = #{quote(name.identifier)} - AND n.nspname = #{name.schema ? quote(name.schema) : "ANY (current_schemas(false))"} - SQL - end - def drop_table(table_name, options = {}) # :nodoc: execute "DROP TABLE#{' IF EXISTS' if options[:if_exists]} #{quote_table_name(table_name)}#{' CASCADE' if options[:force] == :cascade}" end # Returns true if schema exists. def schema_exists?(name) - select_value("SELECT COUNT(*) FROM pg_namespace WHERE nspname = '#{name}'", "SCHEMA").to_i > 0 + query_value("SELECT COUNT(*) FROM pg_namespace WHERE nspname = #{quote(name)}", "SCHEMA").to_i > 0 end # Verifies existence of an index with a given name. - def index_name_exists?(table_name, index_name, default = nil) - unless default.nil? - ActiveSupport::Deprecation.warn(<<-MSG.squish) - Passing default to #index_name_exists? is deprecated without replacement. - MSG - end - table = Utils.extract_schema_qualified_name(table_name.to_s) - index = Utils.extract_schema_qualified_name(index_name.to_s) + def index_name_exists?(table_name, index_name) + table = quoted_scope(table_name) + index = quoted_scope(index_name) - select_value(<<-SQL, "SCHEMA").to_i > 0 + query_value(<<-SQL, "SCHEMA").to_i > 0 SELECT COUNT(*) FROM pg_class t INNER JOIN pg_index d ON t.oid = d.indrelid INNER JOIN pg_class i ON d.indexrelid = i.oid LEFT JOIN pg_namespace n ON n.oid = i.relnamespace WHERE i.relkind = 'i' - AND i.relname = '#{index.identifier}' - AND t.relname = '#{table.identifier}' - AND n.nspname = #{index.schema ? "'#{index.schema}'" : 'ANY (current_schemas(false))'} + AND i.relname = #{index[:name]} + AND t.relname = #{table[:name]} + AND n.nspname = #{index[:schema]} SQL end # Returns an array of indexes for the given table. - def indexes(table_name, name = nil) # :nodoc: - if name - ActiveSupport::Deprecation.warn(<<-MSG.squish) - Passing name to #indexes is deprecated without replacement. - MSG - end - - table = Utils.extract_schema_qualified_name(table_name.to_s) + def indexes(table_name) # :nodoc: + scope = quoted_scope(table_name) result = query(<<-SQL, "SCHEMA") SELECT distinct i.relname, d.indisunique, d.indkey, pg_get_indexdef(d.indexrelid), t.oid, - pg_catalog.obj_description(i.oid, 'pg_class') AS comment, - (SELECT COUNT(*) FROM pg_opclass o - JOIN (SELECT unnest(string_to_array(d.indclass::text, ' '))::int oid) c - ON o.oid = c.oid WHERE o.opcdefault = 'f') + pg_catalog.obj_description(i.oid, 'pg_class') AS comment FROM pg_class t INNER JOIN pg_index d ON t.oid = d.indrelid INNER JOIN pg_class i ON d.indexrelid = i.oid LEFT JOIN pg_namespace n ON n.oid = i.relnamespace WHERE i.relkind = 'i' AND d.indisprimary = 'f' - AND t.relname = '#{table.identifier}' - AND n.nspname = #{table.schema ? "'#{table.schema}'" : 'ANY (current_schemas(false))'} + AND t.relname = #{scope[:name]} + AND n.nspname = #{scope[:schema]} ORDER BY i.relname SQL @@ -188,47 +106,46 @@ module ActiveRecord inddef = row[3] oid = row[4] comment = row[5] - opclass = row[6] - using, expressions, where = inddef.scan(/ USING (\w+?) \((.+?)\)(?: WHERE (.+))?\z/).flatten + using, expressions, where = inddef.scan(/ USING (\w+?) \((.+?)\)(?: WHERE (.+))?\z/m).flatten - if indkey.include?(0) || opclass > 0 + orders = {} + opclasses = {} + + if indkey.include?(0) columns = expressions else - columns = Hash[query(<<-SQL.strip_heredoc, "SCHEMA")].values_at(*indkey).compact + columns = Hash[query(<<~SQL, "SCHEMA")].values_at(*indkey).compact SELECT a.attnum, a.attname FROM pg_attribute a WHERE a.attrelid = #{oid} AND a.attnum IN (#{indkey.join(",")}) SQL - # add info on sort order for columns (only desc order is explicitly specified, asc is the default) - orders = Hash[ - expressions.scan(/(\w+) DESC/).flatten.map { |order_column| [order_column, :desc] } - ] + # add info on sort order (only desc order is explicitly specified, asc is the default) + # and non-default opclasses + expressions.scan(/(?<column>\w+)\s?(?<opclass>\w+_ops)?\s?(?<desc>DESC)?\s?(?<nulls>NULLS (?:FIRST|LAST))?/).each do |column, opclass, desc, nulls| + opclasses[column] = opclass.to_sym if opclass + if nulls + orders[column] = [desc, nulls].compact.join(" ") + else + orders[column] = :desc if desc + end + end end - IndexDefinition.new(table_name, index_name, unique, columns, [], orders, where, nil, using.to_sym, comment.presence) - end.compact - end - - def new_column_from_field(table_name, field) # :nondoc: - column_name, type, default, notnull, oid, fmod, collation, comment = field - oid = oid.to_i - fmod = fmod.to_i - type_metadata = fetch_type_metadata(column_name, type, oid, fmod) - default_value = extract_value_from_default(default) - default_function = extract_default_function(default_value, default) - PostgreSQLColumn.new( - column_name, - default_value, - type_metadata, - !notnull, - table_name, - default_function, - collation, - comment: comment.presence - ) + IndexDefinition.new( + table_name, + index_name, + unique, + columns, + orders: orders, + opclasses: opclasses, + where: where, + using: using.to_sym, + comment: comment.presence + ) + end end def table_options(table_name) # :nodoc: @@ -239,47 +156,47 @@ module ActiveRecord # Returns a comment stored in database for given table def table_comment(table_name) # :nodoc: - name = Utils.extract_schema_qualified_name(table_name.to_s) - if name.identifier - select_value(<<-SQL.strip_heredoc, "SCHEMA") + scope = quoted_scope(table_name, type: "BASE TABLE") + if scope[:name] + query_value(<<~SQL, "SCHEMA") SELECT pg_catalog.obj_description(c.oid, 'pg_class') FROM pg_catalog.pg_class c LEFT JOIN pg_namespace n ON n.oid = c.relnamespace - WHERE c.relname = #{quote(name.identifier)} - AND c.relkind IN ('r') -- (r)elation/table - AND n.nspname = #{name.schema ? quote(name.schema) : 'ANY (current_schemas(false))'} + WHERE c.relname = #{scope[:name]} + AND c.relkind IN (#{scope[:type]}) + AND n.nspname = #{scope[:schema]} SQL end end # Returns the current database name. def current_database - select_value("select current_database()", "SCHEMA") + query_value("SELECT current_database()", "SCHEMA") end # Returns the current schema name. def current_schema - select_value("SELECT current_schema", "SCHEMA") + query_value("SELECT current_schema", "SCHEMA") end # Returns the current database encoding format. def encoding - select_value("SELECT pg_encoding_to_char(encoding) FROM pg_database WHERE datname LIKE '#{current_database}'", "SCHEMA") + query_value("SELECT pg_encoding_to_char(encoding) FROM pg_database WHERE datname = current_database()", "SCHEMA") end # Returns the current database collation. def collation - select_value("SELECT datcollate FROM pg_database WHERE datname LIKE '#{current_database}'", "SCHEMA") + query_value("SELECT datcollate FROM pg_database WHERE datname = current_database()", "SCHEMA") end # Returns the current database ctype. def ctype - select_value("SELECT datctype FROM pg_database WHERE datname LIKE '#{current_database}'", "SCHEMA") + query_value("SELECT datctype FROM pg_database WHERE datname = current_database()", "SCHEMA") end # Returns an array of schema names. def schema_names - select_values(<<-SQL, "SCHEMA") + query_values(<<-SQL, "SCHEMA") SELECT nspname FROM pg_namespace WHERE nspname !~ '^pg_.*' @@ -300,7 +217,7 @@ module ActiveRecord # Sets the schema search path to a string of comma-separated schema names. # Names beginning with $ have to be quoted (e.g. $user => '$user'). - # See: http://www.postgresql.org/docs/current/static/ddl-schemas.html + # See: https://www.postgresql.org/docs/current/static/ddl-schemas.html # # This should be not be called manually but set in database.yml. def schema_search_path=(schema_csv) @@ -312,12 +229,12 @@ module ActiveRecord # Returns the active schema search path. def schema_search_path - @schema_search_path ||= select_value("SHOW search_path", "SCHEMA") + @schema_search_path ||= query_value("SHOW search_path", "SCHEMA") end # Returns the current client message level. def client_min_messages - select_value("SHOW client_min_messages", "SCHEMA") + query_value("SHOW client_min_messages", "SCHEMA") end # Set the client message level. @@ -335,7 +252,7 @@ module ActiveRecord end def serial_sequence(table, column) - select_value("SELECT pg_get_serial_sequence('#{table}', '#{column}')", "SCHEMA") + query_value("SELECT pg_get_serial_sequence(#{quote(table)}, #{quote(column)})", "SCHEMA") end # Sets the sequence of a table's primary key to the specified value. @@ -346,7 +263,7 @@ module ActiveRecord if sequence quoted_sequence = quote_table_name(sequence) - select_value("SELECT setval('#{quoted_sequence}', #{value})", "SCHEMA") + query_value("SELECT setval(#{quote(quoted_sequence)}, #{value})", "SCHEMA") else @logger.warn "#{table} has primary key #{pk} with no default sequence." if @logger end @@ -368,10 +285,16 @@ module ActiveRecord if pk && sequence quoted_sequence = quote_table_name(sequence) + max_pk = query_value("SELECT MAX(#{quote_column_name pk}) FROM #{quote_table_name(table)}", "SCHEMA") + if max_pk.nil? + if postgresql_version >= 100000 + minvalue = query_value("SELECT seqmin FROM pg_sequence WHERE seqrelid = #{quote(quoted_sequence)}::regclass", "SCHEMA") + else + minvalue = query_value("SELECT min_value FROM #{quoted_sequence}", "SCHEMA") + end + end - select_value(<<-end_sql, "SCHEMA") - SELECT setval('#{quoted_sequence}', (SELECT COALESCE(MAX(#{quote_column_name pk})+(SELECT increment_by FROM #{quoted_sequence}), (SELECT min_value FROM #{quoted_sequence})) FROM #{quote_table_name(table)}), false) - end_sql + query_value("SELECT setval(#{quote(quoted_sequence)}, #{max_pk ? max_pk : minvalue}, #{max_pk ? true : false})", "SCHEMA") end end @@ -395,7 +318,7 @@ module ActiveRecord AND seq.relnamespace = nsp.oid AND cons.contype = 'p' AND dep.classid = 'pg_class'::regclass - AND dep.refobjid = '#{quote_table_name(table)}'::regclass + AND dep.refobjid = #{quote(quote_table_name(table))}::regclass end_sql if result.nil? || result.empty? @@ -413,7 +336,7 @@ module ActiveRecord JOIN pg_attrdef def ON (adrelid = attrelid AND adnum = attnum) JOIN pg_constraint cons ON (conrelid = adrelid AND adnum = conkey[1]) JOIN pg_namespace nsp ON (t.relnamespace = nsp.oid) - WHERE t.oid = '#{quote_table_name(table)}'::regclass + WHERE t.oid = #{quote(quote_table_name(table))}::regclass AND cons.contype = 'p' AND pg_get_expr(def.adbin, def.adrelid) ~* 'nextval|uuid_generate' end_sql @@ -430,21 +353,46 @@ module ActiveRecord end def primary_keys(table_name) # :nodoc: - name = Utils.extract_schema_qualified_name(table_name.to_s) - select_values(<<-SQL.strip_heredoc, "SCHEMA") - SELECT column_name - FROM information_schema.key_column_usage kcu - JOIN information_schema.table_constraints tc - ON kcu.table_name = tc.table_name - AND kcu.table_schema = tc.table_schema - AND kcu.constraint_name = tc.constraint_name - WHERE constraint_type = 'PRIMARY KEY' - AND kcu.table_name = #{quote(name.identifier)} - AND kcu.table_schema = #{name.schema ? quote(name.schema) : "ANY (current_schemas(false))"} - ORDER BY kcu.ordinal_position + query_values(<<~SQL, "SCHEMA") + SELECT a.attname + FROM ( + SELECT indrelid, indkey, generate_subscripts(indkey, 1) idx + FROM pg_index + WHERE indrelid = #{quote(quote_table_name(table_name))}::regclass + AND indisprimary + ) i + JOIN pg_attribute a + ON a.attrelid = i.indrelid + AND a.attnum = i.indkey[i.idx] + ORDER BY i.idx SQL end + def bulk_change_table(table_name, operations) + sql_fragments = [] + non_combinable_operations = [] + + operations.each do |command, args| + table, arguments = args.shift, args + method = :"#{command}_for_alter" + + if respond_to?(method, true) + sqls, procs = Array(send(method, table, *arguments)).partition { |v| v.is_a?(String) } + sql_fragments << sqls + non_combinable_operations.concat(procs) + else + execute "ALTER TABLE #{quote_table_name(table_name)} #{sql_fragments.join(", ")}" unless sql_fragments.empty? + non_combinable_operations.each(&:call) + sql_fragments = [] + non_combinable_operations = [] + send(command, table, *arguments) + end + end + + execute "ALTER TABLE #{quote_table_name(table_name)} #{sql_fragments.join(", ")}" unless sql_fragments.empty? + non_combinable_operations.each(&:call) + end + # Renames a table. # Also renames a table's primary key sequence if the sequence name exists and # matches the Active Record default. @@ -455,14 +403,15 @@ module ActiveRecord clear_cache! execute "ALTER TABLE #{quote_table_name(table_name)} RENAME TO #{quote_table_name(new_name)}" pk, seq = pk_and_sequence_for(new_name) - if seq && seq.identifier == "#{table_name}_#{pk}_seq" - new_seq = "#{new_name}_#{pk}_seq" + if pk idx = "#{table_name}_pkey" new_idx = "#{new_name}_pkey" - execute "ALTER TABLE #{seq.quoted} RENAME TO #{quote_table_name(new_seq)}" execute "ALTER INDEX #{quote_table_name(idx)} RENAME TO #{quote_table_name(new_idx)}" + if seq && seq.identifier == "#{table_name}_#{pk}_seq" + new_seq = "#{new_name}_#{pk}_seq" + execute "ALTER TABLE #{seq.quoted} RENAME TO #{quote_table_name(new_seq)}" + end end - rename_table_indexes(table_name, new_name) end @@ -474,50 +423,23 @@ module ActiveRecord def change_column(table_name, column_name, type, options = {}) #:nodoc: clear_cache! - quoted_table_name = quote_table_name(table_name) - quoted_column_name = quote_column_name(column_name) - sql_type = type_to_sql(type, options) - sql = "ALTER TABLE #{quoted_table_name} ALTER COLUMN #{quoted_column_name} TYPE #{sql_type}" - if options[:collation] - sql << " COLLATE \"#{options[:collation]}\"" - end - if options[:using] - sql << " USING #{options[:using]}" - elsif options[:cast_as] - cast_as_type = type_to_sql(options[:cast_as], options) - sql << " USING CAST(#{quoted_column_name} AS #{cast_as_type})" - end - execute sql - - change_column_default(table_name, column_name, options[:default]) if options.key?(:default) - change_column_null(table_name, column_name, options[:null], options[:default]) if options.key?(:null) - change_column_comment(table_name, column_name, options[:comment]) if options.key?(:comment) + sqls, procs = Array(change_column_for_alter(table_name, column_name, type, options)).partition { |v| v.is_a?(String) } + execute "ALTER TABLE #{quote_table_name(table_name)} #{sqls.join(", ")}" + procs.each(&:call) end # Changes the default value of a table column. def change_column_default(table_name, column_name, default_or_changes) # :nodoc: - clear_cache! - column = column_for(table_name, column_name) - return unless column - - default = extract_new_default_value(default_or_changes) - alter_column_query = "ALTER TABLE #{quote_table_name(table_name)} ALTER COLUMN #{quote_column_name(column_name)} %s" - if default.nil? - # <tt>DEFAULT NULL</tt> results in the same behavior as <tt>DROP DEFAULT</tt>. However, PostgreSQL will - # cast the default to the columns type, which leaves us with a default like "default NULL::character varying". - execute alter_column_query % "DROP DEFAULT" - else - execute alter_column_query % "SET DEFAULT #{quote_default_expression(default, column)}" - end + execute "ALTER TABLE #{quote_table_name(table_name)} #{change_column_default_for_alter(table_name, column_name, default_or_changes)}" end def change_column_null(table_name, column_name, null, default = nil) #:nodoc: clear_cache! unless null || default.nil? column = column_for(table_name, column_name) - execute("UPDATE #{quote_table_name(table_name)} SET #{quote_column_name(column_name)}=#{quote_default_expression(default, column)} WHERE #{quote_column_name(column_name)} IS NULL") if column + execute "UPDATE #{quote_table_name(table_name)} SET #{quote_column_name(column_name)}=#{quote_default_expression(default, column)} WHERE #{quote_column_name(column_name)} IS NULL" if column end - execute("ALTER TABLE #{quote_table_name(table_name)} ALTER #{quote_column_name(column_name)} #{null ? 'DROP' : 'SET'} NOT NULL") + execute "ALTER TABLE #{quote_table_name(table_name)} #{change_column_null_for_alter(table_name, column_name, null, default)}" end # Adds comment for given table column or drops it if +comment+ is a +nil+ @@ -540,8 +462,8 @@ module ActiveRecord end def add_index(table_name, column_name, options = {}) #:nodoc: - index_name, index_type, index_columns, index_options, index_algorithm, index_using, comment = add_index_options(table_name, column_name, options) - execute("CREATE #{index_type} INDEX #{index_algorithm} #{quote_column_name(index_name)} ON #{quote_table_name(table_name)} #{index_using} (#{index_columns})#{index_options}").tap do + index_name, index_type, index_columns_and_opclasses, index_options, index_algorithm, index_using, comment = add_index_options(table_name, column_name, options) + execute("CREATE #{index_type} INDEX #{index_algorithm} #{quote_column_name(index_name)} ON #{quote_table_name(table_name)} #{index_using} (#{index_columns_and_opclasses})#{index_options}").tap do execute "COMMENT ON INDEX #{quote_column_name(index_name)} IS #{quote(comment)}" if comment end end @@ -579,8 +501,9 @@ module ActiveRecord end def foreign_keys(table_name) - fk_info = select_all(<<-SQL.strip_heredoc, "SCHEMA") - SELECT t2.oid::regclass::text AS to_table, a1.attname AS column, a2.attname AS primary_key, c.conname AS name, c.confupdtype AS on_update, c.confdeltype AS on_delete + scope = quoted_scope(table_name) + fk_info = exec_query(<<~SQL, "SCHEMA") + SELECT t2.oid::regclass::text AS to_table, a1.attname AS column, a2.attname AS primary_key, c.conname AS name, c.confupdtype AS on_update, c.confdeltype AS on_delete, c.convalidated AS valid FROM pg_constraint c JOIN pg_class t1 ON c.conrelid = t1.oid JOIN pg_class t2 ON c.confrelid = t2.oid @@ -588,8 +511,8 @@ module ActiveRecord JOIN pg_attribute a2 ON a2.attnum = c.confkey[1] AND a2.attrelid = t2.oid JOIN pg_namespace t3 ON c.connamespace = t3.oid WHERE c.contype = 'f' - AND t1.relname = #{quote(table_name)} - AND t3.nspname = ANY (current_schemas(false)) + AND t1.relname = #{scope[:name]} + AND t3.nspname = #{scope[:schema]} ORDER BY c.conname SQL @@ -602,17 +525,18 @@ module ActiveRecord options[:on_delete] = extract_foreign_key_action(row["on_delete"]) options[:on_update] = extract_foreign_key_action(row["on_update"]) + options[:validate] = row["valid"] ForeignKeyDefinition.new(table_name, row["to_table"], options) end end - def extract_foreign_key_action(specifier) # :nodoc: - case specifier - when "c"; :cascade - when "n"; :nullify - when "r"; :restrict - end + def foreign_tables + query_values(data_source_sql(type: "FOREIGN TABLE"), "SCHEMA") + end + + def foreign_table_exists?(table_name) + query_values(data_source_sql(table_name, type: "FOREIGN TABLE"), "SCHEMA").any? if table_name.present? end # Maps logical Rails types to PostgreSQL-specific data types. @@ -644,7 +568,7 @@ module ActiveRecord super end - sql << "[]" if array && type != :primary_key + sql = "#{sql}[]" if array && type != :primary_key sql end @@ -659,20 +583,209 @@ module ActiveRecord .gsub(/\s+NULLS\s+(?:FIRST|LAST)\b/i, "") }.reject(&:blank?).map.with_index { |column, i| "#{column} AS alias_#{i}" } - [super, *order_columns].join(", ") + (order_columns << super).join(", ") + end + + def update_table_definition(table_name, base) # :nodoc: + PostgreSQL::Table.new(table_name, base) + end + + def create_schema_dumper(options) # :nodoc: + PostgreSQL::SchemaDumper.create(self, options) end - def fetch_type_metadata(column_name, sql_type, oid, fmod) - cast_type = get_oid_type(oid, fmod, column_name, sql_type) - simple_type = SqlTypeMetadata.new( - sql_type: sql_type, - type: cast_type.type, - limit: cast_type.limit, - precision: cast_type.precision, - scale: cast_type.scale, - ) - PostgreSQLTypeMetadata.new(simple_type, oid: oid, fmod: fmod) + # Validates the given constraint. + # + # Validates the constraint named +constraint_name+ on +accounts+. + # + # validate_constraint :accounts, :constraint_name + def validate_constraint(table_name, constraint_name) + return unless supports_validate_constraints? + + at = create_alter_table table_name + at.validate_constraint constraint_name + + execute schema_creation.accept(at) end + + # Validates the given foreign key. + # + # Validates the foreign key on +accounts.branch_id+. + # + # validate_foreign_key :accounts, :branches + # + # Validates the foreign key on +accounts.owner_id+. + # + # validate_foreign_key :accounts, column: :owner_id + # + # Validates the foreign key named +special_fk_name+ on the +accounts+ table. + # + # validate_foreign_key :accounts, name: :special_fk_name + # + # The +options+ hash accepts the same keys as SchemaStatements#add_foreign_key. + def validate_foreign_key(from_table, options_or_to_table = {}) + return unless supports_validate_constraints? + + fk_name_to_validate = foreign_key_for!(from_table, options_or_to_table).name + + validate_constraint from_table, fk_name_to_validate + end + + private + def schema_creation + PostgreSQL::SchemaCreation.new(self) + end + + def create_table_definition(*args) + PostgreSQL::TableDefinition.new(*args) + end + + def create_alter_table(name) + PostgreSQL::AlterTable.new create_table_definition(name) + end + + def new_column_from_field(table_name, field) + column_name, type, default, notnull, oid, fmod, collation, comment = field + type_metadata = fetch_type_metadata(column_name, type, oid.to_i, fmod.to_i) + default_value = extract_value_from_default(default) + default_function = extract_default_function(default_value, default) + + PostgreSQLColumn.new( + column_name, + default_value, + type_metadata, + !notnull, + table_name, + default_function, + collation, + comment: comment.presence, + max_identifier_length: max_identifier_length + ) + end + + def fetch_type_metadata(column_name, sql_type, oid, fmod) + cast_type = get_oid_type(oid, fmod, column_name, sql_type) + simple_type = SqlTypeMetadata.new( + sql_type: sql_type, + type: cast_type.type, + limit: cast_type.limit, + precision: cast_type.precision, + scale: cast_type.scale, + ) + PostgreSQLTypeMetadata.new(simple_type, oid: oid, fmod: fmod) + end + + def extract_foreign_key_action(specifier) + case specifier + when "c"; :cascade + when "n"; :nullify + when "r"; :restrict + end + end + + def change_column_sql(table_name, column_name, type, options = {}) + quoted_column_name = quote_column_name(column_name) + sql_type = type_to_sql(type, options) + sql = "ALTER COLUMN #{quoted_column_name} TYPE #{sql_type}".dup + if options[:collation] + sql << " COLLATE \"#{options[:collation]}\"" + end + if options[:using] + sql << " USING #{options[:using]}" + elsif options[:cast_as] + cast_as_type = type_to_sql(options[:cast_as], options) + sql << " USING CAST(#{quoted_column_name} AS #{cast_as_type})" + end + + sql + end + + def add_column_for_alter(table_name, column_name, type, options = {}) + return super unless options.key?(:comment) + [super, Proc.new { change_column_comment(table_name, column_name, options[:comment]) }] + end + + def change_column_for_alter(table_name, column_name, type, options = {}) + sqls = [change_column_sql(table_name, column_name, type, options)] + sqls << change_column_default_for_alter(table_name, column_name, options[:default]) if options.key?(:default) + sqls << change_column_null_for_alter(table_name, column_name, options[:null], options[:default]) if options.key?(:null) + sqls << Proc.new { change_column_comment(table_name, column_name, options[:comment]) } if options.key?(:comment) + sqls + end + + # Changes the default value of a table column. + def change_column_default_for_alter(table_name, column_name, default_or_changes) # :nodoc: + column = column_for(table_name, column_name) + return unless column + + default = extract_new_default_value(default_or_changes) + alter_column_query = "ALTER COLUMN #{quote_column_name(column_name)} %s" + if default.nil? + # <tt>DEFAULT NULL</tt> results in the same behavior as <tt>DROP DEFAULT</tt>. However, PostgreSQL will + # cast the default to the columns type, which leaves us with a default like "default NULL::character varying". + alter_column_query % "DROP DEFAULT" + else + alter_column_query % "SET DEFAULT #{quote_default_expression(default, column)}" + end + end + + def change_column_null_for_alter(table_name, column_name, null, default = nil) #:nodoc: + "ALTER #{quote_column_name(column_name)} #{null ? 'DROP' : 'SET'} NOT NULL" + end + + def add_timestamps_for_alter(table_name, options = {}) + [add_column_for_alter(table_name, :created_at, :datetime, options), add_column_for_alter(table_name, :updated_at, :datetime, options)] + end + + def remove_timestamps_for_alter(table_name, options = {}) + [remove_column_for_alter(table_name, :updated_at), remove_column_for_alter(table_name, :created_at)] + end + + def add_index_opclass(quoted_columns, **options) + opclasses = options_for_index_columns(options[:opclass]) + quoted_columns.each do |name, column| + column << " #{opclasses[name]}" if opclasses[name].present? + end + end + + def add_options_for_index_columns(quoted_columns, **options) + quoted_columns = add_index_opclass(quoted_columns, options) + super + end + + def data_source_sql(name = nil, type: nil) + scope = quoted_scope(name, type: type) + scope[:type] ||= "'r','v','m','p','f'" # (r)elation/table, (v)iew, (m)aterialized view, (p)artitioned table, (f)oreign table + + sql = "SELECT c.relname FROM pg_class c LEFT JOIN pg_namespace n ON n.oid = c.relnamespace".dup + sql << " WHERE n.nspname = #{scope[:schema]}" + sql << " AND c.relname = #{scope[:name]}" if scope[:name] + sql << " AND c.relkind IN (#{scope[:type]})" + sql + end + + def quoted_scope(name = nil, type: nil) + schema, name = extract_schema_qualified_name(name) + type = \ + case type + when "BASE TABLE" + "'r','p'" + when "VIEW" + "'v','m'" + when "FOREIGN TABLE" + "'f'" + end + scope = {} + scope[:schema] = schema ? quote(schema) : "ANY (current_schemas(false))" + scope[:name] = quote(name) if name + scope[:type] = type if type + scope + end + + def extract_schema_qualified_name(string) + name = Utils.extract_schema_qualified_name(string.to_s) + [name.schema, name.identifier] + end end end end diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/type_metadata.rb b/activerecord/lib/active_record/connection_adapters/postgresql/type_metadata.rb index 311988625f..ffd3be26b0 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql/type_metadata.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql/type_metadata.rb @@ -1,6 +1,11 @@ +# frozen_string_literal: true + module ActiveRecord + # :stopdoc: module ConnectionAdapters class PostgreSQLTypeMetadata < DelegateClass(SqlTypeMetadata) + undef to_yaml if method_defined?(:to_yaml) + attr_reader :oid, :fmod, :array def initialize(type_metadata, oid: nil, fmod: nil) diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/utils.rb b/activerecord/lib/active_record/connection_adapters/postgresql/utils.rb index a3f9ce6d64..bfd300723d 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql/utils.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql/utils.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module ActiveRecord module ConnectionAdapters module PostgreSQL @@ -19,9 +21,9 @@ module ActiveRecord def quoted if schema - PGconn.quote_ident(schema) << SEPARATOR << PGconn.quote_ident(identifier) + PG::Connection.quote_ident(schema) << SEPARATOR << PG::Connection.quote_ident(identifier) else - PGconn.quote_ident(identifier) + PG::Connection.quote_ident(identifier) end end diff --git a/activerecord/lib/active_record/connection_adapters/postgresql_adapter.rb b/activerecord/lib/active_record/connection_adapters/postgresql_adapter.rb index bc04565434..3ee344a249 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql_adapter.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql_adapter.rb @@ -1,8 +1,11 @@ +# frozen_string_literal: true + # Make sure we're using pg high enough for type casts and Ruby 2.2+ compatibility -gem "pg", "~> 0.18" +gem "pg", ">= 0.18", "< 2.0" require "pg" require "active_record/connection_adapters/abstract_adapter" +require "active_record/connection_adapters/statement_pool" require "active_record/connection_adapters/postgresql/column" require "active_record/connection_adapters/postgresql/database_statements" require "active_record/connection_adapters/postgresql/explain_pretty_printer" @@ -15,7 +18,6 @@ require "active_record/connection_adapters/postgresql/schema_dumper" require "active_record/connection_adapters/postgresql/schema_statements" require "active_record/connection_adapters/postgresql/type_metadata" require "active_record/connection_adapters/postgresql/utils" -require "active_record/connection_adapters/statement_pool" module ActiveRecord module ConnectionHandling # :nodoc: @@ -29,11 +31,11 @@ module ActiveRecord conn_params[:user] = conn_params.delete(:username) if conn_params[:username] conn_params[:dbname] = conn_params.delete(:database) if conn_params[:database] - # Forward only valid config params to PGconn.connect. - valid_conn_param_keys = PGconn.conndefaults_hash.keys + [:requiressl] + # Forward only valid config params to PG::Connection.connect. + valid_conn_param_keys = PG::Connection.conndefaults_hash.keys + [:requiressl] conn_params.slice!(*valid_conn_param_keys) - # The postgres drivers don't allow the creation of an unconnected PGconn object, + # The postgres drivers don't allow the creation of an unconnected PG::Connection object, # so just pass a nil connection object for the time being. ConnectionAdapters::PostgreSQLAdapter.new(nil, logger, conn_params, config) end @@ -62,11 +64,11 @@ module ActiveRecord # defaults to true. # # Any further options are used as connection parameters to libpq. See - # http://www.postgresql.org/docs/current/static/libpq-connect.html for the + # https://www.postgresql.org/docs/current/static/libpq-connect.html for the # list of parameters. # # In addition, default connection parameters of libpq can be set per environment variables. - # See http://www.postgresql.org/docs/current/static/libpq-envars.html . + # See https://www.postgresql.org/docs/current/static/libpq-envars.html . class PostgreSQLAdapter < AbstractAdapter ADAPTER_NAME = "PostgreSQL".freeze @@ -74,7 +76,7 @@ module ActiveRecord primary_key: "bigserial primary key", string: { name: "character varying" }, text: { name: "text" }, - integer: { name: "integer" }, + integer: { name: "integer", limit: 4 }, float: { name: "float" }, decimal: { name: "decimal" }, datetime: { name: "timestamp" }, @@ -119,19 +121,8 @@ module ActiveRecord include PostgreSQL::ReferentialIntegrity include PostgreSQL::SchemaStatements include PostgreSQL::DatabaseStatements - include PostgreSQL::ColumnDumper - - def schema_creation # :nodoc: - PostgreSQL::SchemaCreation.new self - end - - def arel_visitor # :nodoc: - Arel::Visitors::PostgreSQL.new(self) - end - # Returns true, since this connection adapter supports prepared statement - # caching. - def supports_statement_cache? + def supports_bulk_alter? true end @@ -155,6 +146,10 @@ module ActiveRecord true end + def supports_validate_constraints? + true + end + def supports_views? true end @@ -179,7 +174,7 @@ module ActiveRecord { concurrently: "CONCURRENTLY" } end - class StatementPool < ConnectionAdapters::StatementPool + class StatementPool < ConnectionAdapters::StatementPool # :nodoc: def initialize(connection, max) super(max) @connection = connection @@ -195,14 +190,14 @@ module ActiveRecord end private - def dealloc(key) @connection.query "DEALLOCATE #{key}" if connection_active? + rescue PG::Error end def connection_active? - @connection.status == PGconn::CONNECTION_OK - rescue PGError + @connection.status == PG::CONNECTION_OK + rescue PG::Error false end end @@ -229,7 +224,7 @@ module ActiveRecord add_pg_decoders @type_map = Type::HashLookupTypeMap.new - initialize_type_map(type_map) + initialize_type_map @local_tz = execute("SHOW TIME ZONE", "SCHEMA").first["TimeZone"] @use_insert_returning = @config.key?(:insert_returning) ? self.class.type_cast_config_to_boolean(@config[:insert_returning]) : true end @@ -247,34 +242,47 @@ module ActiveRecord # Is this connection alive and ready for queries? def active? - @connection.query "SELECT 1" + @lock.synchronize do + @connection.query "SELECT 1" + end true - rescue PGError + rescue PG::Error false end # Close then reopen the connection. def reconnect! - super - @connection.reset - configure_connection + @lock.synchronize do + super + @connection.reset + configure_connection + end end def reset! - clear_cache! - reset_transaction - unless @connection.transaction_status == ::PG::PQTRANS_IDLE - @connection.query "ROLLBACK" + @lock.synchronize do + clear_cache! + reset_transaction + unless @connection.transaction_status == ::PG::PQTRANS_IDLE + @connection.query "ROLLBACK" + end + @connection.query "DISCARD ALL" + configure_connection end - @connection.query "DISCARD ALL" - configure_connection end # Disconnects from the database if already connected. Otherwise, this # method does nothing. def disconnect! - super - @connection.close rescue nil + @lock.synchronize do + super + @connection.close rescue nil + end + end + + def discard! # :nodoc: + @connection.socket_io.reopen(IO::NULL) rescue nil + @connection = nil end def native_database_types #:nodoc: @@ -301,8 +309,8 @@ module ActiveRecord true end - # Range datatypes weren't introduced until PostgreSQL 9.2 def supports_ranges? + # Range datatypes weren't introduced until PostgreSQL 9.2 postgresql_version >= 90200 end @@ -310,22 +318,30 @@ module ActiveRecord postgresql_version >= 90300 end + def supports_foreign_tables? + postgresql_version >= 90300 + end + def supports_pgcrypto_uuid? postgresql_version >= 90400 end + def supports_lazy_transactions? + true + end + def get_advisory_lock(lock_id) # :nodoc: unless lock_id.is_a?(Integer) && lock_id.bit_length <= 63 - raise(ArgumentError, "Postgres requires advisory lock ids to be a signed 64 bit integer") + raise(ArgumentError, "PostgreSQL requires advisory lock ids to be a signed 64 bit integer") end - select_value("SELECT pg_try_advisory_lock(#{lock_id});") + query_value("SELECT pg_try_advisory_lock(#{lock_id})") end def release_advisory_lock(lock_id) # :nodoc: unless lock_id.is_a?(Integer) && lock_id.bit_length <= 63 - raise(ArgumentError, "Postgres requires advisory lock ids to be a signed 64 bit integer") + raise(ArgumentError, "PostgreSQL requires advisory lock ids to be a signed 64 bit integer") end - select_value("SELECT pg_advisory_unlock(#{lock_id})") + query_value("SELECT pg_advisory_unlock(#{lock_id})") end def enable_extension(name) @@ -341,46 +357,31 @@ module ActiveRecord end def extension_enabled?(name) - if supports_extensions? - res = exec_query "SELECT EXISTS(SELECT * FROM pg_available_extensions WHERE name = '#{name}' AND installed_version IS NOT NULL) as enabled", - "SCHEMA" - res.cast_values.first - end + res = exec_query("SELECT EXISTS(SELECT * FROM pg_available_extensions WHERE name = '#{name}' AND installed_version IS NOT NULL) as enabled", "SCHEMA") + res.cast_values.first end def extensions - if supports_extensions? - exec_query("SELECT extname from pg_extension", "SCHEMA").cast_values - else - super - end + exec_query("SELECT extname FROM pg_extension", "SCHEMA").cast_values end # Returns the configured supported identifier length supported by PostgreSQL - def table_alias_length - @max_identifier_length ||= select_value("SHOW max_identifier_length", "SCHEMA").to_i + def max_identifier_length + @max_identifier_length ||= query_value("SHOW max_identifier_length", "SCHEMA").to_i end - alias index_name_length table_alias_length + alias table_alias_length max_identifier_length + alias index_name_length max_identifier_length # Set the authorized user for this session def session_auth=(user) clear_cache! - exec_query "SET SESSION AUTHORIZATION #{user}" + execute("SET SESSION AUTHORIZATION #{user}") end def use_insert_returning? @use_insert_returning end - def update_table_definition(table_name, base) #:nodoc: - PostgreSQL::Table.new(table_name, base) - end - - def lookup_cast_type(sql_type) # :nodoc: - oid = execute("SELECT #{quote(sql_type)}::regtype::oid", "SCHEMA").first["oid"].to_i - super(oid) - end - def column_name_for_operation(operation, node) # :nodoc: OPERATION_ALIASES.fetch(operation) { operation.downcase } end @@ -401,8 +402,7 @@ module ActiveRecord end private - - # See http://www.postgresql.org/docs/current/static/errcodes-appendix.html + # See https://www.postgresql.org/docs/current/static/errcodes-appendix.html VALUE_LIMIT_VIOLATION = "22001" NUMERIC_VALUE_OUT_OF_RANGE = "22003" NOT_NULL_VIOLATION = "23502" @@ -410,11 +410,13 @@ module ActiveRecord UNIQUE_VIOLATION = "23505" SERIALIZATION_FAILURE = "40001" DEADLOCK_DETECTED = "40P01" + LOCK_NOT_AVAILABLE = "55P03" + QUERY_CANCELED = "57014" def translate_exception(exception, message) return exception unless exception.respond_to?(:result) - case exception.result.try(:error_field, PGresult::PG_DIAG_SQLSTATE) + case exception.result.try(:error_field, PG::PG_DIAG_SQLSTATE) when UNIQUE_VIOLATION RecordNotUnique.new(message) when FOREIGN_KEY_VIOLATION @@ -429,6 +431,10 @@ module ActiveRecord SerializationFailure.new(message) when DEADLOCK_DETECTED Deadlocked.new(message) + when LOCK_NOT_AVAILABLE + LockWaitTimeout.new(message) + when QUERY_CANCELED + QueryCanceled.new(message) else super end @@ -436,7 +442,7 @@ module ActiveRecord def get_oid_type(oid, fmod, column_name, sql_type = "".freeze) if !type_map.key?(oid) - load_additional_types(type_map, [oid]) + load_additional_types([oid]) end type_map.fetch(oid, fmod, sql_type) { @@ -447,10 +453,10 @@ module ActiveRecord } end - def initialize_type_map(m) - register_class_with_limit m, "int2", Type::Integer - register_class_with_limit m, "int4", Type::Integer - register_class_with_limit m, "int8", Type::Integer + def initialize_type_map(m = type_map) + m.register_type "int2", Type::Integer.new(limit: 2) + m.register_type "int4", Type::Integer.new(limit: 4) + m.register_type "int8", Type::Integer.new(limit: 8) m.register_type "oid", OID::Oid.new m.register_type "float4", Type::Float.new m.alias_type "float8", "float4" @@ -463,13 +469,13 @@ module ActiveRecord register_class_with_limit m, "bit", OID::Bit register_class_with_limit m, "varbit", OID::BitVarying m.alias_type "timestamptz", "timestamp" - m.register_type "date", Type::Date.new + m.register_type "date", OID::Date.new m.register_type "money", OID::Money.new m.register_type "bytea", OID::Bytea.new m.register_type "point", OID::Point.new m.register_type "hstore", OID::Hstore.new - m.register_type "json", OID::Json.new + m.register_type "json", Type::Json.new m.register_type "jsonb", OID::Jsonb.new m.register_type "cidr", OID::Cidr.new m.register_type "inet", OID::Inet.new @@ -514,18 +520,7 @@ module ActiveRecord end end - load_additional_types(m) - end - - def extract_limit(sql_type) - case sql_type - when /^bigint/i, /^int8/i - 8 - when /^smallint/i - 2 - else - super - end + load_additional_types end # Extracts the value from a PostgreSQL column default definition. @@ -560,10 +555,10 @@ module ActiveRecord end def has_default_function?(default_value, default) - !default_value && (%r{\w+\(.*\)|\(.*\)::\w+} === default) + !default_value && %r{\w+\(.*\)|\(.*\)::\w+|CURRENT_DATE|CURRENT_TIMESTAMP}.match?(default) end - def load_additional_types(type_map, oids = nil) + def load_additional_types(oids = nil) initializer = OID::TypeMapInitializer.new(type_map) if supports_ranges? @@ -582,7 +577,7 @@ module ActiveRecord if oids query += "WHERE t.oid::integer IN (%s)" % oids.join(", ") else - query += initializer.query_conditions_for_initial_load(type_map) + query += initializer.query_conditions_for_initial_load end execute_and_clear(query, "SCHEMA", []) do |records| @@ -606,6 +601,8 @@ module ActiveRecord end def exec_no_cache(sql, name, binds) + materialize_transactions + type_casted_binds = type_casted_binds(binds) log(sql, name, binds, type_casted_binds) do ActiveSupport::Dependencies.interlock.permit_concurrent_loads do @@ -615,6 +612,8 @@ module ActiveRecord end def exec_cache(sql, name, binds) + materialize_transactions + stmt_key = prepare_statement(sql) type_casted_binds = type_casted_binds(binds) @@ -647,11 +646,11 @@ module ActiveRecord # ActiveRecord::PreparedStatementCacheExpired # # Check here for more details: - # http://git.postgresql.org/gitweb/?p=postgresql.git;a=blob;f=src/backend/utils/cache/plancache.c#l573 + # https://git.postgresql.org/gitweb/?p=postgresql.git;a=blob;f=src/backend/utils/cache/plancache.c#l573 CACHED_PLAN_HEURISTIC = "cached plan must not change result type".freeze def is_cached_plan_failure?(e) pgerror = e.cause - code = pgerror.result.result_error_field(PGresult::PG_DIAG_SQLSTATE) + code = pgerror.result.result_error_field(PG::PG_DIAG_SQLSTATE) code == FEATURE_NOT_SUPPORTED && pgerror.message.include?(CACHED_PLAN_HEURISTIC) rescue false @@ -690,7 +689,7 @@ module ActiveRecord # Connects to a PostgreSQL server and sets up the adapter depending on the # connected server's characteristics. def connect - @connection = PGconn.connect(@connection_parameters) + @connection = PG.connect(@connection_parameters) configure_connection rescue ::PG::Error => error if error.message.include?("does not exist") @@ -712,18 +711,20 @@ module ActiveRecord # Use standard-conforming strings so we don't have to do the E'...' dance. set_standard_conforming_strings + variables = @config.fetch(:variables, {}).stringify_keys + # If using Active Record's time zone support configure the connection to return # TIMESTAMP WITH ZONE types in UTC. - # (SET TIME ZONE does not use an equals sign like other SET variables) - if ActiveRecord::Base.default_timezone == :utc - execute("SET time zone 'UTC'", "SCHEMA") - elsif @local_tz - execute("SET time zone '#{@local_tz}'", "SCHEMA") + unless variables["timezone"] + if ActiveRecord::Base.default_timezone == :utc + variables["timezone"] = "UTC" + elsif @local_tz + variables["timezone"] = @local_tz + end end # SET statements from :variables config hash - # http://www.postgresql.org/docs/current/static/sql-set.html - variables = @config[:variables] || {} + # https://www.postgresql.org/docs/current/static/sql-set.html variables.map do |k, v| if v == ":default" || v == :default # Sets the value to the global or compile default @@ -734,11 +735,6 @@ module ActiveRecord end end - # Returns the current ID of a table's sequence. - def last_insert_id_result(sequence_name) - exec_query("SELECT currval('#{sequence_name}')", "SQL") - end - # Returns the list of a table's column names, data types, and default values. # # The underlying query is roughly: @@ -777,8 +773,8 @@ module ActiveRecord $1.strip if $1 end - def create_table_definition(*args) - PostgreSQL::TableDefinition.new(*args) + def arel_visitor + Arel::Visitors::PostgreSQL.new(self) end def can_perform_case_insensitive_comparison_for?(column) @@ -849,12 +845,12 @@ module ActiveRecord ActiveRecord::Type.register(:bit_varying, OID::BitVarying, adapter: :postgresql) ActiveRecord::Type.register(:binary, OID::Bytea, adapter: :postgresql) ActiveRecord::Type.register(:cidr, OID::Cidr, adapter: :postgresql) + ActiveRecord::Type.register(:date, OID::Date, adapter: :postgresql) ActiveRecord::Type.register(:datetime, OID::DateTime, adapter: :postgresql) ActiveRecord::Type.register(:decimal, OID::Decimal, adapter: :postgresql) ActiveRecord::Type.register(:enum, OID::Enum, adapter: :postgresql) ActiveRecord::Type.register(:hstore, OID::Hstore, adapter: :postgresql) ActiveRecord::Type.register(:inet, OID::Inet, adapter: :postgresql) - ActiveRecord::Type.register(:json, OID::Json, adapter: :postgresql) ActiveRecord::Type.register(:jsonb, OID::Jsonb, adapter: :postgresql) ActiveRecord::Type.register(:money, OID::Money, adapter: :postgresql) ActiveRecord::Type.register(:point, OID::Point, adapter: :postgresql) diff --git a/activerecord/lib/active_record/connection_adapters/schema_cache.rb b/activerecord/lib/active_record/connection_adapters/schema_cache.rb index 4d339b0a8c..c29cf1f9a1 100644 --- a/activerecord/lib/active_record/connection_adapters/schema_cache.rb +++ b/activerecord/lib/active_record/connection_adapters/schema_cache.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module ActiveRecord module ConnectionAdapters class SchemaCache @@ -26,7 +28,7 @@ module ActiveRecord coder["columns_hash"] = @columns_hash coder["primary_keys"] = @primary_keys coder["data_sources"] = @data_sources - coder["version"] = ActiveRecord::Migrator.current_version + coder["version"] = connection.migration_context.current_version end def init_with(coder) @@ -98,7 +100,7 @@ module ActiveRecord def marshal_dump # if we get current version during initialization, it happens stack over flow. - @version = ActiveRecord::Migrator.current_version + @version = connection.migration_context.current_version [@version, @columns, @columns_hash, @primary_keys, @data_sources] end diff --git a/activerecord/lib/active_record/connection_adapters/sql_type_metadata.rb b/activerecord/lib/active_record/connection_adapters/sql_type_metadata.rb index 9e12ae0de8..8489bcbf1d 100644 --- a/activerecord/lib/active_record/connection_adapters/sql_type_metadata.rb +++ b/activerecord/lib/active_record/connection_adapters/sql_type_metadata.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module ActiveRecord # :stopdoc: module ConnectionAdapters diff --git a/activerecord/lib/active_record/connection_adapters/sqlite3/explain_pretty_printer.rb b/activerecord/lib/active_record/connection_adapters/sqlite3/explain_pretty_printer.rb index 6fe3e1211e..832fdfe5c4 100644 --- a/activerecord/lib/active_record/connection_adapters/sqlite3/explain_pretty_printer.rb +++ b/activerecord/lib/active_record/connection_adapters/sqlite3/explain_pretty_printer.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module ActiveRecord module ConnectionAdapters module SQLite3 diff --git a/activerecord/lib/active_record/connection_adapters/sqlite3/quoting.rb b/activerecord/lib/active_record/connection_adapters/sqlite3/quoting.rb index 7276a65098..abedf01f10 100644 --- a/activerecord/lib/active_record/connection_adapters/sqlite3/quoting.rb +++ b/activerecord/lib/active_record/connection_adapters/sqlite3/quoting.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module ActiveRecord module ConnectionAdapters module SQLite3 @@ -15,13 +17,30 @@ module ActiveRecord end def quoted_time(value) - quoted_date(value) + value = value.change(year: 2000, month: 1, day: 1) + quoted_date(value).sub(/\A\d\d\d\d-\d\d-\d\d /, "2000-01-01 ") end def quoted_binary(value) "x'#{value.hex}'" end + def quoted_true + ActiveRecord::ConnectionAdapters::SQLite3Adapter.represent_boolean_as_integer ? "1".freeze : "'t'".freeze + end + + def unquoted_true + ActiveRecord::ConnectionAdapters::SQLite3Adapter.represent_boolean_as_integer ? 1 : "t".freeze + end + + def quoted_false + ActiveRecord::ConnectionAdapters::SQLite3Adapter.represent_boolean_as_integer ? "0".freeze : "'f'".freeze + end + + def unquoted_false + ActiveRecord::ConnectionAdapters::SQLite3Adapter.represent_boolean_as_integer ? 0 : "f".freeze + end + private def _type_cast(value) diff --git a/activerecord/lib/active_record/connection_adapters/sqlite3/schema_creation.rb b/activerecord/lib/active_record/connection_adapters/sqlite3/schema_creation.rb index bc798d1dbb..b842561317 100644 --- a/activerecord/lib/active_record/connection_adapters/sqlite3/schema_creation.rb +++ b/activerecord/lib/active_record/connection_adapters/sqlite3/schema_creation.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module ActiveRecord module ConnectionAdapters module SQLite3 diff --git a/activerecord/lib/active_record/connection_adapters/sqlite3/schema_definitions.rb b/activerecord/lib/active_record/connection_adapters/sqlite3/schema_definitions.rb index e157e4b218..c9855019c1 100644 --- a/activerecord/lib/active_record/connection_adapters/sqlite3/schema_definitions.rb +++ b/activerecord/lib/active_record/connection_adapters/sqlite3/schema_definitions.rb @@ -1,27 +1,18 @@ +# frozen_string_literal: true + module ActiveRecord module ConnectionAdapters module SQLite3 - module ColumnMethods - def primary_key(name, type = :primary_key, **options) - if %i(integer bigint).include?(type) && (options.delete(:auto_increment) == true || !options.key?(:default)) - type = :primary_key - end - - super - end - end - class TableDefinition < ActiveRecord::ConnectionAdapters::TableDefinition - include ColumnMethods - def references(*args, **options) super(*args, type: :integer, **options) end alias :belongs_to :references - end - class Table < ActiveRecord::ConnectionAdapters::Table - include ColumnMethods + private + def integer_like_primary_key_type(type, options) + :primary_key + end end end end diff --git a/activerecord/lib/active_record/connection_adapters/sqlite3/schema_dumper.rb b/activerecord/lib/active_record/connection_adapters/sqlite3/schema_dumper.rb index eec018eda3..621678ec65 100644 --- a/activerecord/lib/active_record/connection_adapters/sqlite3/schema_dumper.rb +++ b/activerecord/lib/active_record/connection_adapters/sqlite3/schema_dumper.rb @@ -1,9 +1,10 @@ +# frozen_string_literal: true + module ActiveRecord module ConnectionAdapters module SQLite3 - module ColumnDumper # :nodoc: + class SchemaDumper < ConnectionAdapters::SchemaDumper # :nodoc: private - def default_primary_key?(column) schema_type(column) == :integer end diff --git a/activerecord/lib/active_record/connection_adapters/sqlite3/schema_statements.rb b/activerecord/lib/active_record/connection_adapters/sqlite3/schema_statements.rb new file mode 100644 index 0000000000..24e7bc65fa --- /dev/null +++ b/activerecord/lib/active_record/connection_adapters/sqlite3/schema_statements.rb @@ -0,0 +1,106 @@ +# frozen_string_literal: true + +module ActiveRecord + module ConnectionAdapters + module SQLite3 + module SchemaStatements # :nodoc: + # Returns an array of indexes for the given table. + def indexes(table_name) + exec_query("PRAGMA index_list(#{quote_table_name(table_name)})", "SCHEMA").map do |row| + # Indexes SQLite creates implicitly for internal use start with "sqlite_". + # See https://www.sqlite.org/fileformat2.html#intschema + next if row["name"].starts_with?("sqlite_") + + index_sql = query_value(<<-SQL, "SCHEMA") + SELECT sql + FROM sqlite_master + WHERE name = #{quote(row['name'])} AND type = 'index' + UNION ALL + SELECT sql + FROM sqlite_temp_master + WHERE name = #{quote(row['name'])} AND type = 'index' + SQL + + /\sWHERE\s+(?<where>.+)$/i =~ index_sql + + columns = exec_query("PRAGMA index_info(#{quote(row['name'])})", "SCHEMA").map do |col| + col["name"] + end + + # Add info on sort order for columns (only desc order is explicitly specified, asc is + # the default) + orders = {} + if index_sql # index_sql can be null in case of primary key indexes + index_sql.scan(/"(\w+)" DESC/).flatten.each { |order_column| + orders[order_column] = :desc + } + end + + IndexDefinition.new( + table_name, + row["name"], + row["unique"] != 0, + columns, + where: where, + orders: orders + ) + end.compact + end + + def create_schema_dumper(options) + SQLite3::SchemaDumper.create(self, options) + end + + private + def schema_creation + SQLite3::SchemaCreation.new(self) + end + + def create_table_definition(*args) + SQLite3::TableDefinition.new(*args) + end + + def new_column_from_field(table_name, field) + default = \ + case field["dflt_value"] + when /^null$/i + nil + when /^'(.*)'$/m + $1.gsub("''", "'") + when /^"(.*)"$/m + $1.gsub('""', '"') + else + field["dflt_value"] + end + + type_metadata = fetch_type_metadata(field["type"]) + Column.new(field["name"], default, type_metadata, field["notnull"].to_i == 0, table_name, nil, field["collation"]) + end + + def data_source_sql(name = nil, type: nil) + scope = quoted_scope(name, type: type) + scope[:type] ||= "'table','view'" + + sql = "SELECT name FROM sqlite_master WHERE name <> 'sqlite_sequence'".dup + sql << " AND name = #{scope[:name]}" if scope[:name] + sql << " AND type IN (#{scope[:type]})" + sql + end + + def quoted_scope(name = nil, type: nil) + type = \ + case type + when "BASE TABLE" + "'table'" + when "VIEW" + "'view'" + end + scope = {} + scope[:name] = quote(name) if name + scope[:type] = type if type + scope + end + end + end + end +end diff --git a/activerecord/lib/active_record/connection_adapters/sqlite3_adapter.rb b/activerecord/lib/active_record/connection_adapters/sqlite3_adapter.rb index 285b0ec243..efe454fa7f 100644 --- a/activerecord/lib/active_record/connection_adapters/sqlite3_adapter.rb +++ b/activerecord/lib/active_record/connection_adapters/sqlite3_adapter.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + require "active_record/connection_adapters/abstract_adapter" require "active_record/connection_adapters/statement_pool" require "active_record/connection_adapters/sqlite3/explain_pretty_printer" @@ -5,6 +7,7 @@ require "active_record/connection_adapters/sqlite3/quoting" require "active_record/connection_adapters/sqlite3/schema_creation" require "active_record/connection_adapters/sqlite3/schema_definitions" require "active_record/connection_adapters/sqlite3/schema_dumper" +require "active_record/connection_adapters/sqlite3/schema_statements" gem "sqlite3", "~> 1.3.6" require "sqlite3" @@ -12,6 +15,8 @@ require "sqlite3" module ActiveRecord module ConnectionHandling # :nodoc: def sqlite3_connection(config) + config = config.symbolize_keys + # Require database. unless config[:database] raise ArgumentError, "No database file specified. Missing argument: database" @@ -28,7 +33,7 @@ module ActiveRecord db = SQLite3::Database.new( config[:database].to_s, - results_as_hash: true + config.merge(results_as_hash: true) ) db.busy_timeout(ConnectionAdapters::SQLite3Adapter.type_cast_config_to_integer(config[:timeout])) if config[:timeout] @@ -54,10 +59,10 @@ module ActiveRecord ADAPTER_NAME = "SQLite".freeze include SQLite3::Quoting - include SQLite3::ColumnDumper + include SQLite3::SchemaStatements NATIVE_DATABASE_TYPES = { - primary_key: "INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL", + primary_key: "integer PRIMARY KEY AUTOINCREMENT NOT NULL", string: { name: "varchar" }, text: { name: "text" }, integer: { name: "integer" }, @@ -67,35 +72,44 @@ module ActiveRecord time: { name: "time" }, date: { name: "date" }, binary: { name: "blob" }, - boolean: { name: "boolean" } + boolean: { name: "boolean" }, + json: { name: "json" }, } - class StatementPool < ConnectionAdapters::StatementPool - private + ## + # :singleton-method: + # Indicates whether boolean values are stored in sqlite3 databases as 1 + # and 0 or 't' and 'f'. Leaving <tt>ActiveRecord::ConnectionAdapters::SQLite3Adapter.represent_boolean_as_integer</tt> + # set to false is deprecated. SQLite databases have used 't' and 'f' to + # serialize boolean values and must have old data converted to 1 and 0 + # (its native boolean serialization) before setting this flag to true. + # Conversion can be accomplished by setting up a rake task which runs + # + # ExampleModel.where("boolean_column = 't'").update_all(boolean_column: 1) + # ExampleModel.where("boolean_column = 'f'").update_all(boolean_column: 0) + # for all models and all boolean columns, after which the flag must be set + # to true by adding the following to your <tt>application.rb</tt> file: + # + # Rails.application.config.active_record.sqlite3.represent_boolean_as_integer = true + class_attribute :represent_boolean_as_integer, default: false + class StatementPool < ConnectionAdapters::StatementPool # :nodoc: + private def dealloc(stmt) - stmt[:stmt].close unless stmt[:stmt].closed? + stmt.close unless stmt.closed? end end - def update_table_definition(table_name, base) # :nodoc: - SQLite3::Table.new(table_name, base) - end - - def schema_creation # :nodoc: - SQLite3::SchemaCreation.new self - end - - def arel_visitor # :nodoc: - Arel::Visitors::SQLite.new(self) - end - def initialize(connection, logger, connection_options, config) super(connection, logger, config) - @active = nil + @active = true @statements = StatementPool.new(self.class.type_cast_config_to_integer(config[:statement_limit])) + if sqlite_version < "3.8.0" + raise "Your version of SQLite (#{sqlite_version}) is too old. Active Record supports SQLite >= 3.8." + end + configure_connection end @@ -108,12 +122,6 @@ module ActiveRecord end def supports_partial_index? - sqlite_version >= "3.8.0" - end - - # Returns true, since this connection adapter supports prepared statement - # caching. - def supports_statement_cache? true end @@ -122,7 +130,7 @@ module ActiveRecord end def supports_foreign_keys_in_create? - sqlite_version >= "3.6.19" + true end def supports_views? @@ -133,12 +141,12 @@ module ActiveRecord true end - def supports_multi_insert? - sqlite_version >= "3.7.11" + def supports_json? + true end def active? - @active != false + @active end # Disconnects from the database if already connected. Otherwise, this @@ -158,10 +166,6 @@ module ActiveRecord true end - def valid_type?(type) # :nodoc: - true - end - # Returns 62. SQLite supports index names up to 64 # characters. The rest is used by Rails internally to perform # temporary rename operations @@ -182,16 +186,23 @@ module ActiveRecord true end + def supports_lazy_transactions? + true + end + # REFERENTIAL INTEGRITY ==================================== def disable_referential_integrity # :nodoc: - old = select_value("PRAGMA foreign_keys") + old_foreign_keys = query_value("PRAGMA foreign_keys") + old_defer_foreign_keys = query_value("PRAGMA defer_foreign_keys") begin + execute("PRAGMA defer_foreign_keys = ON") execute("PRAGMA foreign_keys = OFF") yield ensure - execute("PRAGMA foreign_keys = #{old}") + execute("PRAGMA defer_foreign_keys = #{old_defer_foreign_keys}") + execute("PRAGMA foreign_keys = #{old_foreign_keys}") end end @@ -205,6 +216,8 @@ module ActiveRecord end def exec_query(sql, name = nil, binds = [], prepare: false) + materialize_transactions + type_casted_binds = type_casted_binds(binds) log(sql, name, binds, type_casted_binds) do @@ -222,11 +235,8 @@ module ActiveRecord stmt.close end else - cache = @statements[sql] ||= { - stmt: @connection.prepare(sql) - } - stmt = cache[:stmt] - cols = cache[:cols] ||= stmt.columns + stmt = @statements[sql] ||= @connection.prepare(sql) + cols = stmt.columns stmt.reset! stmt.bind_params(type_casted_binds) records = stmt.to_a @@ -248,6 +258,8 @@ module ActiveRecord end def execute(sql, name = nil) #:nodoc: + materialize_transactions + log(sql, name) do ActiveSupport::Dependencies.interlock.permit_concurrent_loads do @connection.execute(sql) @@ -269,92 +281,6 @@ module ActiveRecord # SCHEMA STATEMENTS ======================================== - def tables # :nodoc: - select_values("SELECT name FROM sqlite_master WHERE type = 'table' AND name <> 'sqlite_sequence'", "SCHEMA") - end - - def data_sources # :nodoc: - select_values("SELECT name FROM sqlite_master WHERE type IN ('table','view') AND name <> 'sqlite_sequence'", "SCHEMA") - end - - def views # :nodoc: - select_values("SELECT name FROM sqlite_master WHERE type = 'view' AND name <> 'sqlite_sequence'", "SCHEMA") - end - - def table_exists?(table_name) # :nodoc: - return false unless table_name.present? - - sql = "SELECT name FROM sqlite_master WHERE type = 'table' AND name <> 'sqlite_sequence'" - sql << " AND name = #{quote(table_name)}" - - select_values(sql, "SCHEMA").any? - end - - def data_source_exists?(table_name) # :nodoc: - return false unless table_name.present? - - sql = "SELECT name FROM sqlite_master WHERE type IN ('table','view') AND name <> 'sqlite_sequence'" - sql << " AND name = #{quote(table_name)}" - - select_values(sql, "SCHEMA").any? - end - - def view_exists?(view_name) # :nodoc: - return false unless view_name.present? - - sql = "SELECT name FROM sqlite_master WHERE type = 'view' AND name <> 'sqlite_sequence'" - sql << " AND name = #{quote(view_name)}" - - select_values(sql, "SCHEMA").any? - end - - def new_column_from_field(table_name, field) # :nondoc: - case field["dflt_value"] - when /^null$/i - field["dflt_value"] = nil - when /^'(.*)'$/m - field["dflt_value"] = $1.gsub("''", "'") - when /^"(.*)"$/m - field["dflt_value"] = $1.gsub('""', '"') - end - - collation = field["collation"] - sql_type = field["type"] - type_metadata = fetch_type_metadata(sql_type) - new_column(field["name"], field["dflt_value"], type_metadata, field["notnull"].to_i == 0, table_name, nil, collation) - end - - # Returns an array of indexes for the given table. - def indexes(table_name, name = nil) #:nodoc: - if name - ActiveSupport::Deprecation.warn(<<-MSG.squish) - Passing name to #indexes is deprecated without replacement. - MSG - end - - exec_query("PRAGMA index_list(#{quote_table_name(table_name)})", "SCHEMA").map do |row| - sql = <<-SQL - SELECT sql - FROM sqlite_master - WHERE name=#{quote(row['name'])} AND type='index' - UNION ALL - SELECT sql - FROM sqlite_temp_master - WHERE name=#{quote(row['name'])} AND type='index' - SQL - index_sql = exec_query(sql).first["sql"] - match = /\sWHERE\s+(.+)$/i.match(index_sql) - where = match[1] if match - IndexDefinition.new( - table_name, - row["name"], - row["unique"] != 0, - exec_query("PRAGMA index_info('#{row['name']}')", "SCHEMA").map { |col| - col["name"] - }, nil, nil, where) - end - end - def primary_keys(table_name) # :nodoc: pks = table_structure(table_name).select { |f| f["pk"] > 0 } pks.sort_by { |f| f["pk"] }.map { |f| f["name"] } @@ -374,19 +300,18 @@ module ActiveRecord rename_table_indexes(table_name, new_name) end - # See: http://www.sqlite.org/lang_altertable.html - # SQLite has an additional restriction on the ALTER TABLE statement - def valid_alter_table_type?(type) - type.to_sym != :primary_key + def valid_alter_table_type?(type, options = {}) + !invalid_alter_table_type?(type, options) end + deprecate :valid_alter_table_type? def add_column(table_name, column_name, type, options = {}) #:nodoc: - if valid_alter_table_type?(type) - super(table_name, column_name, type, options) - else + if invalid_alter_table_type?(type, options) alter_table(table_name) do |definition| definition.column(column_name, type, options) end + else + super end end @@ -439,7 +364,7 @@ module ActiveRecord alias :add_belongs_to :add_reference def foreign_keys(table_name) - fk_info = select_all("PRAGMA foreign_key_list(#{quote(table_name)})", "SCHEMA") + fk_info = exec_query("PRAGMA foreign_key_list(#{quote(table_name)})", "SCHEMA") fk_info.map do |row| options = { column: row["from"], @@ -451,7 +376,31 @@ module ActiveRecord end end + def insert_fixtures(rows, table_name) + ActiveSupport::Deprecation.warn(<<-MSG.squish) + `insert_fixtures` is deprecated and will be removed in the next version of Rails. + Consider using `insert_fixtures_set` for performance improvement. + MSG + insert_fixtures_set(table_name => rows) + end + + def insert_fixtures_set(fixture_set, tables_to_delete = []) + disable_referential_integrity do + transaction(requires_new: true) do + tables_to_delete.each { |table| delete "DELETE FROM #{quote_table_name(table)}", "Fixture Delete" } + + fixture_set.each do |table_name, rows| + rows.each { |row| insert_fixture(row, table_name) } + end + end + end + end + private + def initialize_type_map(m = type_map) + super + register_class_with_limit m, %r(int)i, SQLite3Integer + end def table_structure(table_name) structure = exec_query("PRAGMA table_info(#{quote_table_name(table_name)})", "SCHEMA") @@ -460,14 +409,33 @@ module ActiveRecord end alias column_definitions table_structure + # See: https://www.sqlite.org/lang_altertable.html + # SQLite has an additional restriction on the ALTER TABLE statement + def invalid_alter_table_type?(type, options) + type.to_sym == :primary_key || options[:primary_key] + end + def alter_table(table_name, options = {}) altered_table_name = "a#{table_name}" - caller = lambda { |definition| yield definition if block_given? } + foreign_keys = foreign_keys(table_name) + + caller = lambda do |definition| + rename = options[:rename] || {} + foreign_keys.each do |fk| + if column = rename[fk.options[:column]] + fk.options[:column] = column + end + definition.foreign_key(fk.to_table, fk.options) + end + + yield definition if block_given? + end transaction do - move_table(table_name, altered_table_name, - options.merge(temporary: true)) - move_table(altered_table_name, table_name, &caller) + disable_referential_integrity do + move_table(table_name, altered_table_name, options.merge(temporary: true)) + move_table(altered_table_name, table_name, &caller) + end end end @@ -481,19 +449,23 @@ module ActiveRecord options[:id] = false create_table(to, options) do |definition| @definition = definition - @definition.primary_key(from_primary_key) if from_primary_key.present? + if from_primary_key.is_a?(Array) + @definition.primary_keys from_primary_key + end columns(from).each do |column| column_name = options[:rename] ? (options[:rename][column.name] || options[:rename][column.name.to_sym] || column.name) : column.name - next if column_name == from_primary_key @definition.column(column_name, column.type, limit: column.limit, default: column.default, precision: column.precision, scale: column.scale, - null: column.null, collation: column.collation) + null: column.null, collation: column.collation, + primary_key: column_name == from_primary_key + ) end + yield @definition if block_given? end copy_table_indexes(from, to, options[:rename] || {}) @@ -520,6 +492,7 @@ module ActiveRecord # index name can't be the same opts = { name: name.gsub(/(^|_)(#{from})_/, "\\1#{to}_"), internal: true } opts[:unique] = true if index.unique + opts[:where] = index.where if index.where add_index(to, columns, opts) end end @@ -539,7 +512,7 @@ module ActiveRecord end def sqlite_version - @sqlite_version ||= SQLite3Adapter::Version.new(select_value("select sqlite_version(*)")) + @sqlite_version ||= SQLite3Adapter::Version.new(query_value("SELECT sqlite_version(*)")) end def translate_exception(exception, message) @@ -600,21 +573,25 @@ module ActiveRecord end end - def create_table_definition(*args) - SQLite3::TableDefinition.new(*args) - end - - def extract_foreign_key_action(specifier) - case specifier - when "CASCADE"; :cascade - when "SET NULL"; :nullify - when "RESTRICT"; :restrict - end + def arel_visitor + Arel::Visitors::SQLite.new(self) end def configure_connection execute("PRAGMA foreign_keys = ON", "SCHEMA") end + + class SQLite3Integer < Type::Integer # :nodoc: + private + def _limit + # INTEGER storage class can be stored 8 bytes value. + # See https://www.sqlite.org/datatype3.html#storage_classes_and_datatypes + limit || 8 + end + end + + ActiveRecord::Type.register(:integer, SQLite3Integer, adapter: :sqlite3) end + ActiveSupport.run_load_hooks(:active_record_sqlite3adapter, SQLite3Adapter) end end diff --git a/activerecord/lib/active_record/connection_adapters/statement_pool.rb b/activerecord/lib/active_record/connection_adapters/statement_pool.rb index 790db56185..46bd831da7 100644 --- a/activerecord/lib/active_record/connection_adapters/statement_pool.rb +++ b/activerecord/lib/active_record/connection_adapters/statement_pool.rb @@ -1,3 +1,5 @@ +# frozen_string_literal: true + module ActiveRecord module ConnectionAdapters class StatementPool # :nodoc: |