diff options
Diffstat (limited to 'activerecord/lib/active_record/connection_adapters')
52 files changed, 2018 insertions, 1091 deletions
diff --git a/activerecord/lib/active_record/connection_adapters/abstract/connection_pool.rb b/activerecord/lib/active_record/connection_adapters/abstract/connection_pool.rb index 99934a0e31..36001efdd5 100644 --- a/activerecord/lib/active_record/connection_adapters/abstract/connection_pool.rb +++ b/activerecord/lib/active_record/connection_adapters/abstract/connection_pool.rb @@ -3,6 +3,7 @@ require "thread" require "concurrent/map" require "monitor" +require "weakref" module ActiveRecord # Raised when a connection could not be obtained within the connection @@ -19,6 +20,26 @@ module ActiveRecord end module ConnectionAdapters + module AbstractPool # :nodoc: + def get_schema_cache(connection) + @schema_cache ||= SchemaCache.new(connection) + @schema_cache.connection = connection + @schema_cache + end + + def set_schema_cache(cache) + @schema_cache = cache + end + end + + class NullPool # :nodoc: + include ConnectionAdapters::AbstractPool + + def initialize + @schema_cache = nil + end + end + # Connection pool base class for managing Active Record database # connections. # @@ -146,7 +167,6 @@ module ActiveRecord end private - def internal_poll(timeout) no_wait_poll || (timeout && wait_poll(timeout)) end @@ -185,7 +205,7 @@ module ActiveRecord def wait_poll(timeout) @num_waiting += 1 - t0 = Time.now + t0 = Concurrent.monotonic_time elapsed = 0 loop do ActiveSupport::Dependencies.interlock.permit_concurrent_loads do @@ -194,7 +214,7 @@ module ActiveRecord return remove if any? - elapsed = Time.now - t0 + elapsed = Concurrent.monotonic_time - t0 if elapsed >= timeout msg = "could not obtain a connection from the pool within %0.3f seconds (waited %0.3f seconds); all pooled connections were in use" % [timeout, elapsed] @@ -294,23 +314,50 @@ module ActiveRecord @frequency = frequency end + @mutex = Mutex.new + @pools = {} + + class << self + def register_pool(pool, frequency) # :nodoc: + @mutex.synchronize do + unless @pools.key?(frequency) + @pools[frequency] = [] + spawn_thread(frequency) + end + @pools[frequency] << WeakRef.new(pool) + end + end + + private + def spawn_thread(frequency) + Thread.new(frequency) do |t| + loop do + sleep t + @mutex.synchronize do + @pools[frequency].select!(&:weakref_alive?) + @pools[frequency].each do |p| + p.reap + p.flush + rescue WeakRef::RefError + end + end + end + end + end + end + def run return unless frequency && frequency > 0 - Thread.new(frequency, pool) { |t, p| - loop do - sleep t - p.reap - p.flush - end - } + self.class.register_pool(pool, frequency) end end include MonitorMixin include QueryCache::ConnectionPoolConfiguration + include ConnectionAdapters::AbstractPool attr_accessor :automatic_reconnect, :checkout_timeout, :schema_cache - attr_reader :spec, :connections, :size, :reaper + attr_reader :spec, :size, :reaper # Creates a new ConnectionPool object. +spec+ is a ConnectionSpecification # object which describes database connection information (e.g. adapter, @@ -379,7 +426,7 @@ module ActiveRecord # #connection can be called any number of times; the connection is # held in a cache keyed by a thread. def connection - @thread_cached_conns[connection_cache_key(@lock_thread || Thread.current)] ||= checkout + @thread_cached_conns[connection_cache_key(current_thread)] ||= checkout end # Returns true if there is an open connection being used for the current thread. @@ -388,7 +435,7 @@ module ActiveRecord # #connection or #with_connection methods. Connections obtained through # #checkout will not be detected by #active_connection? def active_connection? - @thread_cached_conns[connection_cache_key(Thread.current)] + @thread_cached_conns[connection_cache_key(current_thread)] end # Signal that the thread is finished with the current connection. @@ -423,6 +470,21 @@ module ActiveRecord synchronize { @connections.any? } end + # Returns an array containing the connections currently in the pool. + # Access to the array does not require synchronization on the pool because + # the array is newly created and not retained by the pool. + # + # However; this method bypasses the ConnectionPool's thread-safe connection + # access pattern. A returned connection may be owned by another thread, + # unowned, or by happen-stance owned by the calling thread. + # + # Calling methods on a connection without ownership is subject to the + # thread-safety guarantees of the underlying method. Many of the methods + # on connection adapter classes are inherently multi-thread unsafe. + def connections + synchronize { @connections.dup } + end + # Disconnects all connections in the pool, and clears the pool. # # Raises: @@ -668,6 +730,10 @@ module ActiveRecord thread end + def current_thread + @lock_thread || Thread.current + end + # Take control of all existing connections so a "group" action such as # reload/disconnect can be performed safely. It is no longer enough to # wrap it in +synchronize+ because some pool's actions are allowed @@ -686,13 +752,13 @@ module ActiveRecord end newly_checked_out = [] - timeout_time = Time.now + (@checkout_timeout * 2) + timeout_time = Concurrent.monotonic_time + (@checkout_timeout * 2) @available.with_a_bias_for(Thread.current) do loop do synchronize do return if collected_conns.size == @connections.size && @now_connecting == 0 - remaining_timeout = timeout_time - Time.now + remaining_timeout = timeout_time - Concurrent.monotonic_time remaining_timeout = 0 if remaining_timeout < 0 conn = checkout_for_exclusive_access(remaining_timeout) collected_conns << conn @@ -809,7 +875,7 @@ module ActiveRecord def new_connection Base.send(spec.adapter_method, spec.config).tap do |conn| - conn.schema_cache = schema_cache.dup if schema_cache + conn.check_version end end @@ -915,6 +981,16 @@ module ActiveRecord # about the model. The model needs to pass a specification name to the handler, # in order to look up the correct connection pool. class ConnectionHandler + def self.create_owner_to_pool # :nodoc: + Concurrent::Map.new(initial_capacity: 2) do |h, k| + # Discard the parent's connection pools immediately; we have no need + # of them + discard_unowned_pools(h) + + h[k] = Concurrent::Map.new(initial_capacity: 2) + end + end + def self.unowned_pool_finalizer(pid_map) # :nodoc: lambda do |_| discard_unowned_pools(pid_map) @@ -927,21 +1003,30 @@ module ActiveRecord end end + attr_reader :prevent_writes + def initialize # These caches are keyed by spec.name (ConnectionSpecification#name). - @owner_to_pool = Concurrent::Map.new(initial_capacity: 2) do |h, k| - # Discard the parent's connection pools immediately; we have no need - # of them - ConnectionHandler.discard_unowned_pools(h) - - h[k] = Concurrent::Map.new(initial_capacity: 2) - end + @owner_to_pool = ConnectionHandler.create_owner_to_pool + @prevent_writes = false # Backup finalizer: if the forked child never needed a pool, the above # early discard has not occurred ObjectSpace.define_finalizer self, ConnectionHandler.unowned_pool_finalizer(@owner_to_pool) end + # Prevent writing to the database regardless of role. + # + # In some cases you may want to prevent writes to the database + # even if you are on a database that can write. `while_preventing_writes` + # will prevent writes to the database for the duration of the block. + def while_preventing_writes + original, @prevent_writes = @prevent_writes, true + yield + ensure + @prevent_writes = original + end + def connection_pool_list owner_to_pool.values.compact end @@ -1006,7 +1091,16 @@ module ActiveRecord # for (not necessarily the current class). def retrieve_connection(spec_name) #:nodoc: pool = retrieve_connection_pool(spec_name) - raise ConnectionNotEstablished, "No connection pool with '#{spec_name}' found." unless pool + + unless pool + # multiple database application + if ActiveRecord::Base.connection_handler != ActiveRecord::Base.default_connection_handler + raise ConnectionNotEstablished, "No connection pool with '#{spec_name}' found for the '#{ActiveRecord::Base.current_role}' role." + else + raise ConnectionNotEstablished, "No connection pool with '#{spec_name}' found." + end + end + pool.connection end @@ -1050,7 +1144,6 @@ module ActiveRecord end private - def owner_to_pool @owner_to_pool[Process.pid] end diff --git a/activerecord/lib/active_record/connection_adapters/abstract/database_limits.rb b/activerecord/lib/active_record/connection_adapters/abstract/database_limits.rb index 1305216be2..d932f068f2 100644 --- a/activerecord/lib/active_record/connection_adapters/abstract/database_limits.rb +++ b/activerecord/lib/active_record/connection_adapters/abstract/database_limits.rb @@ -1,24 +1,26 @@ # frozen_string_literal: true -require "active_support/deprecation" - module ActiveRecord module ConnectionAdapters # :nodoc: module DatabaseLimits + def max_identifier_length # :nodoc: + 64 + end + # Returns the maximum length of a table alias. def table_alias_length - 255 + max_identifier_length end # Returns the maximum length of a column name. def column_name_length - 64 + max_identifier_length end deprecate :column_name_length # Returns the maximum length of a table name. def table_name_length - 64 + max_identifier_length end deprecate :table_name_length @@ -33,7 +35,7 @@ module ActiveRecord # Returns the maximum length of an index name. def index_name_length - 64 + max_identifier_length end # Returns the maximum number of columns per table. diff --git a/activerecord/lib/active_record/connection_adapters/abstract/database_statements.rb b/activerecord/lib/active_record/connection_adapters/abstract/database_statements.rb index 0059f0b773..044272ea51 100644 --- a/activerecord/lib/active_record/connection_adapters/abstract/database_statements.rb +++ b/activerecord/lib/active_record/connection_adapters/abstract/database_statements.rb @@ -20,9 +20,22 @@ module ActiveRecord raise "Passing bind parameters with an arel AST is forbidden. " \ "The values must be stored on the AST directly" end - sql, binds = visitor.compile(arel_or_sql_string.ast, collector) - [sql.freeze, binds || []] + + if prepared_statements + sql, binds = visitor.compile(arel_or_sql_string.ast, collector) + + if binds.length > bind_params_length + unprepared_statement do + sql, binds = to_sql_and_binds(arel_or_sql_string) + visitor.preparable = false + end + end + else + sql = visitor.compile(arel_or_sql_string.ast, collector) + end + [sql.freeze, binds] else + visitor.preparable = false if prepared_statements [arel_or_sql_string.dup.freeze, binds] end end @@ -47,13 +60,8 @@ module ActiveRecord arel = arel_from_relation(arel) sql, binds = to_sql_and_binds(arel, binds) - if !prepared_statements || (arel.is_a?(String) && preparable.nil?) - preparable = false - elsif binds.length > bind_params_length - sql, binds = unprepared_statement { to_sql_and_binds(arel) } - preparable = false - else - preparable = visitor.preparable + if preparable.nil? + preparable = prepared_statements ? visitor.preparable : false end if prepared_statements && preparable @@ -98,6 +106,11 @@ module ActiveRecord exec_query(sql, name).rows end + # Determines whether the SQL statement is a write query. + def write_query?(sql) + raise NotImplementedError + end + # Executes the SQL statement in the context of this connection and returns # the raw result from the connection adapter. # Note: depending on your database connector, the result returned by this @@ -118,7 +131,7 @@ module ActiveRecord # +binds+ as the bind substitutes. +name+ is logged along with # the executed +sql+ statement. def exec_insert(sql, name = nil, binds = [], pk = nil, sequence_name = nil) - sql, binds = sql_for_insert(sql, pk, nil, sequence_name, binds) + sql, binds = sql_for_insert(sql, pk, binds) exec_query(sql, name, binds) end @@ -129,11 +142,6 @@ module ActiveRecord exec_query(sql, name, binds) end - # Executes the truncate statement. - def truncate(table_name, name = nil) - raise NotImplementedError - end - # Executes update +sql+ statement in the context of this connection using # +binds+ as the bind substitutes. +name+ is logged along with # the executed +sql+ statement. @@ -168,12 +176,22 @@ module ActiveRecord exec_delete(sql, name, binds) end - # Returns +true+ when the connection adapter supports prepared statement - # caching, otherwise returns +false+ - def supports_statement_cache? # :nodoc: - true + # Executes the truncate statement. + def truncate(table_name, name = nil) + execute(build_truncate_statements(table_name), name) + end + + def truncate_tables(*table_names) # :nodoc: + return if table_names.empty? + + with_multi_statements do + disable_referential_integrity do + Array(build_truncate_statements(*table_names)).each do |sql| + execute_batch(sql, "Truncate Tables") + end + end + end end - deprecate :supports_statement_cache? # Runs the given block in a database transaction, and returns the result # of the block. @@ -187,8 +205,6 @@ module ActiveRecord # In order to get around this problem, #transaction will emulate the effect # of nested transactions, by using savepoints: # https://dev.mysql.com/doc/refman/5.7/en/savepoint.html - # Savepoints are supported by MySQL and PostgreSQL. SQLite3 version >= '3.6.8' - # supports savepoints. # # It is safe to call this method if a database transaction is already open, # i.e. if #transaction is called within another #transaction block. In case @@ -331,62 +347,24 @@ module ActiveRecord # Inserts the given fixture into the table. Overridden in adapters that require # something beyond a simple insert (eg. Oracle). - # Most of adapters should implement `insert_fixtures` that leverages bulk SQL insert. + # Most of adapters should implement `insert_fixtures_set` that leverages bulk SQL insert. # We keep this method to provide fallback # for databases like sqlite that do not support bulk inserts. def insert_fixture(fixture, table_name) - fixture = fixture.stringify_keys - - columns = schema_cache.columns_hash(table_name) - binds = fixture.map do |name, value| - if column = columns[name] - type = lookup_cast_type_from_column(column) - Relation::QueryAttribute.new(name, value, type) - else - raise Fixture::FixtureError, %(table "#{table_name}" has no column named #{name.inspect}.) - end - end - - table = Arel::Table.new(table_name) - - values = binds.map do |bind| - value = with_yaml_fallback(bind.value_for_database) - [table[bind.name], value] - end - - manager = Arel::InsertManager.new - manager.into(table) - manager.insert(values) - execute manager.to_sql, "Fixture Insert" - end - - # Inserts a set of fixtures into the table. Overridden in adapters that require - # something beyond a simple insert (eg. Oracle). - def insert_fixtures(fixtures, table_name) - ActiveSupport::Deprecation.warn(<<-MSG.squish) - `insert_fixtures` is deprecated and will be removed in the next version of Rails. - Consider using `insert_fixtures_set` for performance improvement. - MSG - return if fixtures.empty? - - execute(build_fixture_sql(fixtures, table_name), "Fixtures Insert") + execute(build_fixture_sql(Array.wrap(fixture), table_name), "Fixture Insert") end def insert_fixtures_set(fixture_set, tables_to_delete = []) - fixture_inserts = fixture_set.map do |table_name, fixtures| - next if fixtures.empty? - - build_fixture_sql(fixtures, table_name) - end.compact - - table_deletes = tables_to_delete.map { |table| +"DELETE FROM #{quote_table_name table}" } - total_sql = Array.wrap(combine_multi_statements(table_deletes + fixture_inserts)) - - disable_referential_integrity do - transaction(requires_new: true) do - total_sql.each do |sql| - execute sql, "Fixtures Load" - yield if block_given? + fixture_inserts = build_fixture_statements(fixture_set) + table_deletes = tables_to_delete.map { |table| "DELETE FROM #{quote_table_name(table)}" } + total_sql = Array(combine_multi_statements(table_deletes + fixture_inserts)) + + with_multi_statements do + disable_referential_integrity do + transaction(requires_new: true) do + total_sql.each do |sql| + execute_batch(sql, "Fixtures Load") + end end end end @@ -410,15 +388,33 @@ module ActiveRecord end end + # Fixture value is quoted by Arel, however scalar values + # are not quotable. In this case we want to convert + # the column value to YAML. + def with_yaml_fallback(value) # :nodoc: + if value.is_a?(Hash) || value.is_a?(Array) + YAML.dump(value) + else + value + end + end + private + def execute_batch(sql, name = nil) + execute(sql, name) + end + + DEFAULT_INSERT_VALUE = Arel.sql("DEFAULT").freeze + private_constant :DEFAULT_INSERT_VALUE + def default_insert_value(column) - Arel.sql("DEFAULT") + DEFAULT_INSERT_VALUE end def build_fixture_sql(fixtures, table_name) columns = schema_cache.columns_hash(table_name) - values = fixtures.map do |fixture| + values_list = fixtures.map do |fixture| fixture = fixture.stringify_keys unknown_columns = fixture.keys - columns.keys @@ -429,8 +425,7 @@ module ActiveRecord columns.map do |name, column| if fixture.key?(name) type = lookup_cast_type_from_column(column) - bind = Relation::QueryAttribute.new(name, fixture[name], type) - with_yaml_fallback(bind.value_for_database) + with_yaml_fallback(type.serialize(fixture[name])) else default_insert_value(column) end @@ -440,12 +435,43 @@ module ActiveRecord table = Arel::Table.new(table_name) manager = Arel::InsertManager.new manager.into(table) - columns.each_key { |column| manager.columns << table[column] } - manager.values = manager.create_values_list(values) + if values_list.size == 1 + values = values_list.shift + new_values = [] + columns.each_key.with_index { |column, i| + unless values[i].equal?(DEFAULT_INSERT_VALUE) + new_values << values[i] + manager.columns << table[column] + end + } + values_list << new_values + else + columns.each_key { |column| manager.columns << table[column] } + end + + manager.values = manager.create_values_list(values_list) manager.to_sql end + def build_fixture_statements(fixture_set) + fixture_set.map do |table_name, fixtures| + next if fixtures.empty? + build_fixture_sql(fixtures, table_name) + end.compact + end + + def build_truncate_statements(*table_names) + truncate_tables = table_names.map do |table_name| + "TRUNCATE TABLE #{quote_table_name(table_name)}" + end + combine_multi_statements(truncate_tables) + end + + def with_multi_statements + yield + end + def combine_multi_statements(total_sql) total_sql.join(";\n") end @@ -459,7 +485,7 @@ module ActiveRecord exec_query(sql, name, binds, prepare: true) end - def sql_for_insert(sql, pk, id_value, sequence_name, binds) + def sql_for_insert(sql, pk, binds) [sql, binds] end @@ -479,17 +505,6 @@ module ActiveRecord relation end end - - # Fixture value is quoted by Arel, however scalar values - # are not quotable. In this case we want to convert - # the column value to YAML. - def with_yaml_fallback(value) - if value.is_a?(Hash) || value.is_a?(Array) - YAML.dump(value) - else - value - end - end end end end diff --git a/activerecord/lib/active_record/connection_adapters/abstract/query_cache.rb b/activerecord/lib/active_record/connection_adapters/abstract/query_cache.rb index 8aeb934ec2..768122b4d2 100644 --- a/activerecord/lib/active_record/connection_adapters/abstract/query_cache.rb +++ b/activerecord/lib/active_record/connection_adapters/abstract/query_cache.rb @@ -7,7 +7,8 @@ module ActiveRecord module QueryCache class << self def included(base) #:nodoc: - dirties_query_cache base, :insert, :update, :delete, :rollback_to_savepoint, :rollback_db_transaction + dirties_query_cache base, :insert, :update, :delete, :truncate, :truncate_tables, + :rollback_to_savepoint, :rollback_db_transaction base.set_callback :checkout, :after, :configure_query_cache! base.set_callback :checkin, :after, :disable_query_cache! @@ -17,7 +18,7 @@ module ActiveRecord method_names.each do |method_name| base.class_eval <<-end_code, __FILE__, __LINE__ + 1 def #{method_name}(*) - clear_query_cache if @query_cache_enabled + ActiveRecord::Base.clear_query_caches_for_current_thread if @query_cache_enabled super end end_code @@ -32,17 +33,17 @@ module ActiveRecord end def enable_query_cache! - @query_cache_enabled[connection_cache_key(Thread.current)] = true + @query_cache_enabled[connection_cache_key(current_thread)] = true connection.enable_query_cache! if active_connection? end def disable_query_cache! - @query_cache_enabled.delete connection_cache_key(Thread.current) + @query_cache_enabled.delete connection_cache_key(current_thread) connection.disable_query_cache! if active_connection? end def query_cache_enabled - @query_cache_enabled[connection_cache_key(Thread.current)] + @query_cache_enabled[connection_cache_key(current_thread)] end end @@ -96,6 +97,11 @@ module ActiveRecord if @query_cache_enabled && !locked?(arel) arel = arel_from_relation(arel) sql, binds = to_sql_and_binds(arel, binds) + + if preparable.nil? + preparable = prepared_statements ? visitor.preparable : false + end + cache_sql(sql, name, binds) { super(sql, name, binds, preparable: preparable) } else super @@ -103,7 +109,6 @@ module ActiveRecord end private - def cache_sql(sql, name, binds) @lock.synchronize do result = diff --git a/activerecord/lib/active_record/connection_adapters/abstract/quoting.rb b/activerecord/lib/active_record/connection_adapters/abstract/quoting.rb index 07e86afe9a..93273f6cf6 100644 --- a/activerecord/lib/active_record/connection_adapters/abstract/quoting.rb +++ b/activerecord/lib/active_record/connection_adapters/abstract/quoting.rb @@ -114,16 +114,16 @@ module ActiveRecord # if the value is a Time responding to usec. def quoted_date(value) if value.acts_like?(:time) - zone_conversion_method = ActiveRecord::Base.default_timezone == :utc ? :getutc : :getlocal - - if value.respond_to?(zone_conversion_method) - value = value.send(zone_conversion_method) + if ActiveRecord::Base.default_timezone == :utc + value = value.getutc if value.respond_to?(:getutc) && !value.utc? + else + value = value.getlocal if value.respond_to?(:getlocal) end end result = value.to_s(:db) if value.respond_to?(:usec) && value.usec > 0 - "#{result}.#{sprintf("%06d", value.usec)}" + result << "." << sprintf("%06d", value.usec) else result end @@ -138,15 +138,72 @@ module ActiveRecord "'#{quote_string(value.to_s)}'" end - def type_casted_binds(binds) # :nodoc: - if binds.first.is_a?(Array) - binds.map { |column, value| type_cast(value, column) } - else - binds.map { |attr| type_cast(attr.value_for_database) } - end + def sanitize_as_sql_comment(value) # :nodoc: + value.to_s.gsub(%r{ (/ (?: | \g<1>) \*) \+? \s* | \s* (\* (?: | \g<2>) /) }x, "") + end + + def column_name_matcher # :nodoc: + COLUMN_NAME + end + + def column_name_with_order_matcher # :nodoc: + COLUMN_NAME_WITH_ORDER end + # Regexp for column names (with or without a table name prefix). + # Matches the following: + # + # "#{table_name}.#{column_name}" + # "#{column_name}" + COLUMN_NAME = / + \A + ( + (?: + # table_name.column_name | function(one or no argument) + ((?:\w+\.)?\w+) | \w+\((?:|\g<2>)\) + ) + (?:(?:\s+AS)?\s+\w+)? + ) + (?:\s*,\s*\g<1>)* + \z + /ix + + # Regexp for column names with order (with or without a table name prefix, + # with or without various order modifiers). Matches the following: + # + # "#{table_name}.#{column_name}" + # "#{table_name}.#{column_name} #{direction}" + # "#{table_name}.#{column_name} #{direction} NULLS FIRST" + # "#{table_name}.#{column_name} NULLS LAST" + # "#{column_name}" + # "#{column_name} #{direction}" + # "#{column_name} #{direction} NULLS FIRST" + # "#{column_name} NULLS LAST" + COLUMN_NAME_WITH_ORDER = / + \A + ( + (?: + # table_name.column_name | function(one or no argument) + ((?:\w+\.)?\w+) | \w+\((?:|\g<2>)\) + ) + (?:\s+ASC|\s+DESC)? + (?:\s+NULLS\s+(?:FIRST|LAST))? + ) + (?:\s*,\s*\g<1>)* + \z + /ix + + private_constant :COLUMN_NAME, :COLUMN_NAME_WITH_ORDER + private + def type_casted_binds(binds) + if binds.first.is_a?(Array) + binds.map { |column, value| type_cast(value, column) } + else + binds.map { |attr| type_cast(attr.value_for_database) } + end + end + def lookup_cast_type(sql_type) type_map.lookup(sql_type) end @@ -157,13 +214,9 @@ module ActiveRecord end end - def types_which_need_no_typecasting - [nil, Numeric, String] - end - def _quote(value) case value - when String, ActiveSupport::Multibyte::Chars + when String, Symbol, ActiveSupport::Multibyte::Chars "'#{quote_string(value.to_s)}'" when true then quoted_true when false then quoted_false @@ -174,7 +227,6 @@ module ActiveRecord when Type::Binary::Data then quoted_binary(value) when Type::Time::Value then "'#{quoted_time(value)}'" when Date, Time then "'#{quoted_date(value)}'" - when Symbol then "'#{quote_string(value.to_s)}'" when Class then "'#{value}'" else raise TypeError, "can't quote #{value.class.name}" end @@ -188,10 +240,9 @@ module ActiveRecord when false then unquoted_false # BigDecimals need to be put in a non-normalized form and quoted. when BigDecimal then value.to_s("F") + when nil, Numeric, String then value when Type::Time::Value then quoted_time(value) when Date, Time then quoted_date(value) - when *types_which_need_no_typecasting - value else raise TypeError end end diff --git a/activerecord/lib/active_record/connection_adapters/abstract/savepoints.rb b/activerecord/lib/active_record/connection_adapters/abstract/savepoints.rb index 52a796b926..d6dbef3fc8 100644 --- a/activerecord/lib/active_record/connection_adapters/abstract/savepoints.rb +++ b/activerecord/lib/active_record/connection_adapters/abstract/savepoints.rb @@ -8,15 +8,15 @@ module ActiveRecord end def create_savepoint(name = current_savepoint_name) - execute("SAVEPOINT #{name}") + execute("SAVEPOINT #{name}", "TRANSACTION") end def exec_rollback_to_savepoint(name = current_savepoint_name) - execute("ROLLBACK TO SAVEPOINT #{name}") + execute("ROLLBACK TO SAVEPOINT #{name}", "TRANSACTION") end def release_savepoint(name = current_savepoint_name) - execute("RELEASE SAVEPOINT #{name}") + execute("RELEASE SAVEPOINT #{name}", "TRANSACTION") end end end diff --git a/activerecord/lib/active_record/connection_adapters/abstract/schema_creation.rb b/activerecord/lib/active_record/connection_adapters/abstract/schema_creation.rb index 2cb0a2a4df..23c993cfc3 100644 --- a/activerecord/lib/active_record/connection_adapters/abstract/schema_creation.rb +++ b/activerecord/lib/active_record/connection_adapters/abstract/schema_creation.rb @@ -15,11 +15,10 @@ module ActiveRecord end delegate :quote_column_name, :quote_table_name, :quote_default_expression, :type_to_sql, - :options_include_default?, :supports_indexes_in_create?, :supports_foreign_keys_in_create?, :foreign_key_options, + :options_include_default?, :supports_indexes_in_create?, :supports_foreign_keys?, :foreign_key_options, to: :@conn, private: true private - def visit_AlterTable(o) sql = +"ALTER TABLE #{quote_table_name(o.name)} " sql << o.adds.map { |col| accept col }.join(" ") @@ -50,7 +49,7 @@ module ActiveRecord statements.concat(o.indexes.map { |column_name, options| index_in_create(o.name, column_name, options) }) end - if supports_foreign_keys_in_create? + if supports_foreign_keys? statements.concat(o.foreign_keys.map { |to_table, options| foreign_key_in_create(o.name, to_table, options) }) end @@ -127,6 +126,9 @@ module ActiveRecord end def foreign_key_in_create(from_table, to_table, options) + prefix = ActiveRecord::Base.table_name_prefix + suffix = ActiveRecord::Base.table_name_suffix + to_table = "#{prefix}#{to_table}#{suffix}" options = foreign_key_options(from_table, to_table, options) accept ForeignKeyDefinition.new(from_table, to_table, options) end diff --git a/activerecord/lib/active_record/connection_adapters/abstract/schema_definitions.rb b/activerecord/lib/active_record/connection_adapters/abstract/schema_definitions.rb index db489143af..dbd533b4b3 100644 --- a/activerecord/lib/active_record/connection_adapters/abstract/schema_definitions.rb +++ b/activerecord/lib/active_record/connection_adapters/abstract/schema_definitions.rb @@ -1,7 +1,5 @@ # frozen_string_literal: true -require "active_support/deprecation" - module ActiveRecord module ConnectionAdapters #:nodoc: # Abstract representation of an index definition on a table. Instances of @@ -104,16 +102,12 @@ module ActiveRecord alias validated? validate? def export_name_on_schema_dump? - name !~ ActiveRecord::SchemaDumper.fk_ignore_pattern + !ActiveRecord::SchemaDumper.fk_ignore_pattern.match?(name) if name end - def defined_for?(to_table_ord = nil, to_table: nil, **options) - if to_table_ord - self.to_table == to_table_ord.to_s - else - (to_table.nil? || to_table.to_s == self.to_table) && - options.all? { |k, v| self.options[k].to_s == v.to_s } - end + def defined_for?(to_table: nil, **options) + (to_table.nil? || to_table.to_s == self.to_table) && + options.all? { |k, v| self.options[k].to_s == v.to_s } end private @@ -200,41 +194,44 @@ module ActiveRecord end module ColumnMethods + extend ActiveSupport::Concern + # Appends a primary key definition to the table definition. # Can be called multiple times, but this is probably not a good idea. def primary_key(name, type = :primary_key, **options) column(name, type, options.merge(primary_key: true)) end + ## + # :method: column + # :call-seq: column(name, type, **options) + # # Appends a column or columns of a specified type. # # t.string(:goat) # t.string(:goat, :sheep) # # See TableDefinition#column - [ - :bigint, - :binary, - :boolean, - :date, - :datetime, - :decimal, - :float, - :integer, - :json, - :string, - :text, - :time, - :timestamp, - :virtual, - ].each do |column_type| - module_eval <<-CODE, __FILE__, __LINE__ + 1 - def #{column_type}(*args, **options) - args.each { |name| column(name, :#{column_type}, options) } + + included do + define_column_methods :bigint, :binary, :boolean, :date, :datetime, :decimal, + :float, :integer, :json, :string, :text, :time, :timestamp, :virtual + + alias :numeric :decimal + end + + class_methods do + private def define_column_methods(*column_types) # :nodoc: + column_types.each do |column_type| + module_eval <<-RUBY, __FILE__, __LINE__ + 1 + def #{column_type}(*names, **options) + raise ArgumentError, "Missing column name(s) for #{column_type}" if names.empty? + names.each { |name| column(name, :#{column_type}, options) } + end + RUBY end - CODE + end end - alias_method :numeric, :decimal end # Represents the schema of an SQL table in an abstract way. This class @@ -259,18 +256,17 @@ module ActiveRecord include ColumnMethods attr_reader :name, :temporary, :if_not_exists, :options, :as, :comment, :indexes, :foreign_keys - attr_writer :indexes - deprecate :indexes= def initialize( + conn, name, temporary: false, if_not_exists: false, options: nil, as: nil, - comment: nil, - ** + comment: nil ) + @conn = conn @columns_hash = {} @indexes = [] @foreign_keys = [] @@ -363,7 +359,7 @@ module ActiveRecord # t.references :tagger, polymorphic: true # t.references :taggable, polymorphic: { default: 'Photo' }, index: false # end - def column(name, type, options = {}) + def column(name, type, **options) name = name.to_s type = type.to_sym if type options = options.dup @@ -397,10 +393,7 @@ module ActiveRecord end def foreign_key(table_name, options = {}) # :nodoc: - table_name_prefix = ActiveRecord::Base.table_name_prefix - table_name_suffix = ActiveRecord::Base.table_name_suffix - table_name = "#{table_name_prefix}#{table_name}#{table_name_suffix}" - foreign_keys.push([table_name, options]) + foreign_keys << [table_name, options] end # Appends <tt>:datetime</tt> columns <tt>:created_at</tt> and @@ -410,6 +403,10 @@ module ActiveRecord def timestamps(**options) options[:null] = false if options[:null].nil? + if !options.key?(:precision) && @conn.supports_datetime_with_precision? + options[:precision] = 6 + end + column(:created_at, :datetime, options) column(:updated_at, :datetime, options) end @@ -418,6 +415,7 @@ module ActiveRecord # # t.references(:user) # t.belongs_to(:supplier, foreign_key: true) + # t.belongs_to(:supplier, foreign_key: true, type: :integer) # # See {connection.add_reference}[rdoc-ref:SchemaStatements#add_reference] for details of the options you can use. def references(*args, **options) @@ -517,6 +515,7 @@ module ActiveRecord # t.json # t.virtual # t.remove + # t.remove_foreign_key # t.remove_references # t.remove_belongs_to # t.remove_index @@ -538,7 +537,7 @@ module ActiveRecord # t.column(:name, :string) # # See TableDefinition#column for details of the options you can use. - def column(column_name, type, options = {}) + def column(column_name, type, **options) index_options = options.delete(:index) @base.add_column(name, column_name, type, options) index(column_name, index_options.is_a?(Hash) ? index_options : {}) if index_options @@ -680,15 +679,26 @@ module ActiveRecord end alias :remove_belongs_to :remove_references - # Adds a foreign key. + # Adds a foreign key to the table using a supplied table name. # # t.foreign_key(:authors) + # t.foreign_key(:authors, column: :author_id, primary_key: "id") # # See {connection.add_foreign_key}[rdoc-ref:SchemaStatements#add_foreign_key] def foreign_key(*args) @base.add_foreign_key(name, *args) end + # Removes the given foreign key from the table. + # + # t.remove_foreign_key(:authors) + # t.remove_foreign_key(column: :author_id) + # + # See {connection.remove_foreign_key}[rdoc-ref:SchemaStatements#remove_foreign_key] + def remove_foreign_key(*args) + @base.remove_foreign_key(name, *args) + end + # Checks to see if a foreign key exists. # # t.foreign_key(:authors) unless t.foreign_key_exists?(:authors) diff --git a/activerecord/lib/active_record/connection_adapters/abstract/schema_dumper.rb b/activerecord/lib/active_record/connection_adapters/abstract/schema_dumper.rb index 622e00fffb..fb56e712be 100644 --- a/activerecord/lib/active_record/connection_adapters/abstract/schema_dumper.rb +++ b/activerecord/lib/active_record/connection_adapters/abstract/schema_dumper.rb @@ -15,7 +15,7 @@ module ActiveRecord def column_spec_for_primary_key(column) return {} if default_primary_key?(column) spec = { id: schema_type(column).inspect } - spec.merge!(prepare_column_options(column).except!(:null)) + spec.merge!(prepare_column_options(column).except!(:null, :comment)) spec[:default] ||= "nil" if explicit_primary_key_default?(column) spec end diff --git a/activerecord/lib/active_record/connection_adapters/abstract/schema_statements.rb b/activerecord/lib/active_record/connection_adapters/abstract/schema_statements.rb index 38cfc3a241..13f94a4722 100644 --- a/activerecord/lib/active_record/connection_adapters/abstract/schema_statements.rb +++ b/activerecord/lib/active_record/connection_adapters/abstract/schema_statements.rb @@ -129,11 +129,11 @@ module ActiveRecord # column_exists?(:suppliers, :name, :string, null: false) # column_exists?(:suppliers, :tax, :decimal, precision: 8, scale: 2) # - def column_exists?(table_name, column_name, type = nil, options = {}) + def column_exists?(table_name, column_name, type = nil, **options) column_name = column_name.to_s checks = [] checks << lambda { |c| c.name == column_name } - checks << lambda { |c| c.type == type } if type + checks << lambda { |c| c.type == type.to_sym rescue nil } if type column_options_keys.each do |attr| checks << lambda { |c| c.send(attr) == options[attr] } if options.key?(attr) end @@ -290,25 +290,27 @@ module ActiveRecord # SELECT * FROM orders INNER JOIN line_items ON order_id=orders.id # # See also TableDefinition#column for details on how to create columns. - def create_table(table_name, **options) - td = create_table_definition(table_name, options) + def create_table(table_name, id: :primary_key, primary_key: nil, force: nil, **options) + td = create_table_definition( + table_name, options.extract!(:temporary, :if_not_exists, :options, :as, :comment) + ) - if options[:id] != false && !options[:as] - pk = options.fetch(:primary_key) do - Base.get_primary_key table_name.to_s.singularize - end + if id && !td.as + pk = primary_key || Base.get_primary_key(table_name.to_s.singularize) if pk.is_a?(Array) td.primary_keys pk else - td.primary_key pk, options.fetch(:id, :primary_key), options + td.primary_key pk, id, options end end yield td if block_given? - if options[:force] - drop_table(table_name, options.merge(if_exists: true)) + if force + drop_table(table_name, force: force, if_exists: true) + else + schema_cache.clear_data_source_cache!(table_name.to_s) end result = execute schema_creation.accept td @@ -320,7 +322,7 @@ module ActiveRecord end if supports_comments? && !supports_comments_in_create? - if table_comment = options[:comment].presence + if table_comment = td.comment.presence change_table_comment(table_name, table_comment) end @@ -498,6 +500,7 @@ module ActiveRecord # it can be helpful to provide these in a migration's +change+ method so it can be reverted. # In that case, +options+ and the block will be used by #create_table. def drop_table(table_name, options = {}) + schema_cache.clear_data_source_cache!(table_name.to_s) execute "DROP TABLE#{' IF EXISTS' if options[:if_exists]} #{quote_table_name(table_name)}" end @@ -517,14 +520,15 @@ module ActiveRecord # Available options are (none of these exists by default): # * <tt>:limit</tt> - # Requests a maximum column length. This is the number of characters for a <tt>:string</tt> column - # and number of bytes for <tt>:text</tt>, <tt>:binary</tt> and <tt>:integer</tt> columns. + # and number of bytes for <tt>:text</tt>, <tt>:binary</tt>, and <tt>:integer</tt> columns. # This option is ignored by some backends. # * <tt>:default</tt> - # The column's default value. Use +nil+ for +NULL+. # * <tt>:null</tt> - # Allows or disallows +NULL+ values in the column. # * <tt>:precision</tt> - - # Specifies the precision for the <tt>:decimal</tt> and <tt>:numeric</tt> columns. + # Specifies the precision for the <tt>:decimal</tt>, <tt>:numeric</tt>, + # <tt>:datetime</tt>, and <tt>:time</tt> columns. # * <tt>:scale</tt> - # Specifies the scale for the <tt>:decimal</tt> and <tt>:numeric</tt> columns. # * <tt>:collation</tt> - @@ -583,7 +587,7 @@ module ActiveRecord # # Defines a column with a database-specific type. # add_column(:shapes, :triangle, 'polygon') # # ALTER TABLE "shapes" ADD "triangle" polygon - def add_column(table_name, column_name, type, options = {}) + def add_column(table_name, column_name, type, **options) at = create_alter_table table_name at.add_column(column_name, type, options) execute schema_creation.accept at @@ -734,7 +738,7 @@ module ActiveRecord # # CREATE UNIQUE INDEX index_accounts_on_branch_id_and_party_id ON accounts(branch_id, party_id) WHERE active # - # Note: Partial indexes are only supported for PostgreSQL and SQLite 3.8.0+. + # Note: Partial indexes are only supported for PostgreSQL and SQLite. # # ====== Creating an index with a specific method # @@ -769,6 +773,17 @@ module ActiveRecord # CREATE FULLTEXT INDEX index_developers_on_name ON developers (name) -- MySQL # # Note: only supported by MySQL. + # + # ====== Creating an index with a specific algorithm + # + # add_index(:developers, :name, algorithm: :concurrently) + # # CREATE INDEX CONCURRENTLY developers_on_name on developers (name) + # + # Note: only supported by PostgreSQL. + # + # Concurrently adding an index is not supported in a transaction. + # + # For more information see the {"Transactional Migrations" section}[rdoc-ref:Migration]. def add_index(table_name, column_name, options = {}) index_name, index_type, index_columns, index_options = add_index_options(table_name, column_name, options) execute "CREATE #{index_type} INDEX #{quote_column_name(index_name)} ON #{quote_table_name(table_name)} (#{index_columns})#{index_options}" @@ -792,6 +807,15 @@ module ActiveRecord # # remove_index :accounts, name: :by_branch_party # + # Removes the index named +by_branch_party+ in the +accounts+ table +concurrently+. + # + # remove_index :accounts, name: :by_branch_party, algorithm: :concurrently + # + # Note: only supported by PostgreSQL. + # + # Concurrently removing an index is not supported in a transaction. + # + # For more information see the {"Transactional Migrations" section}[rdoc-ref:Migration]. def remove_index(table_name, options = {}) index_name = index_name_for_remove(table_name, options) execute "DROP INDEX #{quote_column_name(index_name)} ON #{quote_table_name(table_name)}" @@ -851,7 +875,7 @@ module ActiveRecord # [<tt>:null</tt>] # Whether the column allows nulls. Defaults to true. # - # ====== Create a user_id bigint column without a index + # ====== Create a user_id bigint column without an index # # add_reference(:products, :user, index: false) # @@ -965,7 +989,7 @@ module ActiveRecord # [<tt>:on_update</tt>] # Action that happens <tt>ON UPDATE</tt>. Valid values are +:nullify+, +:cascade+ and +:restrict+ # [<tt>:validate</tt>] - # (Postgres only) Specify whether or not the constraint should be validated. Defaults to +true+. + # (PostgreSQL only) Specify whether or not the constraint should be validated. Defaults to +true+. def add_foreign_key(from_table, to_table, options = {}) return unless supports_foreign_keys? @@ -1001,10 +1025,10 @@ module ActiveRecord # with an addition of # [<tt>:to_table</tt>] # The name of the table that contains the referenced primary key. - def remove_foreign_key(from_table, options_or_to_table = {}) + def remove_foreign_key(from_table, to_table = nil, **options) return unless supports_foreign_keys? - fk_name_to_delete = foreign_key_for!(from_table, options_or_to_table).name + fk_name_to_delete = foreign_key_for!(from_table, to_table: to_table, **options).name at = create_alter_table from_table at.drop_foreign_key fk_name_to_delete @@ -1023,14 +1047,12 @@ module ActiveRecord # # Checks to see if a foreign key with a custom name exists. # foreign_key_exists?(:accounts, name: "special_fk_name") # - def foreign_key_exists?(from_table, options_or_to_table = {}) - foreign_key_for(from_table, options_or_to_table).present? + def foreign_key_exists?(from_table, to_table = nil, **options) + foreign_key_for(from_table, to_table: to_table, **options).present? end def foreign_key_column_for(table_name) # :nodoc: - prefix = Base.table_name_prefix - suffix = Base.table_name_suffix - name = table_name.to_s =~ /#{prefix}(.+)#{suffix}/ ? $1 : table_name.to_s + name = strip_table_name_prefix_and_suffix(table_name) "#{name.singularize}_id" end @@ -1041,8 +1063,8 @@ module ActiveRecord options end - def dump_schema_information #:nodoc: - versions = ActiveRecord::SchemaMigration.all_versions + def dump_schema_information # :nodoc: + versions = schema_migration.all_versions insert_versions_sql(versions) if versions.any? end @@ -1050,15 +1072,18 @@ module ActiveRecord { primary_key: true } end - def assume_migrated_upto_version(version, migrations_paths) - migrations_paths = Array(migrations_paths) + def assume_migrated_upto_version(version, migrations_paths = nil) + unless migrations_paths.nil? + ActiveSupport::Deprecation.warn(<<~MSG.squish) + Passing migrations_paths to #assume_migrated_upto_version is deprecated and will be removed in Rails 6.1. + MSG + end + version = version.to_i - sm_table = quote_table_name(ActiveRecord::SchemaMigration.table_name) + sm_table = quote_table_name(schema_migration.table_name) - migrated = ActiveRecord::SchemaMigration.all_versions.map(&:to_i) - versions = migration_context.migration_files.map do |file| - migration_context.parse_migration_filename(file).first.to_i - end + migrated = migration_context.get_all_versions + versions = migration_context.migrations.map(&:version) unless migrated.include?(version) execute "INSERT INTO #{sm_table} (version) VALUES (#{quote(version)})" @@ -1095,7 +1120,7 @@ module ActiveRecord if (0..6) === precision column_type_sql << "(#{precision})" else - raise(ActiveRecordError, "No #{native[:name]} type has precision of #{precision}. The allowed range of precision is from 0 to 6") + raise ArgumentError, "No #{native[:name]} type has precision of #{precision}. The allowed range of precision is from 0 to 6" end elsif (type != :primary_key) && (limit ||= native.is_a?(Hash) && native[:limit]) column_type_sql << "(#{limit})" @@ -1125,6 +1150,10 @@ module ActiveRecord def add_timestamps(table_name, options = {}) options[:null] = false if options[:null].nil? + if !options.key?(:precision) && supports_datetime_with_precision? + options[:precision] = 6 + end + add_column table_name, :created_at, :datetime, options add_column table_name, :updated_at, :datetime, options end @@ -1179,12 +1208,22 @@ module ActiveRecord end # Changes the comment for a table or removes it if +nil+. - def change_table_comment(table_name, comment) + # + # Passing a hash containing +:from+ and +:to+ will make this change + # reversible in migration: + # + # change_table_comment(:posts, from: "old_comment", to: "new_comment") + def change_table_comment(table_name, comment_or_changes) raise NotImplementedError, "#{self.class} does not support changing table comments" end # Changes the comment for a column or removes it if +nil+. - def change_column_comment(table_name, column_name, comment) + # + # Passing a hash containing +:from+ and +:to+ will make this change + # reversible in migration: + # + # change_column_comment(:posts, :state, from: "old_comment", to: "new_comment") + def change_column_comment(table_name, column_name, comment_or_changes) raise NotImplementedError, "#{self.class} does not support changing column comments" end @@ -1286,7 +1325,7 @@ module ActiveRecord end def create_table_definition(*args) - TableDefinition.new(*args) + TableDefinition.new(self, *args) end def create_alter_table(name) @@ -1320,6 +1359,12 @@ module ActiveRecord { column: column_names } end + def strip_table_name_prefix_and_suffix(table_name) + prefix = Base.table_name_prefix + suffix = Base.table_name_suffix + table_name.to_s =~ /#{prefix}(.+)#{suffix}/ ? $1 : table_name.to_s + end + def foreign_key_name(table_name, options) options.fetch(:name) do identifier = "#{table_name}_#{options.fetch(:column)}_fk" @@ -1329,14 +1374,14 @@ module ActiveRecord end end - def foreign_key_for(from_table, options_or_to_table = {}) + def foreign_key_for(from_table, **options) return unless supports_foreign_keys? - foreign_keys(from_table).detect { |fk| fk.defined_for? options_or_to_table } + foreign_keys(from_table).detect { |fk| fk.defined_for?(options) } end - def foreign_key_for!(from_table, options_or_to_table = {}) - foreign_key_for(from_table, options_or_to_table) || \ - raise(ArgumentError, "Table '#{from_table}' has no foreign key for #{options_or_to_table}") + def foreign_key_for!(from_table, to_table: nil, **options) + foreign_key_for(from_table, to_table: to_table, **options) || + raise(ArgumentError, "Table '#{from_table}' has no foreign key for #{to_table || options}") end def extract_foreign_key_action(specifier) @@ -1362,11 +1407,37 @@ module ActiveRecord default_or_changes end end + alias :extract_new_comment_value :extract_new_default_value def can_remove_index_by_name?(options) options.is_a?(Hash) && options.key?(:name) && options.except(:name, :algorithm).empty? end + def bulk_change_table(table_name, operations) + sql_fragments = [] + non_combinable_operations = [] + + operations.each do |command, args| + table, arguments = args.shift, args + method = :"#{command}_for_alter" + + if respond_to?(method, true) + sqls, procs = Array(send(method, table, *arguments)).partition { |v| v.is_a?(String) } + sql_fragments << sqls + non_combinable_operations.concat(procs) + else + execute "ALTER TABLE #{quote_table_name(table_name)} #{sql_fragments.join(", ")}" unless sql_fragments.empty? + non_combinable_operations.each(&:call) + sql_fragments = [] + non_combinable_operations = [] + send(command, table, *arguments) + end + end + + execute "ALTER TABLE #{quote_table_name(table_name)} #{sql_fragments.join(", ")}" unless sql_fragments.empty? + non_combinable_operations.each(&:call) + end + def add_column_for_alter(table_name, column_name, type, options = {}) td = create_table_definition(table_name) cd = td.new_column_definition(column_name, type, options) @@ -1382,7 +1453,7 @@ module ActiveRecord end def insert_versions_sql(versions) - sm_table = quote_table_name(ActiveRecord::SchemaMigration.table_name) + sm_table = quote_table_name(schema_migration.table_name) if versions.is_a?(Array) sql = +"INSERT INTO #{sm_table} (version) VALUES\n" diff --git a/activerecord/lib/active_record/connection_adapters/abstract/transaction.rb b/activerecord/lib/active_record/connection_adapters/abstract/transaction.rb index 0f2b1e85ff..53ce8df491 100644 --- a/activerecord/lib/active_record/connection_adapters/abstract/transaction.rb +++ b/activerecord/lib/active_record/connection_adapters/abstract/transaction.rb @@ -5,10 +5,11 @@ module ActiveRecord class TransactionState def initialize(state = nil) @state = state - @children = [] + @children = nil end def add_child(state) + @children ||= [] @children << state end @@ -40,31 +41,13 @@ module ActiveRecord committed? || rolledback? end - def set_state(state) - ActiveSupport::Deprecation.warn(<<-MSG.squish) - The set_state method is deprecated and will be removed in - Rails 6.0. Please use rollback! or commit! to set transaction - state directly. - MSG - case state - when :rolledback - rollback! - when :committed - commit! - when nil - nullify! - else - raise ArgumentError, "Invalid transaction state: #{state}" - end - end - def rollback! - @children.each { |c| c.rollback! } + @children&.each { |c| c.rollback! } @state = :rolledback end def full_rollback! - @children.each { |c| c.rollback! } + @children&.each { |c| c.rollback! } @state = :fully_rolledback end @@ -93,18 +76,19 @@ module ActiveRecord class Transaction #:nodoc: attr_reader :connection, :state, :records, :savepoint_name, :isolation_level - def initialize(connection, options, run_commit_callbacks: false) + def initialize(connection, isolation: nil, joinable: true, run_commit_callbacks: false) @connection = connection @state = TransactionState.new - @records = [] - @isolation_level = options[:isolation] + @records = nil + @isolation_level = isolation @materialized = false - @joinable = options.fetch(:joinable, true) + @joinable = joinable @run_commit_callbacks = run_commit_callbacks end def add_record(record) - records << record + @records ||= [] + @records << record end def materialize! @@ -116,32 +100,42 @@ module ActiveRecord end def rollback_records - ite = records.uniq + return unless records + ite = records.uniq(&:object_id) + already_run_callbacks = {} while record = ite.shift - record.rolledback!(force_restore_state: full_rollback?) + trigger_callbacks = record.trigger_transactional_callbacks? + should_run_callbacks = !already_run_callbacks[record] && trigger_callbacks + already_run_callbacks[record] ||= trigger_callbacks + record.rolledback!(force_restore_state: full_rollback?, should_run_callbacks: should_run_callbacks) end ensure - ite.each do |i| + ite&.each do |i| i.rolledback!(force_restore_state: full_rollback?, should_run_callbacks: false) end end def before_commit_records - records.uniq.each(&:before_committed!) if @run_commit_callbacks + records.uniq.each(&:before_committed!) if records && @run_commit_callbacks end def commit_records - ite = records.uniq + return unless records + ite = records.uniq(&:object_id) + already_run_callbacks = {} while record = ite.shift if @run_commit_callbacks - record.committed! + trigger_callbacks = record.trigger_transactional_callbacks? + should_run_callbacks = !already_run_callbacks[record] && trigger_callbacks + already_run_callbacks[record] ||= trigger_callbacks + record.committed!(should_run_callbacks: should_run_callbacks) else # if not running callbacks, only adds the record to the parent transaction connection.add_transaction_record(record) end end ensure - ite.each { |i| i.committed!(should_run_callbacks: false) } + ite&.each { |i| i.committed!(should_run_callbacks: false) } end def full_rollback?; true; end @@ -151,8 +145,8 @@ module ActiveRecord end class SavepointTransaction < Transaction - def initialize(connection, savepoint_name, parent_transaction, *args) - super(connection, *args) + def initialize(connection, savepoint_name, parent_transaction, **options) + super(connection, options) parent_transaction.state.add_child(@state) @@ -212,20 +206,34 @@ module ActiveRecord @lazy_transactions_enabled = true end - def begin_transaction(options = {}) + def begin_transaction(isolation: nil, joinable: true, _lazy: true) @connection.lock.synchronize do run_commit_callbacks = !current_transaction.joinable? transaction = if @stack.empty? - RealTransaction.new(@connection, options, run_commit_callbacks: run_commit_callbacks) + RealTransaction.new( + @connection, + isolation: isolation, + joinable: joinable, + run_commit_callbacks: run_commit_callbacks + ) else - SavepointTransaction.new(@connection, "active_record_#{@stack.size}", @stack.last, options, - run_commit_callbacks: run_commit_callbacks) + SavepointTransaction.new( + @connection, + "active_record_#{@stack.size}", + @stack.last, + isolation: isolation, + joinable: joinable, + run_commit_callbacks: run_commit_callbacks + ) end - transaction.materialize! unless @connection.supports_lazy_transactions? && lazy_transactions_enabled? + if @connection.supports_lazy_transactions? && lazy_transactions_enabled? && _lazy + @has_unmaterialized_transactions = true + else + transaction.materialize! + end @stack.push(transaction) - @has_unmaterialized_transactions = true if @connection.supports_lazy_transactions? transaction end end @@ -281,28 +289,26 @@ module ActiveRecord end end - def within_new_transaction(options = {}) + def within_new_transaction(isolation: nil, joinable: true) @connection.lock.synchronize do - begin - transaction = begin_transaction options - yield - rescue Exception => error - if transaction + transaction = begin_transaction(isolation: isolation, joinable: joinable) + yield + rescue Exception => error + if transaction + rollback_transaction + after_failure_actions(transaction, error) + end + raise + ensure + if !error && transaction + if Thread.current.status == "aborting" rollback_transaction - after_failure_actions(transaction, error) - end - raise - ensure - unless error - if Thread.current.status == "aborting" - rollback_transaction if transaction - else - begin - commit_transaction if transaction - rescue Exception - rollback_transaction(transaction) unless transaction.state.completed? - raise - end + else + begin + commit_transaction + rescue Exception + rollback_transaction(transaction) unless transaction.state.completed? + raise end end end @@ -318,7 +324,6 @@ module ActiveRecord end private - NULL_TRANSACTION = NullTransaction.new # Deallocate invalidated prepared statements outside of the transaction diff --git a/activerecord/lib/active_record/connection_adapters/abstract_adapter.rb b/activerecord/lib/active_record/connection_adapters/abstract_adapter.rb index 0fe868478c..dc970c384b 100644 --- a/activerecord/lib/active_record/connection_adapters/abstract_adapter.rb +++ b/activerecord/lib/active_record/connection_adapters/abstract_adapter.rb @@ -76,8 +76,8 @@ module ActiveRecord SIMPLE_INT = /\A\d+\z/ - attr_accessor :visitor, :pool - attr_reader :schema_cache, :owner, :logger, :prepared_statements, :lock + attr_accessor :pool + attr_reader :visitor, :owner, :logger, :lock, :prepared_statements alias :in_use? :owner set_callback :checkin, :after, :enable_lazy_transactions! @@ -100,6 +100,19 @@ module ActiveRecord end end + def self.build_read_query_regexp(*parts) # :nodoc: + parts = parts.map { |part| /\A[\(\s]*#{part}/i } + Regexp.union(*parts) + end + + def self.quoted_column_names # :nodoc: + @quoted_column_names ||= {} + end + + def self.quoted_table_names # :nodoc: + @quoted_table_names ||= {} + end + def initialize(connection, logger = nil, config = {}) # :nodoc: super() @@ -108,11 +121,10 @@ module ActiveRecord @instrumenter = ActiveSupport::Notifications.instrumenter @logger = logger @config = config - @pool = nil + @pool = ActiveRecord::ConnectionAdapters::NullPool.new @idle_since = Concurrent.monotonic_time - @schema_cache = SchemaCache.new self - @quoted_column_names, @quoted_table_names = {}, {} @visitor = arel_visitor + @statements = build_statement_pool @lock = ActiveSupport::Concurrency::LoadInterlockAwareMonitor.new if self.class.type_cast_config_to_boolean(config.fetch(:prepared_statements) { true }) @@ -125,27 +137,51 @@ module ActiveRecord @advisory_locks_enabled = self.class.type_cast_config_to_boolean( config.fetch(:advisory_locks, true) ) - - check_version end def replica? @config[:replica] || false end + # Determines whether writes are currently being prevents. + # + # Returns true if the connection is a replica, or if +prevent_writes+ + # is set to true. + def preventing_writes? + replica? || ActiveRecord::Base.connection_handler.prevent_writes + end + def migrations_paths # :nodoc: @config[:migrations_paths] || Migrator.migrations_paths end def migration_context # :nodoc: - MigrationContext.new(migrations_paths) + MigrationContext.new(migrations_paths, schema_migration) + end + + def schema_migration # :nodoc: + @schema_migration ||= begin + conn = self + spec_name = conn.pool.spec.name + name = "#{spec_name}::SchemaMigration" + + Class.new(ActiveRecord::SchemaMigration) do + define_singleton_method(:name) { name } + define_singleton_method(:to_s) { name } + + self.connection_specification_name = spec_name + end + end end class Version include Comparable - def initialize(version_string) + attr_reader :full_version_string + + def initialize(version_string, full_version_string = nil) @version = version_string.split(".").map(&:to_i) + @full_version_string = full_version_string end def <=>(version_string) @@ -177,9 +213,13 @@ module ActiveRecord @owner = Thread.current end + def schema_cache + @pool.get_schema_cache(self) + end + def schema_cache=(cache) cache.connection = self - @schema_cache = cache + @pool.set_schema_cache(cache) end # this method must only be called while holding connection pool's mutex @@ -230,6 +270,11 @@ module ActiveRecord self.class::ADAPTER_NAME end + # Does the database for this adapter exist? + def self.database_exists?(config) + raise NotImplementedError + end + # Does this adapter support DDL rollbacks in transactions? That is, would # CREATE TABLE or ALTER TABLE get rolled back by a transaction? def supports_ddl_transactions? @@ -308,12 +353,18 @@ module ActiveRecord def supports_foreign_keys_in_create? supports_foreign_keys? end + deprecate :supports_foreign_keys_in_create? # Does this adapter support views? def supports_views? false end + # Does this adapter support materialized views? + def supports_materialized_views? + false + end + # Does this adapter support datetime with precision? def supports_datetime_with_precision? false @@ -350,10 +401,31 @@ module ActiveRecord false end + # Does this adapter support optimizer hints? + def supports_optimizer_hints? + false + end + def supports_lazy_transactions? false end + def supports_insert_returning? + false + end + + def supports_insert_on_duplicate_skip? + false + end + + def supports_insert_on_duplicate_update? + false + end + + def supports_insert_conflict_target? + false + end + # This is meant to be implemented by the adapters that support extensions def disable_extension(name) end @@ -431,6 +503,9 @@ module ActiveRecord # # Prevent @connection's finalizer from touching the socket, or # otherwise communicating with its server, when it is collected. + if schema_cache.connection == self + schema_cache.connection = nil + end end # Reset the state of this connection, directing the DBMS to clear @@ -443,11 +518,9 @@ module ActiveRecord # this should be overridden by concrete adapters end - ### - # Clear any caching the database adapter may be doing, for example - # clearing the prepared statement cache. This is database specific. + # Clear any caching the database adapter may be doing. def clear_cache! - # this should be overridden by concrete adapters + @lock.synchronize { @statements.clear } if @statements end # Returns true if its required to reload the connection between requests for development mode. @@ -473,15 +546,21 @@ module ActiveRecord @connection end - def case_sensitive_comparison(table, attribute, column, value) # :nodoc: - table[attribute].eq(value) + def default_uniqueness_comparison(attribute, value, klass) # :nodoc: + attribute.eq(value) end - def case_insensitive_comparison(table, attribute, column, value) # :nodoc: + def case_sensitive_comparison(attribute, value) # :nodoc: + attribute.eq(value) + end + + def case_insensitive_comparison(attribute, value) # :nodoc: + column = column_for_attribute(attribute) + if can_perform_case_insensitive_comparison_for?(column) - table[attribute].lower.eq(table.lower(value)) + attribute.lower.eq(attribute.relation.lower(value)) else - table[attribute].eq(value) + attribute.eq(value) end end @@ -503,10 +582,30 @@ module ActiveRecord index.using.nil? end - private - def check_version + # Called by ActiveRecord::InsertAll, + # Passed an instance of ActiveRecord::InsertAll::Builder, + # This method implements standard bulk inserts for all databases, but + # should be overridden by adapters to implement common features with + # non-standard syntax like handling duplicates or returning values. + def build_insert_sql(insert) # :nodoc: + if insert.skip_duplicates? || insert.update_duplicates? + raise NotImplementedError, "#{self.class} should define `build_insert_sql` to implement adapter-specific logic for handling duplicates during INSERT" end + "INSERT #{insert.into} #{insert.values_list}" + end + + def get_database_version # :nodoc: + end + + def database_version # :nodoc: + schema_cache.database_version + end + + def check_version # :nodoc: + end + + private def type_map @type_map ||= Type::TypeMap.new.tap do |mapping| initialize_type_map(mapping) @@ -580,14 +679,12 @@ module ActiveRecord $1.to_i if sql_type =~ /\((.*)\)/ end - def translate_exception_class(e, sql) - begin - message = "#{e.class.name}: #{e.message}: #{sql}" - rescue Encoding::CompatibilityError - message = "#{e.class.name}: #{e.message.force_encoding sql.encoding}: #{sql}" - end + def translate_exception_class(e, sql, binds) + message = "#{e.class.name}: #{e.message}" - exception = translate_exception(e, message) + exception = translate_exception( + e, message: message, sql: sql, binds: binds + ) exception.set_backtrace e.backtrace exception end @@ -600,24 +697,23 @@ module ActiveRecord binds: binds, type_casted_binds: type_casted_binds, statement_name: statement_name, - connection_id: object_id) do - begin - @lock.synchronize do - yield - end - rescue => e - raise translate_exception_class(e, sql) + connection_id: object_id, + connection: self) do + @lock.synchronize do + yield end + rescue => e + raise translate_exception_class(e, sql, binds) end end - def translate_exception(exception, message) + def translate_exception(exception, message:, sql:, binds:) # override in derived class case exception when RuntimeError exception else - ActiveRecord::StatementInvalid.new(message) + ActiveRecord::StatementInvalid.new(message, sql: sql, binds: binds) end end @@ -631,6 +727,11 @@ module ActiveRecord raise(ActiveRecordError, "No such column: #{table_name}.#{column_name}") end + def column_for_attribute(attribute) + table_name = attribute.relation.name + schema_cache.columns_hash(table_name)[attribute.name.to_s] + end + def collector if prepared_statements Arel::Collectors::Composite.new( @@ -648,6 +749,9 @@ module ActiveRecord def arel_visitor Arel::Visitors::ToSql.new(self) end + + def build_statement_pool + end end end end diff --git a/activerecord/lib/active_record/connection_adapters/abstract_mysql_adapter.rb b/activerecord/lib/active_record/connection_adapters/abstract_mysql_adapter.rb index 13c799b64a..405fecb603 100644 --- a/activerecord/lib/active_record/connection_adapters/abstract_mysql_adapter.rb +++ b/activerecord/lib/active_record/connection_adapters/abstract_mysql_adapter.rb @@ -29,7 +29,7 @@ module ActiveRecord NATIVE_DATABASE_TYPES = { primary_key: "bigint auto_increment PRIMARY KEY", string: { name: "varchar", limit: 255 }, - text: { name: "text", limit: 65535 }, + text: { name: "text" }, integer: { name: "int", limit: 4 }, float: { name: "float", limit: 24 }, decimal: { name: "decimal" }, @@ -37,14 +37,14 @@ module ActiveRecord timestamp: { name: "timestamp" }, time: { name: "time" }, date: { name: "date" }, - binary: { name: "blob", limit: 65535 }, + binary: { name: "blob" }, + blob: { name: "blob" }, boolean: { name: "tinyint", limit: 1 }, json: { name: "json" }, } class StatementPool < ConnectionAdapters::StatementPool # :nodoc: private - def dealloc(stmt) stmt.close end @@ -52,28 +52,28 @@ module ActiveRecord def initialize(connection, logger, connection_options, config) super(connection, logger, config) - - @statements = StatementPool.new(self.class.type_cast_config_to_integer(config[:statement_limit])) end - def version #:nodoc: - @version ||= Version.new(version_string) + def get_database_version #:nodoc: + full_version_string = get_full_version + version_string = version_string(full_version_string) + Version.new(version_string, full_version_string) end def mariadb? # :nodoc: /mariadb/i.match?(full_version) end - def supports_bulk_alter? #:nodoc: + def supports_bulk_alter? true end def supports_index_sort_order? - !mariadb? && version >= "8.0.1" + !mariadb? && database_version >= "8.0.1" end def supports_expression_index? - !mariadb? && version >= "8.0.13" + !mariadb? && database_version >= "8.0.13" end def supports_transaction_isolation? @@ -97,31 +97,28 @@ module ActiveRecord end def supports_datetime_with_precision? - if mariadb? - version >= "5.3.0" - else - version >= "5.6.4" - end + mariadb? || database_version >= "5.6.4" end def supports_virtual_columns? - if mariadb? - version >= "5.2.0" - else - version >= "5.7.5" - end + mariadb? || database_version >= "5.7.5" + end + + # See https://dev.mysql.com/doc/refman/8.0/en/optimizer-hints.html for more details. + def supports_optimizer_hints? + !mariadb? && database_version >= "5.7.7" end def supports_advisory_locks? true end - def supports_longer_index_key_prefix? - if mariadb? - version >= "10.2.2" - else - version >= "5.7.9" - end + def supports_insert_on_duplicate_skip? + true + end + + def supports_insert_on_duplicate_update? + true end def get_advisory_lock(lock_name, timeout = 0) # :nodoc: @@ -169,25 +166,15 @@ module ActiveRecord # CONNECTION MANAGEMENT ==================================== - # Clears the prepared statements cache. - def clear_cache! + def clear_cache! # :nodoc: reload_type_map - @statements.clear + super end #-- # DATABASE STATEMENTS ====================================== #++ - def explain(arel, binds = []) - sql = "EXPLAIN #{to_sql(arel, binds)}" - start = Time.now - result = exec_query(sql, "EXPLAIN", binds) - elapsed = Time.now - start - - MySQL::ExplainPrettyPrinter.new.pp(result, elapsed) - end - # Executes the SQL statement in the context of this connection. def execute(sql, name = nil) materialize_transactions @@ -207,7 +194,7 @@ module ActiveRecord end def begin_db_transaction - execute "BEGIN" + execute("BEGIN", "TRANSACTION") end def begin_isolated_db_transaction(isolation) @@ -216,11 +203,11 @@ module ActiveRecord end def commit_db_transaction #:nodoc: - execute "COMMIT" + execute("COMMIT", "TRANSACTION") end def exec_rollback_db_transaction #:nodoc: - execute "ROLLBACK" + execute("ROLLBACK", "TRANSACTION") end def empty_insert_statement_value(primary_key = nil) @@ -250,7 +237,7 @@ module ActiveRecord execute "CREATE DATABASE #{quote_table_name(name)} DEFAULT COLLATE #{quote_table_name(options[:collation])}" elsif options[:charset] execute "CREATE DATABASE #{quote_table_name(name)} DEFAULT CHARACTER SET #{quote_table_name(options[:charset])}" - elsif supports_longer_index_key_prefix? + elsif row_format_dynamic_by_default? execute "CREATE DATABASE #{quote_table_name(name)} DEFAULT CHARACTER SET `utf8mb4`" else raise "Configure a supported :charset and ensure innodb_large_prefix is enabled to support indexes on varchar(255) string columns." @@ -279,10 +266,6 @@ module ActiveRecord show_variable "collation_database" end - def truncate(table_name, name = nil) - execute "TRUNCATE TABLE #{quote_table_name(table_name)}", name - end - def table_comment(table_name) # :nodoc: scope = quoted_scope(table_name) @@ -294,22 +277,8 @@ module ActiveRecord SQL end - def bulk_change_table(table_name, operations) #:nodoc: - sqls = operations.flat_map do |command, args| - table, arguments = args.shift, args - method = :"#{command}_for_alter" - - if respond_to?(method, true) - send(method, table, *arguments) - else - raise "Unknown method called : #{method}(#{arguments.inspect})" - end - end.join(", ") - - execute("ALTER TABLE #{quote_table_name(table_name)} #{sqls}") - end - - def change_table_comment(table_name, comment) #:nodoc: + def change_table_comment(table_name, comment_or_changes) # :nodoc: + comment = extract_new_comment_value(comment_or_changes) comment = "" if comment.nil? execute("ALTER TABLE #{quote_table_name(table_name)} COMMENT #{quote(comment)}") end @@ -319,6 +288,8 @@ module ActiveRecord # Example: # rename_table('octopuses', 'octopi') def rename_table(table_name, new_name) + schema_cache.clear_data_source_cache!(table_name.to_s) + schema_cache.clear_data_source_cache!(new_name.to_s) execute "RENAME TABLE #{quote_table_name(table_name)} TO #{quote_table_name(new_name)}" rename_table_indexes(table_name, new_name) end @@ -339,6 +310,7 @@ module ActiveRecord # it can be helpful to provide these in a migration's +change+ method so it can be reverted. # In that case, +options+ and the block will be used by create_table. def drop_table(table_name, options = {}) + schema_cache.clear_data_source_cache!(table_name.to_s) execute "DROP#{' TEMPORARY' if options[:temporary]} TABLE#{' IF EXISTS' if options[:if_exists]} #{quote_table_name(table_name)}#{' CASCADE' if options[:force] == :cascade}" end @@ -365,7 +337,8 @@ module ActiveRecord change_column table_name, column_name, nil, null: null end - def change_column_comment(table_name, column_name, comment) #:nodoc: + def change_column_comment(table_name, column_name, comment_or_changes) # :nodoc: + comment = extract_new_comment_value(comment_or_changes) change_column table_name, column_name, nil, comment: comment end @@ -446,30 +419,6 @@ module ActiveRecord table_options end - # Maps logical Rails types to MySQL-specific data types. - def type_to_sql(type, limit: nil, precision: nil, scale: nil, unsigned: nil, **) # :nodoc: - sql = \ - case type.to_s - when "integer" - integer_to_sql(limit) - when "text" - text_to_sql(limit) - when "blob" - binary_to_sql(limit) - when "binary" - if (0..0xfff) === limit - "varbinary(#{limit})" - else - binary_to_sql(limit) - end - else - super - end - - sql = "#{sql} unsigned" if unsigned && type != :primary_key - sql - end - # SHOW VARIABLES LIKE 'name' def show_variable(name) query_value("SELECT @@#{name}", "SCHEMA") @@ -492,9 +441,26 @@ module ActiveRecord SQL end - def case_sensitive_comparison(table, attribute, column, value) # :nodoc: + def default_uniqueness_comparison(attribute, value, klass) # :nodoc: + column = column_for_attribute(attribute) + + if column.collation && !column.case_sensitive? && !value.nil? + ActiveSupport::Deprecation.warn(<<~MSG.squish) + Uniqueness validator will no longer enforce case sensitive comparison in Rails 6.1. + To continue case sensitive comparison on the :#{attribute.name} attribute in #{klass} model, + pass `case_sensitive: true` option explicitly to the uniqueness validator. + MSG + attribute.eq(Arel::Nodes::Bin.new(value)) + else + super + end + end + + def case_sensitive_comparison(attribute, value) # :nodoc: + column = column_for_attribute(attribute) + if column.collation && !column.case_sensitive? - table[attribute].eq(Arel::Nodes::Bin.new(value)) + attribute.eq(Arel::Nodes::Bin.new(value)) else super end @@ -528,46 +494,27 @@ module ActiveRecord index.using == :btree || super end - def insert_fixtures_set(fixture_set, tables_to_delete = []) - with_multi_statements do - super { discard_remaining_results } - end - end + def build_insert_sql(insert) # :nodoc: + sql = +"INSERT #{insert.into} #{insert.values_list}" - private - def check_version - if version < "5.5.8" - raise "Your version of MySQL (#{version_string}) is too old. Active Record supports MySQL >= 5.5.8." - end + if insert.skip_duplicates? + no_op_column = quote_column_name(insert.keys.first) + sql << " ON DUPLICATE KEY UPDATE #{no_op_column}=#{no_op_column}" + elsif insert.update_duplicates? + sql << " ON DUPLICATE KEY UPDATE " + sql << insert.updatable_columns.map { |column| "#{column}=VALUES(#{column})" }.join(",") end - def combine_multi_statements(total_sql) - total_sql.each_with_object([]) do |sql, total_sql_chunks| - previous_packet = total_sql_chunks.last - sql << ";\n" - if max_allowed_packet_reached?(sql, previous_packet) || total_sql_chunks.empty? - total_sql_chunks << sql - else - previous_packet << sql - end - end - end - - def max_allowed_packet_reached?(current_packet, previous_packet) - if current_packet.bytesize > max_allowed_packet - raise ActiveRecordError, "Fixtures set is too large #{current_packet.bytesize}. Consider increasing the max_allowed_packet variable." - elsif previous_packet.nil? - false - else - (current_packet.bytesize + previous_packet.bytesize) > max_allowed_packet - end - end + sql + end - def max_allowed_packet - bytes_margin = 2 - @max_allowed_packet ||= (show_variable("max_allowed_packet") - bytes_margin) + def check_version # :nodoc: + if database_version < "5.5.8" + raise "Your version of MySQL (#{database_version}) is too old. Active Record supports MySQL >= 5.5.8." end + end + private def initialize_type_map(m = type_map) super @@ -595,13 +542,13 @@ module ActiveRecord m.alias_type %r(bit)i, "binary" m.register_type(%r(enum)i) do |sql_type| - limit = sql_type[/^enum\((.+)\)/i, 1] + limit = sql_type[/^enum\s*\((.+)\)/i, 1] .split(",").map { |enum| enum.strip.length - 2 }.max MysqlString.new(limit: limit) end m.register_type(%r(^set)i) do |sql_type| - limit = sql_type[/^set\((.+)\)/i, 1] + limit = sql_type[/^set\s*\((.+)\)/i, 1] .split(",").map { |set| set.strip.length - 1 }.sum - 1 MysqlString.new(limit: limit) end @@ -641,37 +588,42 @@ module ActiveRecord ER_LOCK_WAIT_TIMEOUT = 1205 ER_QUERY_INTERRUPTED = 1317 ER_QUERY_TIMEOUT = 3024 + ER_FK_INCOMPATIBLE_COLUMNS = 3780 - def translate_exception(exception, message) + def translate_exception(exception, message:, sql:, binds:) case error_number(exception) when ER_DUP_ENTRY - RecordNotUnique.new(message) + RecordNotUnique.new(message, sql: sql, binds: binds) when ER_NO_REFERENCED_ROW, ER_ROW_IS_REFERENCED, ER_ROW_IS_REFERENCED_2, ER_NO_REFERENCED_ROW_2 - InvalidForeignKey.new(message) - when ER_CANNOT_ADD_FOREIGN - mismatched_foreign_key(message) + InvalidForeignKey.new(message, sql: sql, binds: binds) + when ER_CANNOT_ADD_FOREIGN, ER_FK_INCOMPATIBLE_COLUMNS + mismatched_foreign_key(message, sql: sql, binds: binds) when ER_CANNOT_CREATE_TABLE if message.include?("errno: 150") - mismatched_foreign_key(message) + mismatched_foreign_key(message, sql: sql, binds: binds) else super end when ER_DATA_TOO_LONG - ValueTooLong.new(message) + ValueTooLong.new(message, sql: sql, binds: binds) when ER_OUT_OF_RANGE - RangeError.new(message) + RangeError.new(message, sql: sql, binds: binds) when ER_NOT_NULL_VIOLATION, ER_DO_NOT_HAVE_DEFAULT - NotNullViolation.new(message) + NotNullViolation.new(message, sql: sql, binds: binds) when ER_LOCK_DEADLOCK - Deadlocked.new(message) + Deadlocked.new(message, sql: sql, binds: binds) when ER_LOCK_WAIT_TIMEOUT - LockWaitTimeout.new(message) + LockWaitTimeout.new(message, sql: sql, binds: binds) when ER_QUERY_TIMEOUT - StatementTimeout.new(message) + StatementTimeout.new(message, sql: sql, binds: binds) when ER_QUERY_INTERRUPTED - QueryCanceled.new(message) + QueryCanceled.new(message, sql: sql, binds: binds) else - super + if exception.is_a?(Mysql2::Error::TimeoutError) + ActiveRecord::AdapterTimeout.new(message, sql: sql, binds: binds) + else + super + end end end @@ -722,6 +674,12 @@ module ActiveRecord end def add_timestamps_for_alter(table_name, options = {}) + options[:null] = false if options[:null].nil? + + if !options.key?(:precision) && supports_datetime_with_precision? + options[:precision] = 6 + end + [add_column_for_alter(table_name, :created_at, :datetime, options), add_column_for_alter(table_name, :updated_at, :datetime, options)] end @@ -730,7 +688,7 @@ module ActiveRecord end def supports_rename_index? - mariadb? ? false : version >= "5.7.6" + mariadb? ? false : database_version >= "5.7.6" end def configure_connection @@ -783,7 +741,7 @@ module ActiveRecord end.compact.join(", ") # ...and send them all in one query - execute "SET #{encoding} #{sql_mode_assignment} #{variable_assignments}" + execute("SET #{encoding} #{sql_mode_assignment} #{variable_assignments}", "SCHEMA") end def column_definitions(table_name) # :nodoc: @@ -800,51 +758,36 @@ module ActiveRecord Arel::Visitors::MySQL.new(self) end - def mismatched_foreign_key(message) - parts = message.scan(/`(\w+)`[ $)]/).flatten - MismatchedForeignKey.new( - self, - message: message, - table: parts[0], - foreign_key: parts[1], - target_table: parts[2], - primary_key: parts[3], - ) + def build_statement_pool + StatementPool.new(self.class.type_cast_config_to_integer(@config[:statement_limit])) end - def integer_to_sql(limit) # :nodoc: - case limit - when 1; "tinyint" - when 2; "smallint" - when 3; "mediumint" - when nil, 4; "int" - when 5..8; "bigint" - else raise(ActiveRecordError, "No integer type has byte size #{limit}. Use a decimal with scale 0 instead.") - end - end + def mismatched_foreign_key(message, sql:, binds:) + match = %r/ + (?:CREATE|ALTER)\s+TABLE\s*(?:`?\w+`?\.)?`?(?<table>\w+)`?.+? + FOREIGN\s+KEY\s*\(`?(?<foreign_key>\w+)`?\)\s* + REFERENCES\s*(`?(?<target_table>\w+)`?)\s*\(`?(?<primary_key>\w+)`?\) + /xmi.match(sql) - def text_to_sql(limit) # :nodoc: - case limit - when 0..0xff; "tinytext" - when nil, 0x100..0xffff; "text" - when 0x10000..0xffffff; "mediumtext" - when 0x1000000..0xffffffff; "longtext" - else raise(ActiveRecordError, "No text type has byte length #{limit}") - end - end + options = { + message: message, + sql: sql, + binds: binds, + } - def binary_to_sql(limit) # :nodoc: - case limit - when 0..0xff; "tinyblob" - when nil, 0x100..0xffff; "blob" - when 0x10000..0xffffff; "mediumblob" - when 0x1000000..0xffffffff; "longblob" - else raise(ActiveRecordError, "No binary type has byte length #{limit}") + if match + options[:table] = match[:table] + options[:foreign_key] = match[:foreign_key] + options[:target_table] = match[:target_table] + options[:primary_key] = match[:primary_key] + options[:primary_key_column] = column_for(match[:target_table], match[:primary_key]) end + + MismatchedForeignKey.new(options) end - def version_string - full_version.match(/^(?:5\.5\.5-)?(\d+\.\d+\.\d+)/)[1] + def version_string(full_version_string) + full_version_string.match(/^(?:5\.5\.5-)?(\d+\.\d+\.\d+)/)[1] end class MysqlString < Type::String # :nodoc: @@ -857,7 +800,6 @@ module ActiveRecord end private - def cast_value(value) case value when true then "1" diff --git a/activerecord/lib/active_record/connection_adapters/column.rb b/activerecord/lib/active_record/connection_adapters/column.rb index 5d81de9fe1..2708d2756b 100644 --- a/activerecord/lib/active_record/connection_adapters/column.rb +++ b/activerecord/lib/active_record/connection_adapters/column.rb @@ -5,7 +5,9 @@ module ActiveRecord module ConnectionAdapters # An abstract definition of a column in a table. class Column - attr_reader :name, :default, :sql_type_metadata, :null, :table_name, :default_function, :collation, :comment + include Deduplicable + + attr_reader :name, :default, :sql_type_metadata, :null, :default_function, :collation, :comment delegate :precision, :scale, :limit, :type, :sql_type, to: :sql_type_metadata, allow_nil: true @@ -15,9 +17,8 @@ module ActiveRecord # +default+ is the type-casted default value, such as +new+ in <tt>sales_stage varchar(20) default 'new'</tt>. # +sql_type_metadata+ is various information about the type of the column # +null+ determines if this column allows +NULL+ values. - def initialize(name, default, sql_type_metadata = nil, null = true, table_name = nil, default_function = nil, collation = nil, comment: nil, **) + def initialize(name, default, sql_type_metadata = nil, null = true, default_function = nil, collation: nil, comment: nil, **) @name = name.freeze - @table_name = table_name @sql_type_metadata = sql_type_metadata @null = null @default = default @@ -44,7 +45,6 @@ module ActiveRecord def init_with(coder) @name = coder["name"] - @table_name = coder["table_name"] @sql_type_metadata = coder["sql_type_metadata"] @null = coder["null"] @default = coder["default"] @@ -55,7 +55,6 @@ module ActiveRecord def encode_with(coder) coder["name"] = @name - coder["table_name"] = @table_name coder["sql_type_metadata"] = @sql_type_metadata coder["null"] = @null coder["default"] = @default @@ -66,18 +65,37 @@ module ActiveRecord def ==(other) other.is_a?(Column) && - attributes_for_hash == other.attributes_for_hash + name == other.name && + default == other.default && + sql_type_metadata == other.sql_type_metadata && + null == other.null && + default_function == other.default_function && + collation == other.collation && + comment == other.comment end alias :eql? :== def hash - attributes_for_hash.hash + Column.hash ^ + name.hash ^ + name.encoding.hash ^ + default.hash ^ + sql_type_metadata.hash ^ + null.hash ^ + default_function.hash ^ + collation.hash ^ + comment.hash end - protected - - def attributes_for_hash - [self.class, name, default, sql_type_metadata, null, table_name, default_function, collation] + private + def deduplicated + @name = -name + @sql_type_metadata = sql_type_metadata.deduplicate if sql_type_metadata + @default = -default if default + @default_function = -default_function if default_function + @collation = -collation if collation + @comment = -comment if comment + super end end diff --git a/activerecord/lib/active_record/connection_adapters/connection_specification.rb b/activerecord/lib/active_record/connection_adapters/connection_specification.rb index 2e7a78215a..df26f67c6e 100644 --- a/activerecord/lib/active_record/connection_adapters/connection_specification.rb +++ b/activerecord/lib/active_record/connection_adapters/connection_specification.rb @@ -56,7 +56,6 @@ module ActiveRecord end private - attr_reader :uri def uri_parser @@ -174,12 +173,12 @@ module ActiveRecord if e.path == path_to_adapter # We can assume that a non-builtin adapter was specified, so it's # either misspelled or missing from Gemfile. - raise e.class, "Could not load the '#{spec[:adapter]}' Active Record adapter. Ensure that the adapter is spelled correctly in config/database.yml and that you've added the necessary adapter gem to your Gemfile.", e.backtrace + raise LoadError, "Could not load the '#{spec[:adapter]}' Active Record adapter. Ensure that the adapter is spelled correctly in config/database.yml and that you've added the necessary adapter gem to your Gemfile.", e.backtrace # Bubbled up from the adapter require. Prefix the exception message # with some guidance about how to address it and reraise. else - raise e.class, "Error loading the '#{spec[:adapter]}' Active Record adapter. Missing a gem it depends on? #{e.message}", e.backtrace + raise LoadError, "Error loading the '#{spec[:adapter]}' Active Record adapter. Missing a gem it depends on? #{e.message}", e.backtrace end end @@ -248,10 +247,29 @@ module ActiveRecord if db_config resolve_connection(db_config.config).merge("name" => pool_name.to_s) else - raise(AdapterNotSpecified, "'#{env_name}' database is not configured. Available: #{configurations.configurations.map(&:env_name).join(", ")}") + raise AdapterNotSpecified, <<~MSG + The `#{env_name}` database is not configured for the `#{ActiveRecord::ConnectionHandling::DEFAULT_ENV.call}` environment. + + Available databases configurations are: + + #{build_configuration_sentence} + MSG end end + def build_configuration_sentence # :nodoc: + configs = configurations.configs_for(include_replicas: true) + + configs.group_by(&:env_name).map do |env, config| + namespaces = config.map(&:spec_name) + if namespaces.size > 1 + "#{env}: #{namespaces.join(", ")}" + else + env + end + end.join("\n") + end + # Accepts a hash. Expands the "url" key that contains a # URL database connection to a full connection # hash and merges with the rest of the hash. diff --git a/activerecord/lib/active_record/connection_adapters/deduplicable.rb b/activerecord/lib/active_record/connection_adapters/deduplicable.rb new file mode 100644 index 0000000000..fb2fd60bbc --- /dev/null +++ b/activerecord/lib/active_record/connection_adapters/deduplicable.rb @@ -0,0 +1,29 @@ +# frozen_string_literal: true + +module ActiveRecord + module ConnectionAdapters # :nodoc: + module Deduplicable + extend ActiveSupport::Concern + + module ClassMethods + def registry + @registry ||= {} + end + + def new(*) + super.deduplicate + end + end + + def deduplicate + self.class.registry[self] ||= deduplicated + end + alias :-@ :deduplicate + + private + def deduplicated + freeze + end + end + end +end diff --git a/activerecord/lib/active_record/connection_adapters/determine_if_preparable_visitor.rb b/activerecord/lib/active_record/connection_adapters/determine_if_preparable_visitor.rb index 883747b84b..97d74df529 100644 --- a/activerecord/lib/active_record/connection_adapters/determine_if_preparable_visitor.rb +++ b/activerecord/lib/active_record/connection_adapters/determine_if_preparable_visitor.rb @@ -3,9 +3,9 @@ module ActiveRecord module ConnectionAdapters module DetermineIfPreparableVisitor - attr_reader :preparable + attr_accessor :preparable - def accept(*) + def accept(object, collector) @preparable = true super end @@ -20,7 +20,7 @@ module ActiveRecord super end - def visit_Arel_Nodes_SqlLiteral(*) + def visit_Arel_Nodes_SqlLiteral(o, collector) @preparable = false super end diff --git a/activerecord/lib/active_record/connection_adapters/mysql/database_statements.rb b/activerecord/lib/active_record/connection_adapters/mysql/database_statements.rb index 684c7042a7..bbcdc96cdc 100644 --- a/activerecord/lib/active_record/connection_adapters/mysql/database_statements.rb +++ b/activerecord/lib/active_record/connection_adapters/mysql/database_statements.rb @@ -11,7 +11,7 @@ module ActiveRecord else super end - discard_remaining_results + @connection.abandon_results! result end @@ -19,8 +19,28 @@ module ActiveRecord execute(sql, name).to_a end + READ_QUERY = ActiveRecord::ConnectionAdapters::AbstractAdapter.build_read_query_regexp(:begin, :commit, :explain, :select, :set, :show, :release, :savepoint, :rollback) # :nodoc: + private_constant :READ_QUERY + + def write_query?(sql) # :nodoc: + !READ_QUERY.match?(sql) + end + + def explain(arel, binds = []) + sql = "EXPLAIN #{to_sql(arel, binds)}" + start = Concurrent.monotonic_time + result = exec_query(sql, "EXPLAIN", binds) + elapsed = Concurrent.monotonic_time - start + + MySQL::ExplainPrettyPrinter.new.pp(result, elapsed) + end + # Executes the SQL statement in the context of this connection. def execute(sql, name = nil) + if preventing_writes? && write_query?(sql) + raise ActiveRecord::ReadOnlyError, "Write query attempted while in readonly mode: #{sql}" + end + # make sure we carry over any changes to ActiveRecord::Base.default_timezone that have been # made since we established the connection @connection.query_options[:database_timezone] = ActiveRecord::Base.default_timezone @@ -29,24 +49,30 @@ module ActiveRecord end def exec_query(sql, name = "SQL", binds = [], prepare: false) - materialize_transactions - if without_prepared_statement?(binds) execute_and_free(sql, name) do |result| - ActiveRecord::Result.new(result.fields, result.to_a) if result + if result + ActiveRecord::Result.new(result.fields, result.to_a) + else + ActiveRecord::Result.new([], []) + end end else exec_stmt_and_free(sql, name, binds, cache_stmt: prepare) do |_, result| - ActiveRecord::Result.new(result.fields, result.to_a) if result + if result + ActiveRecord::Result.new(result.fields, result.to_a) + else + ActiveRecord::Result.new([], []) + end end end end def exec_delete(sql, name = nil, binds = []) - materialize_transactions - if without_prepared_statement?(binds) - execute_and_free(sql, name) { @connection.affected_rows } + @lock.synchronize do + execute_and_free(sql, name) { @connection.affected_rows } + end else exec_stmt_and_free(sql, name, binds) { |stmt| stmt.affected_rows } end @@ -54,22 +80,31 @@ module ActiveRecord alias :exec_update :exec_delete private + def execute_batch(sql, name = nil) + super + @connection.abandon_results! + end + def default_insert_value(column) - Arel.sql("DEFAULT") unless column.auto_increment? + super unless column.auto_increment? end def last_inserted_id(result) @connection.last_id end - def discard_remaining_results - @connection.abandon_results! - end - def supports_set_server_option? @connection.respond_to?(:set_server_option) end + def build_truncate_statements(*table_names) + if table_names.size == 1 + super.first + else + super + end + end + def multi_statements_enabled?(flags) if flags.is_a?(Array) flags.include?("MULTI_STATEMENTS") @@ -102,7 +137,40 @@ module ActiveRecord end end + def combine_multi_statements(total_sql) + total_sql.each_with_object([]) do |sql, total_sql_chunks| + previous_packet = total_sql_chunks.last + if max_allowed_packet_reached?(sql, previous_packet) + total_sql_chunks << +sql + else + previous_packet << ";\n" + previous_packet << sql + end + end + end + + def max_allowed_packet_reached?(current_packet, previous_packet) + if current_packet.bytesize > max_allowed_packet + raise ActiveRecordError, + "Fixtures set is too large #{current_packet.bytesize}. Consider increasing the max_allowed_packet variable." + elsif previous_packet.nil? + true + else + (current_packet.bytesize + previous_packet.bytesize + 2) > max_allowed_packet + end + end + + def max_allowed_packet + @max_allowed_packet ||= show_variable("max_allowed_packet") + end + def exec_stmt_and_free(sql, name, binds, cache_stmt: false) + if preventing_writes? && write_query?(sql) + raise ActiveRecord::ReadOnlyError, "Write query attempted while in readonly mode: #{sql}" + end + + materialize_transactions + # make sure we carry over any changes to ActiveRecord::Base.default_timezone that have been # made since we established the connection @connection.query_options[:database_timezone] = ActiveRecord::Base.default_timezone diff --git a/activerecord/lib/active_record/connection_adapters/mysql/explain_pretty_printer.rb b/activerecord/lib/active_record/connection_adapters/mysql/explain_pretty_printer.rb index 20c3c83664..edd5ea0542 100644 --- a/activerecord/lib/active_record/connection_adapters/mysql/explain_pretty_printer.rb +++ b/activerecord/lib/active_record/connection_adapters/mysql/explain_pretty_printer.rb @@ -37,7 +37,6 @@ module ActiveRecord end private - def compute_column_widths(result) [].tap do |widths| result.columns.each_with_index do |column, i| diff --git a/activerecord/lib/active_record/connection_adapters/mysql/quoting.rb b/activerecord/lib/active_record/connection_adapters/mysql/quoting.rb index 75564a61d6..0069f5871c 100644 --- a/activerecord/lib/active_record/connection_adapters/mysql/quoting.rb +++ b/activerecord/lib/active_record/connection_adapters/mysql/quoting.rb @@ -5,11 +5,11 @@ module ActiveRecord module MySQL module Quoting # :nodoc: def quote_column_name(name) - @quoted_column_names[name] ||= "`#{super.gsub('`', '``')}`" + self.class.quoted_column_names[name] ||= "`#{super.gsub('`', '``')}`" end def quote_table_name(name) - @quoted_table_names[name] ||= super.gsub(".", "`.`").freeze + self.class.quoted_table_names[name] ||= super.gsub(".", "`.`").freeze end def unquoted_true @@ -32,12 +32,49 @@ module ActiveRecord "x'#{value.hex}'" end - def _type_cast(value) - case value - when Date, Time then value - else super - end + def column_name_matcher + COLUMN_NAME + end + + def column_name_with_order_matcher + COLUMN_NAME_WITH_ORDER end + + COLUMN_NAME = / + \A + ( + (?: + # `table_name`.`column_name` | function(one or no argument) + ((?:\w+\.|`\w+`\.)?(?:\w+|`\w+`)) | \w+\((?:|\g<2>)\) + ) + (?:(?:\s+AS)?\s+(?:\w+|`\w+`))? + ) + (?:\s*,\s*\g<1>)* + \z + /ix + + COLUMN_NAME_WITH_ORDER = / + \A + ( + (?: + # `table_name`.`column_name` | function(one or no argument) + ((?:\w+\.|`\w+`\.)?(?:\w+|`\w+`)) | \w+\((?:|\g<2>)\) + ) + (?:\s+ASC|\s+DESC)? + ) + (?:\s*,\s*\g<1>)* + \z + /ix + + private_constant :COLUMN_NAME, :COLUMN_NAME_WITH_ORDER + + private + def _type_cast(value) + case value + when Date, Time then value + else super + end + end end end end diff --git a/activerecord/lib/active_record/connection_adapters/mysql/schema_creation.rb b/activerecord/lib/active_record/connection_adapters/mysql/schema_creation.rb index 82ed320617..0f5ab7562a 100644 --- a/activerecord/lib/active_record/connection_adapters/mysql/schema_creation.rb +++ b/activerecord/lib/active_record/connection_adapters/mysql/schema_creation.rb @@ -7,7 +7,6 @@ module ActiveRecord delegate :add_sql_comment!, :mariadb?, to: :@conn, private: true private - def visit_DropForeignKey(name) "DROP FOREIGN KEY #{name}" end diff --git a/activerecord/lib/active_record/connection_adapters/mysql/schema_definitions.rb b/activerecord/lib/active_record/connection_adapters/mysql/schema_definitions.rb index 2ed4ad16ae..d21535a709 100644 --- a/activerecord/lib/active_record/connection_adapters/mysql/schema_definitions.rb +++ b/activerecord/lib/active_record/connection_adapters/mysql/schema_definitions.rb @@ -4,48 +4,56 @@ module ActiveRecord module ConnectionAdapters module MySQL module ColumnMethods - def blob(*args, **options) - args.each { |name| column(name, :blob, options) } - end + extend ActiveSupport::Concern - def tinyblob(*args, **options) - args.each { |name| column(name, :tinyblob, options) } - end + ## + # :method: blob + # :call-seq: blob(*names, **options) - def mediumblob(*args, **options) - args.each { |name| column(name, :mediumblob, options) } - end + ## + # :method: tinyblob + # :call-seq: tinyblob(*names, **options) - def longblob(*args, **options) - args.each { |name| column(name, :longblob, options) } - end + ## + # :method: mediumblob + # :call-seq: mediumblob(*names, **options) - def tinytext(*args, **options) - args.each { |name| column(name, :tinytext, options) } - end + ## + # :method: longblob + # :call-seq: longblob(*names, **options) - def mediumtext(*args, **options) - args.each { |name| column(name, :mediumtext, options) } - end + ## + # :method: tinytext + # :call-seq: tinytext(*names, **options) - def longtext(*args, **options) - args.each { |name| column(name, :longtext, options) } - end + ## + # :method: mediumtext + # :call-seq: mediumtext(*names, **options) - def unsigned_integer(*args, **options) - args.each { |name| column(name, :unsigned_integer, options) } - end + ## + # :method: longtext + # :call-seq: longtext(*names, **options) - def unsigned_bigint(*args, **options) - args.each { |name| column(name, :unsigned_bigint, options) } - end + ## + # :method: unsigned_integer + # :call-seq: unsigned_integer(*names, **options) - def unsigned_float(*args, **options) - args.each { |name| column(name, :unsigned_float, options) } - end + ## + # :method: unsigned_bigint + # :call-seq: unsigned_bigint(*names, **options) + + ## + # :method: unsigned_float + # :call-seq: unsigned_float(*names, **options) + + ## + # :method: unsigned_decimal + # :call-seq: unsigned_decimal(*names, **options) - def unsigned_decimal(*args, **options) - args.each { |name| column(name, :unsigned_decimal, options) } + included do + define_column_methods :blob, :tinyblob, :mediumblob, :longblob, + :tinytext, :mediumtext, :longtext, :unsigned_integer, :unsigned_bigint, + :unsigned_float, :unsigned_decimal end end diff --git a/activerecord/lib/active_record/connection_adapters/mysql/schema_dumper.rb b/activerecord/lib/active_record/connection_adapters/mysql/schema_dumper.rb index d23178e43c..bcd300f3db 100644 --- a/activerecord/lib/active_record/connection_adapters/mysql/schema_dumper.rb +++ b/activerecord/lib/active_record/connection_adapters/mysql/schema_dumper.rb @@ -10,6 +10,10 @@ module ActiveRecord spec[:unsigned] = "true" if column.unsigned? spec[:auto_increment] = "true" if column.auto_increment? + if /\A(?<size>tiny|medium|long)(?:text|blob)/ =~ column.sql_type + spec = { size: size.to_sym.inspect }.merge!(spec) + end + if @connection.supports_virtual_columns? && column.virtual? spec[:as] = extract_expression_for_virtual_column(column) spec[:stored] = "true" if /\b(?:STORED|PERSISTENT)\b/.match?(column.extra) @@ -37,19 +41,23 @@ module ActiveRecord case column.sql_type when /\Atimestamp\b/ :timestamp - when "tinyblob" - :blob + when /\A(?:enum|set)\b/ + column.sql_type else super end end + def schema_limit(column) + super unless /\A(?:enum|set|(?:tiny|medium|long)?(?:text|blob))\b/.match?(column.sql_type) + end + def schema_precision(column) super unless /\A(?:date)?time(?:stamp)?\b/.match?(column.sql_type) && column.precision == 0 end def schema_collation(column) - if column.collation && table_name = column.table_name + if column.collation @table_collation_cache ||= {} @table_collation_cache[table_name] ||= @connection.exec_query("SHOW TABLE STATUS LIKE #{@connection.quote(table_name)}", "SCHEMA").first["Collation"] @@ -58,14 +66,14 @@ module ActiveRecord end def extract_expression_for_virtual_column(column) - if @connection.mariadb? && @connection.version < "10.2.5" - create_table_info = @connection.send(:create_table_info, column.table_name) + if @connection.mariadb? && @connection.database_version < "10.2.5" + create_table_info = @connection.send(:create_table_info, table_name) column_name = @connection.quote_column_name(column.name) if %r/#{column_name} #{Regexp.quote(column.sql_type)}(?: COLLATE \w+)? AS \((?<expression>.+?)\) #{column.extra}/ =~ create_table_info $~[:expression].inspect end else - scope = @connection.send(:quoted_scope, column.table_name) + scope = @connection.send(:quoted_scope, table_name) column_name = @connection.quote(column.name) sql = "SELECT generation_expression FROM information_schema.columns" \ " WHERE table_schema = #{scope[:schema]}" \ diff --git a/activerecord/lib/active_record/connection_adapters/mysql/schema_statements.rb b/activerecord/lib/active_record/connection_adapters/mysql/schema_statements.rb index 4894fd1c08..25a1fb234a 100644 --- a/activerecord/lib/active_record/connection_adapters/mysql/schema_statements.rb +++ b/activerecord/lib/active_record/connection_adapters/mysql/schema_statements.rb @@ -77,9 +77,13 @@ module ActiveRecord super end + def create_table(table_name, options: default_row_format, **) + super + end + def internal_string_options_for_primary_key super.tap do |options| - if CHARSETS_OF_4BYTES_MAXLEN.include?(charset) && (mariadb? || version < "8.0.0") + if !row_format_dynamic_by_default? && CHARSETS_OF_4BYTES_MAXLEN.include?(charset) options[:collation] = collation.sub(/\A[^_]+/, "utf8") end end @@ -93,15 +97,65 @@ module ActiveRecord MySQL::SchemaDumper.create(self, options) end + # Maps logical Rails types to MySQL-specific data types. + def type_to_sql(type, limit: nil, precision: nil, scale: nil, size: limit_to_size(limit, type), unsigned: nil, **) + sql = + case type.to_s + when "integer" + integer_to_sql(limit) + when "text" + type_with_size_to_sql("text", size) + when "blob" + type_with_size_to_sql("blob", size) + when "binary" + if (0..0xfff) === limit + "varbinary(#{limit})" + else + type_with_size_to_sql("blob", size) + end + else + super + end + + sql = "#{sql} unsigned" if unsigned && type != :primary_key + sql + end + + def table_alias_length + 256 # https://dev.mysql.com/doc/refman/8.0/en/identifiers.html + end + private CHARSETS_OF_4BYTES_MAXLEN = ["utf8mb4", "utf16", "utf16le", "utf32"] + def row_format_dynamic_by_default? + if mariadb? + database_version >= "10.2.2" + else + database_version >= "5.7.9" + end + end + + def default_row_format + return if row_format_dynamic_by_default? + + unless defined?(@default_row_format) + if query_value("SELECT @@innodb_file_per_table = 1 AND @@innodb_file_format = 'Barracuda'") == 1 + @default_row_format = "ROW_FORMAT=DYNAMIC" + else + @default_row_format = nil + end + end + + @default_row_format + end + def schema_creation MySQL::SchemaCreation.new(self) end def create_table_definition(*args) - MySQL::TableDefinition.new(*args) + MySQL::TableDefinition.new(self, *args) end def new_column_from_field(table_name, field) @@ -120,9 +174,8 @@ module ActiveRecord default, type_metadata, field[:Null] == "YES", - table_name, default_function, - field[:Collation], + collation: field[:Collation], comment: field[:Comment].presence ) end @@ -171,6 +224,40 @@ module ActiveRecord schema, name = nil, schema unless name [schema, name] end + + def type_with_size_to_sql(type, size) + case size&.to_s + when nil, "tiny", "medium", "long" + "#{size}#{type}" + else + raise ArgumentError, + "#{size.inspect} is invalid :size value. Only :tiny, :medium, and :long are allowed." + end + end + + def limit_to_size(limit, type) + case type.to_s + when "text", "blob", "binary" + case limit + when 0..0xff; "tiny" + when nil, 0x100..0xffff; nil + when 0x10000..0xffffff; "medium" + when 0x1000000..0xffffffff; "long" + else raise ArgumentError, "No #{type} type has byte size #{limit}" + end + end + end + + def integer_to_sql(limit) + case limit + when 1; "tinyint" + when 2; "smallint" + when 3; "mediumint" + when nil, 4; "int" + when 5..8; "bigint" + else raise ArgumentError, "No integer type has byte size #{limit}. Use a decimal with scale 0 instead." + end + end end end end diff --git a/activerecord/lib/active_record/connection_adapters/mysql/type_metadata.rb b/activerecord/lib/active_record/connection_adapters/mysql/type_metadata.rb index 7ad0944d51..a7232fa249 100644 --- a/activerecord/lib/active_record/connection_adapters/mysql/type_metadata.rb +++ b/activerecord/lib/active_record/connection_adapters/mysql/type_metadata.rb @@ -6,28 +6,33 @@ module ActiveRecord class TypeMetadata < DelegateClass(SqlTypeMetadata) # :nodoc: undef to_yaml if method_defined?(:to_yaml) + include Deduplicable + attr_reader :extra - def initialize(type_metadata, extra: "") + def initialize(type_metadata, extra: nil) super(type_metadata) - @type_metadata = type_metadata @extra = extra end def ==(other) - other.is_a?(MySQL::TypeMetadata) && - attributes_for_hash == other.attributes_for_hash + other.is_a?(TypeMetadata) && + __getobj__ == other.__getobj__ && + extra == other.extra end alias eql? == def hash - attributes_for_hash.hash + TypeMetadata.hash ^ + __getobj__.hash ^ + extra.hash end - protected - - def attributes_for_hash - [self.class, @type_metadata, extra] + private + def deduplicated + __setobj__(__getobj__.deduplicate) + @extra = -extra if extra + super end end end diff --git a/activerecord/lib/active_record/connection_adapters/mysql2_adapter.rb b/activerecord/lib/active_record/connection_adapters/mysql2_adapter.rb index 9bdaa00336..1df9ac32c9 100644 --- a/activerecord/lib/active_record/connection_adapters/mysql2_adapter.rb +++ b/activerecord/lib/active_record/connection_adapters/mysql2_adapter.rb @@ -8,6 +8,8 @@ require "mysql2" module ActiveRecord module ConnectionHandling # :nodoc: + ER_BAD_DB_ERROR = 1049 + # Establishes a connection to the database that's used by all Active Record objects. def mysql2_connection(config) config = config.symbolize_keys @@ -22,7 +24,7 @@ module ActiveRecord client = Mysql2::Client.new(config) ConnectionAdapters::Mysql2Adapter.new(client, logger, nil, config) rescue Mysql2::Error => error - if error.message.include?("Unknown database") + if error.error_number == ER_BAD_DB_ERROR raise ActiveRecord::NoDatabaseError else raise @@ -42,8 +44,14 @@ module ActiveRecord configure_connection end + def self.database_exists?(config) + !!ActiveRecord::Base.mysql2_connection(config) + rescue ActiveRecord::NoDatabaseError + false + end + def supports_json? - !mariadb? && version >= "5.7.8" + !mariadb? && database_version >= "5.7.8" end def supports_comments? @@ -109,12 +117,12 @@ module ActiveRecord end def discard! # :nodoc: + super @connection.automatic_close = false @connection = nil end private - def connect @connection = Mysql2::Client.new(@config) configure_connection @@ -126,7 +134,11 @@ module ActiveRecord end def full_version - @full_version ||= @connection.server_info[:version] + schema_cache.database_version.full_version_string + end + + def get_full_version + @connection.server_info[:version] end end end diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/column.rb b/activerecord/lib/active_record/connection_adapters/postgresql/column.rb index 3ccc7271ab..f1ecf6df30 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql/column.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql/column.rb @@ -2,42 +2,52 @@ module ActiveRecord module ConnectionAdapters - # PostgreSQL-specific extensions to column definitions in a table. - class PostgreSQLColumn < Column #:nodoc: - delegate :array, :oid, :fmod, to: :sql_type_metadata - alias :array? :array - - def initialize(*, max_identifier_length: 63, **) - super - @max_identifier_length = max_identifier_length - end + module PostgreSQL + class Column < ConnectionAdapters::Column # :nodoc: + delegate :oid, :fmod, to: :sql_type_metadata - def serial? - return unless default_function + def initialize(*, serial: nil, **) + super + @serial = serial + end - if %r{\Anextval\('"?(?<sequence_name>.+_(?<suffix>seq\d*))"?'::regclass\)\z} =~ default_function - sequence_name_from_parts(table_name, name, suffix) == sequence_name + def serial? + @serial end - end - private - attr_reader :max_identifier_length + def array + sql_type_metadata.sql_type.end_with?("[]") + end + alias :array? :array + + def sql_type + super.sub(/\[\]\z/, "") + end - def sequence_name_from_parts(table_name, column_name, suffix) - over_length = [table_name, column_name, suffix].map(&:length).sum + 2 - max_identifier_length + def init_with(coder) + @serial = coder["serial"] + super + end - if over_length > 0 - column_name_length = [(max_identifier_length - suffix.length - 2) / 2, column_name.length].min - over_length -= column_name.length - column_name_length - column_name = column_name[0, column_name_length - [over_length, 0].min] - end + def encode_with(coder) + coder["serial"] = @serial + super + end - if over_length > 0 - table_name = table_name[0, table_name.length - over_length] - end + def ==(other) + other.is_a?(Column) && + super && + serial? == other.serial? + end + alias :eql? :== - "#{table_name}_#{column_name}_#{suffix}" + def hash + Column.hash ^ + super.hash ^ + serial?.hash end + end end + PostgreSQLColumn = PostgreSQL::Column # :nodoc: end end diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/database_statements.rb b/activerecord/lib/active_record/connection_adapters/postgresql/database_statements.rb index 6bd6b67165..45ec79ca78 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql/database_statements.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql/database_statements.rb @@ -67,11 +67,22 @@ module ActiveRecord end end + READ_QUERY = ActiveRecord::ConnectionAdapters::AbstractAdapter.build_read_query_regexp(:begin, :commit, :explain, :select, :set, :show, :release, :savepoint, :rollback) # :nodoc: + private_constant :READ_QUERY + + def write_query?(sql) # :nodoc: + !READ_QUERY.match?(sql) + end + # Executes an SQL statement, returning a PG::Result object on success # or raising a PG::Error exception otherwise. # Note: the PG::Result object is manually memory managed; if you don't # need it specifically, you may want consider the <tt>exec_query</tt> wrapper. def execute(sql, name = nil) + if preventing_writes? && write_query?(sql) + raise ActiveRecord::ReadOnlyError, "Write query attempted while in readonly mode: #{sql}" + end + materialize_transactions log(sql, name) do @@ -99,7 +110,7 @@ module ActiveRecord end alias :exec_update :exec_delete - def sql_for_insert(sql, pk, id_value, sequence_name, binds) # :nodoc: + def sql_for_insert(sql, pk, binds) # :nodoc: if pk.nil? # Extract the table from the insert sql. Yuck. table_ref = extract_table_ref_from_insert_sql(sql) @@ -134,7 +145,7 @@ module ActiveRecord # Begins a transaction. def begin_db_transaction - execute "BEGIN" + execute("BEGIN", "TRANSACTION") end def begin_isolated_db_transaction(isolation) @@ -144,15 +155,19 @@ module ActiveRecord # Commits a transaction. def commit_db_transaction - execute "COMMIT" + execute("COMMIT", "TRANSACTION") end # Aborts a transaction. def exec_rollback_db_transaction - execute "ROLLBACK" + execute("ROLLBACK", "TRANSACTION") end private + def build_truncate_statements(*table_names) + "TRUNCATE TABLE #{table_names.map(&method(:quote_table_name)).join(", ")}" + end + # Returns the current ID of a table's sequence. def last_insert_id_result(sequence_name) exec_query("SELECT currval(#{quote(sequence_name)})", "SQL") diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/oid/array.rb b/activerecord/lib/active_record/connection_adapters/postgresql/oid/array.rb index 6fbeaa2b9e..0bbe98145a 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql/oid/array.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql/oid/array.rb @@ -5,7 +5,7 @@ module ActiveRecord module PostgreSQL module OID # :nodoc: class Array < Type::Value # :nodoc: - include Type::Helpers::Mutable + include ActiveModel::Type::Helpers::Mutable Data = Struct.new(:encoder, :values) # :nodoc: @@ -77,7 +77,6 @@ module ActiveRecord end private - def type_cast_array(value, method) if value.is_a?(::Array) value.map { |item| type_cast_array(item, method) } diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/oid/enum.rb b/activerecord/lib/active_record/connection_adapters/postgresql/oid/enum.rb index f70f09ad95..bae34472e1 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql/oid/enum.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql/oid/enum.rb @@ -10,7 +10,6 @@ module ActiveRecord end private - def cast_value(value) value.to_s end diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/oid/hstore.rb b/activerecord/lib/active_record/connection_adapters/postgresql/oid/hstore.rb index aabe83b85d..8d4dacbd64 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql/oid/hstore.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql/oid/hstore.rb @@ -5,7 +5,7 @@ module ActiveRecord module PostgreSQL module OID # :nodoc: class Hstore < Type::Value # :nodoc: - include Type::Helpers::Mutable + include ActiveModel::Type::Helpers::Mutable def type :hstore @@ -46,7 +46,6 @@ module ActiveRecord end private - HstorePair = begin quoted_string = /"[^"\\]*(?:\\.[^"\\]*)*"/ unquoted_string = /(?:\\.|[^\s,])[^\s=,\\]*(?:\\.[^\s=,\\]*|=[^,>])*/ diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/oid/legacy_point.rb b/activerecord/lib/active_record/connection_adapters/postgresql/oid/legacy_point.rb index 7b057a8452..e52d4385ef 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql/oid/legacy_point.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql/oid/legacy_point.rb @@ -5,7 +5,7 @@ module ActiveRecord module PostgreSQL module OID # :nodoc: class LegacyPoint < Type::Value # :nodoc: - include Type::Helpers::Mutable + include ActiveModel::Type::Helpers::Mutable def type :point @@ -34,7 +34,6 @@ module ActiveRecord end private - def number_for_point(number) number.to_s.gsub(/\.0$/, "") end diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/oid/money.rb b/activerecord/lib/active_record/connection_adapters/postgresql/oid/money.rb index 6434377b57..357493dfc0 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql/oid/money.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql/oid/money.rb @@ -26,9 +26,9 @@ module ActiveRecord value = value.sub(/^\((.+)\)$/, '-\1') # (4) case value - when /^-?\D+[\d,]+\.\d{2}$/ # (1) + when /^-?\D*[\d,]+\.\d{2}$/ # (1) value.gsub!(/[^-\d.]/, "") - when /^-?\D+[\d.]+,\d{2}$/ # (2) + when /^-?\D*[\d.]+,\d{2}$/ # (2) value.gsub!(/[^-\d,]/, "").sub!(/,/, ".") end diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/oid/point.rb b/activerecord/lib/active_record/connection_adapters/postgresql/oid/point.rb index 02a9c506f6..e81e18ff70 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql/oid/point.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql/oid/point.rb @@ -7,7 +7,7 @@ module ActiveRecord module PostgreSQL module OID # :nodoc: class Point < Type::Value # :nodoc: - include Type::Helpers::Mutable + include ActiveModel::Type::Helpers::Mutable def type :point @@ -50,7 +50,6 @@ module ActiveRecord end private - def number_for_point(number) number.to_s.gsub(/\.0$/, "") end diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/oid/range.rb b/activerecord/lib/active_record/connection_adapters/postgresql/oid/range.rb index d85f9ab3ef..d19f1f9cf8 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql/oid/range.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql/oid/range.rb @@ -58,13 +58,12 @@ module ActiveRecord end private - def type_cast_single(value) infinity?(value) ? value : @subtype.deserialize(value) end def type_cast_single_for_database(value) - infinity?(value) ? value : @subtype.serialize(value) + infinity?(value) ? value : @subtype.serialize(@subtype.cast(value)) end def extract_bounds(value) diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/oid/type_map_initializer.rb b/activerecord/lib/active_record/connection_adapters/postgresql/oid/type_map_initializer.rb index 83c21ba6ea..203087bc36 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql/oid/type_map_initializer.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql/oid/type_map_initializer.rb @@ -36,7 +36,7 @@ module ActiveRecord def query_conditions_for_initial_load known_type_names = @store.keys.map { |n| "'#{n}'" } known_type_types = %w('r' 'e' 'd') - <<-SQL % [known_type_names.join(", "), known_type_types.join(", ")] + <<~SQL % [known_type_names.join(", "), known_type_types.join(", ")] WHERE t.typname IN (%s) OR t.typtype IN (%s) diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/oid/uuid.rb b/activerecord/lib/active_record/connection_adapters/postgresql/oid/uuid.rb index bc9b8dbfcf..74a28eef58 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql/oid/uuid.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql/oid/uuid.rb @@ -13,9 +13,11 @@ module ActiveRecord :uuid end - def cast(value) - value.to_s[ACCEPTABLE_UUID, 0] - end + private + def cast_value(value) + casted = value.to_s + casted if casted.match?(ACCEPTABLE_UUID) + end end end end diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/quoting.rb b/activerecord/lib/active_record/connection_adapters/postgresql/quoting.rb index 0895d06356..07b66de366 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql/quoting.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql/quoting.rb @@ -30,7 +30,7 @@ module ActiveRecord # - "schema.name".table_name # - "schema.name"."table.name" def quote_table_name(name) # :nodoc: - @quoted_table_names[name] ||= Utils.extract_schema_qualified_name(name.to_s).quoted.freeze + self.class.quoted_table_names[name] ||= Utils.extract_schema_qualified_name(name.to_s).quoted.freeze end # Quotes schema names for use in SQL queries. @@ -44,7 +44,7 @@ module ActiveRecord # Quotes column names for use in SQL queries. def quote_column_name(name) # :nodoc: - @quoted_column_names[name] ||= PG::Connection.quote_ident(super).freeze + self.class.quoted_column_names[name] ||= PG::Connection.quote_ident(super).freeze end # Quote date/time values for use in SQL input. @@ -78,6 +78,43 @@ module ActiveRecord type_map.lookup(column.oid, column.fmod, column.sql_type) end + def column_name_matcher + COLUMN_NAME + end + + def column_name_with_order_matcher + COLUMN_NAME_WITH_ORDER + end + + COLUMN_NAME = / + \A + ( + (?: + # "table_name"."column_name"::type_name | function(one or no argument)::type_name + ((?:\w+\.|"\w+"\.)?(?:\w+|"\w+")(?:::\w+)?) | \w+\((?:|\g<2>)\)(?:::\w+)? + ) + (?:(?:\s+AS)?\s+(?:\w+|"\w+"))? + ) + (?:\s*,\s*\g<1>)* + \z + /ix + + COLUMN_NAME_WITH_ORDER = / + \A + ( + (?: + # "table_name"."column_name"::type_name | function(one or no argument)::type_name + ((?:\w+\.|"\w+"\.)?(?:\w+|"\w+")(?:::\w+)?) | \w+\((?:|\g<2>)\)(?:::\w+)? + ) + (?:\s+ASC|\s+DESC)? + (?:\s+NULLS\s+(?:FIRST|LAST))? + ) + (?:\s*,\s*\g<1>)* + \z + /ix + + private_constant :COLUMN_NAME, :COLUMN_NAME_WITH_ORDER + private def lookup_cast_type(sql_type) super(query_value("SELECT #{quote(sql_type)}::regtype::oid", "SCHEMA").to_i) @@ -138,7 +175,7 @@ module ActiveRecord end def encode_range(range) - "[#{type_cast_range_value(range.first)},#{type_cast_range_value(range.last)}#{range.exclude_end? ? ')' : ']'}" + "[#{type_cast_range_value(range.begin)},#{type_cast_range_value(range.end)}#{range.exclude_end? ? ')' : ']'}" end def determine_encoding_of_strings_in_array(value) diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/schema_creation.rb b/activerecord/lib/active_record/connection_adapters/postgresql/schema_creation.rb index ceb8b40bd9..84dd28907b 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql/schema_creation.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql/schema_creation.rb @@ -17,6 +17,42 @@ module ActiveRecord "VALIDATE CONSTRAINT #{quote_column_name(name)}" end + def visit_ChangeColumnDefinition(o) + column = o.column + column.sql_type = type_to_sql(column.type, column.options) + quoted_column_name = quote_column_name(o.name) + + change_column_sql = +"ALTER COLUMN #{quoted_column_name} TYPE #{column.sql_type}" + + options = column_options(column) + + if options[:collation] + change_column_sql << " COLLATE \"#{options[:collation]}\"" + end + + if options[:using] + change_column_sql << " USING #{options[:using]}" + elsif options[:cast_as] + cast_as_type = type_to_sql(options[:cast_as], options) + change_column_sql << " USING CAST(#{quoted_column_name} AS #{cast_as_type})" + end + + if options.key?(:default) + if options[:default].nil? + change_column_sql << ", ALTER COLUMN #{quoted_column_name} DROP DEFAULT" + else + quoted_default = quote_default_expression(options[:default], column) + change_column_sql << ", ALTER COLUMN #{quoted_column_name} SET DEFAULT #{quoted_default}" + end + end + + if options.key?(:null) + change_column_sql << ", ALTER COLUMN #{quoted_column_name} #{options[:null] ? 'DROP' : 'SET'} NOT NULL" + end + + change_column_sql + end + def add_column_options!(sql, options) if options[:collation] sql << " COLLATE \"#{options[:collation]}\"" diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/schema_definitions.rb b/activerecord/lib/active_record/connection_adapters/postgresql/schema_definitions.rb index dc4a0bb26e..3bb7c52899 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql/schema_definitions.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql/schema_definitions.rb @@ -4,6 +4,8 @@ module ActiveRecord module ConnectionAdapters module PostgreSQL module ColumnMethods + extend ActiveSupport::Concern + # Defines the primary key field. # Use of the native PostgreSQL UUID type is supported, and can be used # by defining your tables as such: @@ -51,124 +53,131 @@ module ActiveRecord super end - def bigserial(*args, **options) - args.each { |name| column(name, :bigserial, options) } - end + ## + # :method: bigserial + # :call-seq: bigserial(*names, **options) - def bit(*args, **options) - args.each { |name| column(name, :bit, options) } - end + ## + # :method: bit + # :call-seq: bit(*names, **options) - def bit_varying(*args, **options) - args.each { |name| column(name, :bit_varying, options) } - end + ## + # :method: bit_varying + # :call-seq: bit_varying(*names, **options) - def cidr(*args, **options) - args.each { |name| column(name, :cidr, options) } - end + ## + # :method: cidr + # :call-seq: cidr(*names, **options) - def citext(*args, **options) - args.each { |name| column(name, :citext, options) } - end + ## + # :method: citext + # :call-seq: citext(*names, **options) - def daterange(*args, **options) - args.each { |name| column(name, :daterange, options) } - end + ## + # :method: daterange + # :call-seq: daterange(*names, **options) - def hstore(*args, **options) - args.each { |name| column(name, :hstore, options) } - end + ## + # :method: hstore + # :call-seq: hstore(*names, **options) - def inet(*args, **options) - args.each { |name| column(name, :inet, options) } - end + ## + # :method: inet + # :call-seq: inet(*names, **options) - def interval(*args, **options) - args.each { |name| column(name, :interval, options) } - end + ## + # :method: interval + # :call-seq: interval(*names, **options) - def int4range(*args, **options) - args.each { |name| column(name, :int4range, options) } - end + ## + # :method: int4range + # :call-seq: int4range(*names, **options) - def int8range(*args, **options) - args.each { |name| column(name, :int8range, options) } - end + ## + # :method: int8range + # :call-seq: int8range(*names, **options) - def jsonb(*args, **options) - args.each { |name| column(name, :jsonb, options) } - end + ## + # :method: jsonb + # :call-seq: jsonb(*names, **options) - def ltree(*args, **options) - args.each { |name| column(name, :ltree, options) } - end + ## + # :method: ltree + # :call-seq: ltree(*names, **options) - def macaddr(*args, **options) - args.each { |name| column(name, :macaddr, options) } - end + ## + # :method: macaddr + # :call-seq: macaddr(*names, **options) - def money(*args, **options) - args.each { |name| column(name, :money, options) } - end + ## + # :method: money + # :call-seq: money(*names, **options) - def numrange(*args, **options) - args.each { |name| column(name, :numrange, options) } - end + ## + # :method: numrange + # :call-seq: numrange(*names, **options) - def oid(*args, **options) - args.each { |name| column(name, :oid, options) } - end + ## + # :method: oid + # :call-seq: oid(*names, **options) - def point(*args, **options) - args.each { |name| column(name, :point, options) } - end + ## + # :method: point + # :call-seq: point(*names, **options) - def line(*args, **options) - args.each { |name| column(name, :line, options) } - end + ## + # :method: line + # :call-seq: line(*names, **options) - def lseg(*args, **options) - args.each { |name| column(name, :lseg, options) } - end + ## + # :method: lseg + # :call-seq: lseg(*names, **options) - def box(*args, **options) - args.each { |name| column(name, :box, options) } - end + ## + # :method: box + # :call-seq: box(*names, **options) - def path(*args, **options) - args.each { |name| column(name, :path, options) } - end + ## + # :method: path + # :call-seq: path(*names, **options) - def polygon(*args, **options) - args.each { |name| column(name, :polygon, options) } - end + ## + # :method: polygon + # :call-seq: polygon(*names, **options) - def circle(*args, **options) - args.each { |name| column(name, :circle, options) } - end + ## + # :method: circle + # :call-seq: circle(*names, **options) - def serial(*args, **options) - args.each { |name| column(name, :serial, options) } - end + ## + # :method: serial + # :call-seq: serial(*names, **options) - def tsrange(*args, **options) - args.each { |name| column(name, :tsrange, options) } - end + ## + # :method: tsrange + # :call-seq: tsrange(*names, **options) - def tstzrange(*args, **options) - args.each { |name| column(name, :tstzrange, options) } - end + ## + # :method: tstzrange + # :call-seq: tstzrange(*names, **options) - def tsvector(*args, **options) - args.each { |name| column(name, :tsvector, options) } - end + ## + # :method: tsvector + # :call-seq: tsvector(*names, **options) - def uuid(*args, **options) - args.each { |name| column(name, :uuid, options) } - end + ## + # :method: uuid + # :call-seq: uuid(*names, **options) + + ## + # :method: xml + # :call-seq: xml(*names, **options) - def xml(*args, **options) - args.each { |name| column(name, :xml, options) } + included do + define_column_methods :bigserial, :bit, :bit_varying, :cidr, :citext, :daterange, + :hstore, :inet, :interval, :int4range, :int8range, :jsonb, :ltree, :macaddr, + :money, :numrange, :oid, :point, :line, :lseg, :box, :path, :polygon, :circle, + :serial, :tsrange, :tstzrange, :tsvector, :uuid, :xml end end diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/schema_dumper.rb b/activerecord/lib/active_record/connection_adapters/postgresql/schema_dumper.rb index 84643d20da..d201e40190 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql/schema_dumper.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql/schema_dumper.rb @@ -5,7 +5,6 @@ module ActiveRecord module PostgreSQL class SchemaDumper < ConnectionAdapters::SchemaDumper # :nodoc: private - def extensions(stream) extensions = @connection.extensions if extensions.any? diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/schema_statements.rb b/activerecord/lib/active_record/connection_adapters/postgresql/schema_statements.rb index fae3ddbad4..0062952667 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql/schema_statements.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql/schema_statements.rb @@ -22,8 +22,8 @@ module ActiveRecord def create_database(name, options = {}) options = { encoding: "utf8" }.merge!(options.symbolize_keys) - option_string = options.inject("") do |memo, (key, value)| - memo += case key + option_string = options.each_with_object(+"") do |(key, value), memo| + memo << case key when :owner " OWNER = \"#{value}\"" when :template @@ -55,6 +55,7 @@ module ActiveRecord end def drop_table(table_name, options = {}) # :nodoc: + schema_cache.clear_data_source_cache!(table_name.to_s) execute "DROP TABLE#{' IF EXISTS' if options[:if_exists]} #{quote_table_name(table_name)}#{' CASCADE' if options[:force] == :cascade}" end @@ -68,7 +69,7 @@ module ActiveRecord table = quoted_scope(table_name) index = quoted_scope(index_name) - query_value(<<-SQL, "SCHEMA").to_i > 0 + query_value(<<~SQL, "SCHEMA").to_i > 0 SELECT COUNT(*) FROM pg_class t INNER JOIN pg_index d ON t.oid = d.indrelid @@ -85,7 +86,7 @@ module ActiveRecord def indexes(table_name) # :nodoc: scope = quoted_scope(table_name) - result = query(<<-SQL, "SCHEMA") + result = query(<<~SQL, "SCHEMA") SELECT distinct i.relname, d.indisunique, d.indkey, pg_get_indexdef(d.indexrelid), t.oid, pg_catalog.obj_description(i.oid, 'pg_class') AS comment FROM pg_class t @@ -124,7 +125,7 @@ module ActiveRecord # add info on sort order (only desc order is explicitly specified, asc is the default) # and non-default opclasses - expressions.scan(/(?<column>\w+)\s?(?<opclass>\w+_ops)?\s?(?<desc>DESC)?\s?(?<nulls>NULLS (?:FIRST|LAST))?/).each do |column, opclass, desc, nulls| + expressions.scan(/(?<column>\w+)"?\s?(?<opclass>\w+_ops)?\s?(?<desc>DESC)?\s?(?<nulls>NULLS (?:FIRST|LAST))?/).each do |column, opclass, desc, nulls| opclasses[column] = opclass.to_sym if opclass if nulls orders[column] = [desc, nulls].compact.join(" ") @@ -196,7 +197,7 @@ module ActiveRecord # Returns an array of schema names. def schema_names - query_values(<<-SQL, "SCHEMA") + query_values(<<~SQL, "SCHEMA") SELECT nspname FROM pg_namespace WHERE nspname !~ '^pg_.*' @@ -287,7 +288,7 @@ module ActiveRecord quoted_sequence = quote_table_name(sequence) max_pk = query_value("SELECT MAX(#{quote_column_name pk}) FROM #{quote_table_name(table)}", "SCHEMA") if max_pk.nil? - if postgresql_version >= 100000 + if database_version >= 100000 minvalue = query_value("SELECT seqmin FROM pg_sequence WHERE seqrelid = #{quote(quoted_sequence)}::regclass", "SCHEMA") else minvalue = query_value("SELECT min_value FROM #{quoted_sequence}", "SCHEMA") @@ -302,7 +303,7 @@ module ActiveRecord def pk_and_sequence_for(table) #:nodoc: # First try looking for a sequence with a dependency on the # given table's primary key. - result = query(<<-end_sql, "SCHEMA")[0] + result = query(<<~SQL, "SCHEMA")[0] SELECT attr.attname, nsp.nspname, seq.relname FROM pg_class seq, pg_attribute attr, @@ -319,10 +320,10 @@ module ActiveRecord AND cons.contype = 'p' AND dep.classid = 'pg_class'::regclass AND dep.refobjid = #{quote(quote_table_name(table))}::regclass - end_sql + SQL if result.nil? || result.empty? - result = query(<<-end_sql, "SCHEMA")[0] + result = query(<<~SQL, "SCHEMA")[0] SELECT attr.attname, nsp.nspname, CASE WHEN pg_get_expr(def.adbin, def.adrelid) !~* 'nextval' THEN NULL @@ -339,7 +340,7 @@ module ActiveRecord WHERE t.oid = #{quote(quote_table_name(table))}::regclass AND cons.contype = 'p' AND pg_get_expr(def.adbin, def.adrelid) ~* 'nextval|uuid_generate' - end_sql + SQL end pk = result.shift @@ -368,31 +369,6 @@ module ActiveRecord SQL end - def bulk_change_table(table_name, operations) - sql_fragments = [] - non_combinable_operations = [] - - operations.each do |command, args| - table, arguments = args.shift, args - method = :"#{command}_for_alter" - - if respond_to?(method, true) - sqls, procs = Array(send(method, table, *arguments)).partition { |v| v.is_a?(String) } - sql_fragments << sqls - non_combinable_operations.concat(procs) - else - execute "ALTER TABLE #{quote_table_name(table_name)} #{sql_fragments.join(", ")}" unless sql_fragments.empty? - non_combinable_operations.each(&:call) - sql_fragments = [] - non_combinable_operations = [] - send(command, table, *arguments) - end - end - - execute "ALTER TABLE #{quote_table_name(table_name)} #{sql_fragments.join(", ")}" unless sql_fragments.empty? - non_combinable_operations.each(&:call) - end - # Renames a table. # Also renames a table's primary key sequence if the sequence name exists and # matches the Active Record default. @@ -401,6 +377,8 @@ module ActiveRecord # rename_table('octopuses', 'octopi') def rename_table(table_name, new_name) clear_cache! + schema_cache.clear_data_source_cache!(table_name.to_s) + schema_cache.clear_data_source_cache!(new_name.to_s) execute "ALTER TABLE #{quote_table_name(table_name)} RENAME TO #{quote_table_name(new_name)}" pk, seq = pk_and_sequence_for(new_name) if pk @@ -443,14 +421,16 @@ module ActiveRecord end # Adds comment for given table column or drops it if +comment+ is a +nil+ - def change_column_comment(table_name, column_name, comment) # :nodoc: + def change_column_comment(table_name, column_name, comment_or_changes) # :nodoc: clear_cache! + comment = extract_new_comment_value(comment_or_changes) execute "COMMENT ON COLUMN #{quote_table_name(table_name)}.#{quote_column_name(column_name)} IS #{quote(comment)}" end # Adds comment for given table or drops it if +comment+ is a +nil+ - def change_table_comment(table_name, comment) # :nodoc: + def change_table_comment(table_name, comment_or_changes) # :nodoc: clear_cache! + comment = extract_new_comment_value(comment_or_changes) execute "COMMENT ON TABLE #{quote_table_name(table_name)} IS #{quote(comment)}" end @@ -548,21 +528,21 @@ module ActiveRecord # The hard limit is 1GB, because of a 32-bit size field, and TOAST. case limit when nil, 0..0x3fffffff; super(type) - else raise(ActiveRecordError, "No binary type has byte size #{limit}.") + else raise ArgumentError, "No binary type has byte size #{limit}. The limit on binary can be at most 1GB - 1byte." end when "text" # PostgreSQL doesn't support limits on text columns. # The hard limit is 1GB, according to section 8.3 in the manual. case limit when nil, 0..0x3fffffff; super(type) - else raise(ActiveRecordError, "The limit on text can be at most 1GB - 1byte.") + else raise ArgumentError, "No text type has byte size #{limit}. The limit on text can be at most 1GB - 1byte." end when "integer" case limit when 1, 2; "smallint" when nil, 3, 4; "integer" when 5..8; "bigint" - else raise(ActiveRecordError, "No integer type has byte size #{limit}. Use a numeric with scale 0 instead.") + else raise ArgumentError, "No integer type has byte size #{limit}. Use a numeric with scale 0 instead." end else super @@ -623,10 +603,10 @@ module ActiveRecord # validate_foreign_key :accounts, name: :special_fk_name # # The +options+ hash accepts the same keys as SchemaStatements#add_foreign_key. - def validate_foreign_key(from_table, options_or_to_table = {}) + def validate_foreign_key(from_table, to_table = nil, **options) return unless supports_validate_constraints? - fk_name_to_validate = foreign_key_for!(from_table, options_or_to_table).name + fk_name_to_validate = foreign_key_for!(from_table, to_table: to_table, **options).name validate_constraint from_table, fk_name_to_validate end @@ -637,7 +617,7 @@ module ActiveRecord end def create_table_definition(*args) - PostgreSQL::TableDefinition.new(*args) + PostgreSQL::TableDefinition.new(self, *args) end def create_alter_table(name) @@ -650,16 +630,19 @@ module ActiveRecord default_value = extract_value_from_default(default) default_function = extract_default_function(default_value, default) - PostgreSQLColumn.new( + if match = default_function&.match(/\Anextval\('"?(?<sequence_name>.+_(?<suffix>seq\d*))"?'::regclass\)\z/) + serial = sequence_name_from_parts(table_name, column_name, match[:suffix]) == match[:sequence_name] + end + + PostgreSQL::Column.new( column_name, default_value, type_metadata, !notnull, - table_name, default_function, - collation, + collation: collation, comment: comment.presence, - max_identifier_length: max_identifier_length + serial: serial ) end @@ -672,7 +655,23 @@ module ActiveRecord precision: cast_type.precision, scale: cast_type.scale, ) - PostgreSQLTypeMetadata.new(simple_type, oid: oid, fmod: fmod) + PostgreSQL::TypeMetadata.new(simple_type, oid: oid, fmod: fmod) + end + + def sequence_name_from_parts(table_name, column_name, suffix) + over_length = [table_name, column_name, suffix].sum(&:length) + 2 - max_identifier_length + + if over_length > 0 + column_name_length = [(max_identifier_length - suffix.length - 2) / 2, column_name.length].min + over_length -= column_name.length - column_name_length + column_name = column_name[0, column_name_length - [over_length, 0].min] + end + + if over_length > 0 + table_name = table_name[0, table_name.length - over_length] + end + + "#{table_name}_#{column_name}_#{suffix}" end def extract_foreign_key_action(specifier) @@ -683,38 +682,20 @@ module ActiveRecord end end - def change_column_sql(table_name, column_name, type, options = {}) - quoted_column_name = quote_column_name(column_name) - sql_type = type_to_sql(type, options) - sql = +"ALTER COLUMN #{quoted_column_name} TYPE #{sql_type}" - if options[:collation] - sql << " COLLATE \"#{options[:collation]}\"" - end - if options[:using] - sql << " USING #{options[:using]}" - elsif options[:cast_as] - cast_as_type = type_to_sql(options[:cast_as], options) - sql << " USING CAST(#{quoted_column_name} AS #{cast_as_type})" - end - - sql - end - def add_column_for_alter(table_name, column_name, type, options = {}) return super unless options.key?(:comment) [super, Proc.new { change_column_comment(table_name, column_name, options[:comment]) }] end def change_column_for_alter(table_name, column_name, type, options = {}) - sqls = [change_column_sql(table_name, column_name, type, options)] - sqls << change_column_default_for_alter(table_name, column_name, options[:default]) if options.key?(:default) - sqls << change_column_null_for_alter(table_name, column_name, options[:null], options[:default]) if options.key?(:null) + td = create_table_definition(table_name) + cd = td.new_column_definition(column_name, type, options) + sqls = [schema_creation.accept(ChangeColumnDefinition.new(cd, column_name))] sqls << Proc.new { change_column_comment(table_name, column_name, options[:comment]) } if options.key?(:comment) sqls end - # Changes the default value of a table column. - def change_column_default_for_alter(table_name, column_name, default_or_changes) # :nodoc: + def change_column_default_for_alter(table_name, column_name, default_or_changes) column = column_for(table_name, column_name) return unless column @@ -729,11 +710,17 @@ module ActiveRecord end end - def change_column_null_for_alter(table_name, column_name, null, default = nil) #:nodoc: - "ALTER #{quote_column_name(column_name)} #{null ? 'DROP' : 'SET'} NOT NULL" + def change_column_null_for_alter(table_name, column_name, null, default = nil) + "ALTER COLUMN #{quote_column_name(column_name)} #{null ? 'DROP' : 'SET'} NOT NULL" end def add_timestamps_for_alter(table_name, options = {}) + options[:null] = false if options[:null].nil? + + if !options.key?(:precision) && supports_datetime_with_precision? + options[:precision] = 6 + end + [add_column_for_alter(table_name, :created_at, :datetime, options), add_column_for_alter(table_name, :updated_at, :datetime, options)] end diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/type_metadata.rb b/activerecord/lib/active_record/connection_adapters/postgresql/type_metadata.rb index cd69d28139..b7f6479357 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql/type_metadata.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql/type_metadata.rb @@ -3,38 +3,42 @@ module ActiveRecord # :stopdoc: module ConnectionAdapters - class PostgreSQLTypeMetadata < DelegateClass(SqlTypeMetadata) - undef to_yaml if method_defined?(:to_yaml) + module PostgreSQL + class TypeMetadata < DelegateClass(SqlTypeMetadata) + undef to_yaml if method_defined?(:to_yaml) - attr_reader :oid, :fmod, :array + include Deduplicable - def initialize(type_metadata, oid: nil, fmod: nil) - super(type_metadata) - @type_metadata = type_metadata - @oid = oid - @fmod = fmod - @array = /\[\]$/.match?(type_metadata.sql_type) - end - - def sql_type - super.gsub(/\[\]$/, "") - end - - def ==(other) - other.is_a?(PostgreSQLTypeMetadata) && - attributes_for_hash == other.attributes_for_hash - end - alias eql? == + attr_reader :oid, :fmod - def hash - attributes_for_hash.hash - end + def initialize(type_metadata, oid: nil, fmod: nil) + super(type_metadata) + @oid = oid + @fmod = fmod + end - protected + def ==(other) + other.is_a?(TypeMetadata) && + __getobj__ == other.__getobj__ && + oid == other.oid && + fmod == other.fmod + end + alias eql? == - def attributes_for_hash - [self.class, @type_metadata, oid, fmod] + def hash + TypeMetadata.hash ^ + __getobj__.hash ^ + oid.hash ^ + fmod.hash end + + private + def deduplicated + __setobj__(__getobj__.deduplicate) + super + end + end end + PostgreSQLTypeMetadata = PostgreSQL::TypeMetadata end end diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/utils.rb b/activerecord/lib/active_record/connection_adapters/postgresql/utils.rb index bfd300723d..e8caeb8132 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql/utils.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql/utils.rb @@ -37,7 +37,6 @@ module ActiveRecord end protected - def parts @parts ||= [@schema, @identifier].compact end @@ -68,7 +67,7 @@ module ActiveRecord # * <tt>"schema_name".table_name</tt> # * <tt>"schema.name"."table name"</tt> def extract_schema_qualified_name(string) - schema, table = string.scan(/[^".\s]+|"[^"]*"/) + schema, table = string.scan(/[^".]+|"[^"]*"/) if table.nil? table = schema schema = nil diff --git a/activerecord/lib/active_record/connection_adapters/postgresql_adapter.rb b/activerecord/lib/active_record/connection_adapters/postgresql_adapter.rb index d2ed699ee2..0a7c6d8ac4 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql_adapter.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql_adapter.rb @@ -46,7 +46,7 @@ module ActiveRecord conn = PG.connect(conn_params) ConnectionAdapters::PostgreSQLAdapter.new(conn, logger, conn_params, config) rescue ::PG::Error => error - if error.message.include?("does not exist") + if error.message.include?(conn_params[:dbname]) raise ActiveRecord::NoDatabaseError else raise @@ -185,7 +185,7 @@ module ActiveRecord end def supports_json? - postgresql_version >= 90200 + true end def supports_comments? @@ -196,6 +196,17 @@ module ActiveRecord true end + def supports_insert_returning? + true + end + + def supports_insert_on_conflict? + database_version >= 90500 + end + alias supports_insert_on_duplicate_skip? supports_insert_on_conflict? + alias supports_insert_on_duplicate_update? supports_insert_on_conflict? + alias supports_insert_conflict_target? supports_insert_on_conflict? + def index_algorithms { concurrently: "CONCURRENTLY" } end @@ -240,9 +251,6 @@ module ActiveRecord configure_connection add_pg_encoders - @statements = StatementPool.new @connection, - self.class.type_cast_config_to_integer(config[:statement_limit]) - add_pg_decoders @type_map = Type::HashLookupTypeMap.new @@ -251,15 +259,10 @@ module ActiveRecord @use_insert_returning = @config.key?(:insert_returning) ? self.class.type_cast_config_to_boolean(@config[:insert_returning]) : true end - # Clears the prepared statements cache. - def clear_cache! - @lock.synchronize do - @statements.clear - end - end - - def truncate(table_name, name = nil) - exec_query "TRUNCATE TABLE #{quote_table_name(table_name)}", name, [] + def self.database_exists?(config) + !!ActiveRecord::Base.postgresql_connection(config) + rescue ActiveRecord::NoDatabaseError + false end # Is this connection alive and ready for queries? @@ -278,6 +281,8 @@ module ActiveRecord super @connection.reset configure_connection + rescue PG::ConnectionBad + connect end end @@ -303,6 +308,7 @@ module ActiveRecord end def discard! # :nodoc: + super @connection.socket_io.reopen(IO::NULL) rescue nil @connection = nil end @@ -332,20 +338,27 @@ module ActiveRecord end def supports_ranges? - # Range datatypes weren't introduced until PostgreSQL 9.2 - postgresql_version >= 90200 + true end + deprecate :supports_ranges? def supports_materialized_views? - postgresql_version >= 90300 + true end def supports_foreign_tables? - postgresql_version >= 90300 + true end def supports_pgcrypto_uuid? - postgresql_version >= 90400 + database_version >= 90400 + end + + def supports_optimizer_hints? + unless defined?(@has_pg_hint_plan) + @has_pg_hint_plan = extension_available?("pg_hint_plan") + end + @has_pg_hint_plan end def supports_lazy_transactions? @@ -378,9 +391,12 @@ module ActiveRecord } end + def extension_available?(name) + query_value("SELECT true FROM pg_available_extensions WHERE name = #{quote(name)}", "SCHEMA") + end + def extension_enabled?(name) - res = exec_query("SELECT EXISTS(SELECT * FROM pg_available_extensions WHERE name = '#{name}' AND installed_version IS NOT NULL) as enabled", "SCHEMA") - res.cast_values.first + query_value("SELECT installed_version IS NOT NULL FROM pg_available_extensions WHERE name = #{quote(name)}", "SCHEMA") end def extensions @@ -391,8 +407,6 @@ module ActiveRecord def max_identifier_length @max_identifier_length ||= query_value("SHOW max_identifier_length", "SCHEMA").to_i end - alias table_alias_length max_identifier_length - alias index_name_length max_identifier_length # Set the authorized user for this session def session_auth=(user) @@ -415,21 +429,36 @@ module ActiveRecord } # Returns the version of the connected PostgreSQL server. - def postgresql_version + def get_database_version # :nodoc: @connection.server_version end + alias :postgresql_version :database_version def default_index_type?(index) # :nodoc: index.using == :btree || super end - private - def check_version - if postgresql_version < 90100 - raise "Your version of PostgreSQL (#{postgresql_version}) is too old. Active Record supports PostgreSQL >= 9.1." - end + def build_insert_sql(insert) # :nodoc: + sql = +"INSERT #{insert.into} #{insert.values_list}" + + if insert.skip_duplicates? + sql << " ON CONFLICT #{insert.conflict_target} DO NOTHING" + elsif insert.update_duplicates? + sql << " ON CONFLICT #{insert.conflict_target} DO UPDATE SET " + sql << insert.updatable_columns.map { |column| "#{column}=excluded.#{column}" }.join(",") end + sql << " RETURNING #{insert.returning}" if insert.returning + sql + end + + def check_version # :nodoc: + if database_version < 90300 + raise "Your version of PostgreSQL (#{database_version}) is too old. Active Record supports PostgreSQL >= 9.3." + end + end + + private # See https://www.postgresql.org/docs/current/static/errcodes-appendix.html VALUE_LIMIT_VIOLATION = "22001" NUMERIC_VALUE_OUT_OF_RANGE = "22003" @@ -441,28 +470,28 @@ module ActiveRecord LOCK_NOT_AVAILABLE = "55P03" QUERY_CANCELED = "57014" - def translate_exception(exception, message) + def translate_exception(exception, message:, sql:, binds:) return exception unless exception.respond_to?(:result) case exception.result.try(:error_field, PG::PG_DIAG_SQLSTATE) when UNIQUE_VIOLATION - RecordNotUnique.new(message) + RecordNotUnique.new(message, sql: sql, binds: binds) when FOREIGN_KEY_VIOLATION - InvalidForeignKey.new(message) + InvalidForeignKey.new(message, sql: sql, binds: binds) when VALUE_LIMIT_VIOLATION - ValueTooLong.new(message) + ValueTooLong.new(message, sql: sql, binds: binds) when NUMERIC_VALUE_OUT_OF_RANGE - RangeError.new(message) + RangeError.new(message, sql: sql, binds: binds) when NOT_NULL_VIOLATION - NotNullViolation.new(message) + NotNullViolation.new(message, sql: sql, binds: binds) when SERIALIZATION_FAILURE - SerializationFailure.new(message) + SerializationFailure.new(message, sql: sql, binds: binds) when DEADLOCK_DETECTED - Deadlocked.new(message) + Deadlocked.new(message, sql: sql, binds: binds) when LOCK_NOT_AVAILABLE - LockWaitTimeout.new(message) + LockWaitTimeout.new(message, sql: sql, binds: binds) when QUERY_CANCELED - QueryCanceled.new(message) + QueryCanceled.new(message, sql: sql, binds: binds) else super end @@ -589,18 +618,11 @@ module ActiveRecord def load_additional_types(oids = nil) initializer = OID::TypeMapInitializer.new(type_map) - if supports_ranges? - query = <<-SQL - SELECT t.oid, t.typname, t.typelem, t.typdelim, t.typinput, r.rngsubtype, t.typtype, t.typbasetype - FROM pg_type as t - LEFT JOIN pg_range as r ON oid = rngtypid - SQL - else - query = <<-SQL - SELECT t.oid, t.typname, t.typelem, t.typdelim, t.typinput, t.typtype, t.typbasetype - FROM pg_type as t - SQL - end + query = <<~SQL + SELECT t.oid, t.typname, t.typelem, t.typdelim, t.typinput, r.rngsubtype, t.typtype, t.typbasetype + FROM pg_type as t + LEFT JOIN pg_range as r ON oid = rngtypid + SQL if oids query += "WHERE t.oid::integer IN (%s)" % oids.join(", ") @@ -616,6 +638,10 @@ module ActiveRecord FEATURE_NOT_SUPPORTED = "0A000" #:nodoc: def execute_and_clear(sql, name, binds, prepare: false) + if preventing_writes? && write_query?(sql) + raise ActiveRecord::ReadOnlyError, "Write query attempted while in readonly mode: #{sql}" + end + if without_prepared_statement?(binds) result = exec_no_cache(sql, name, []) elsif !prepare @@ -631,6 +657,10 @@ module ActiveRecord def exec_no_cache(sql, name, binds) materialize_transactions + # make sure we carry over any changes to ActiveRecord::Base.default_timezone that have been + # made since we established the connection + update_typemap_for_default_timezone + type_casted_binds = type_casted_binds(binds) log(sql, name, binds, type_casted_binds) do ActiveSupport::Dependencies.interlock.permit_concurrent_loads do @@ -641,8 +671,9 @@ module ActiveRecord def exec_cache(sql, name, binds) materialize_transactions + update_typemap_for_default_timezone - stmt_key = prepare_statement(sql) + stmt_key = prepare_statement(sql, binds) type_casted_binds = type_casted_binds(binds) log(sql, name, binds, type_casted_binds, stmt_key) do @@ -696,7 +727,7 @@ module ActiveRecord # Prepare the statement if it hasn't been prepared, return # the statement key. - def prepare_statement(sql) + def prepare_statement(sql, binds) @lock.synchronize do sql_key = sql_key(sql) unless @statements.key? sql_key @@ -704,7 +735,7 @@ module ActiveRecord begin @connection.prepare nextkey, sql rescue => e - raise translate_exception_class(e, sql) + raise translate_exception_class(e, sql, binds) end # Clear the queue @connection.get_last_result @@ -719,6 +750,8 @@ module ActiveRecord def connect @connection = PG.connect(@connection_parameters) configure_connection + add_pg_encoders + add_pg_decoders end # Configures the encoding, verbosity, schema search path, and time zone of the connection. @@ -776,7 +809,7 @@ module ActiveRecord # - format_type includes the column size constraint, e.g. varchar(50) # - ::regclass is a function that gives the id for a table name def column_definitions(table_name) - query(<<-end_sql, "SCHEMA") + query(<<~SQL, "SCHEMA") SELECT a.attname, format_type(a.atttypid, a.atttypmod), pg_get_expr(d.adbin, d.adrelid), a.attnotnull, a.atttypid, a.atttypmod, c.collname, col_description(a.attrelid, a.attnum) AS comment @@ -787,7 +820,7 @@ module ActiveRecord WHERE a.attrelid = #{quote(quote_table_name(table_name))}::regclass AND a.attnum > 0 AND NOT a.attisdropped ORDER BY a.attnum - end_sql + SQL end def extract_table_ref_from_insert_sql(sql) @@ -799,10 +832,14 @@ module ActiveRecord Arel::Visitors::PostgreSQL.new(self) end + def build_statement_pool + StatementPool.new(@connection, self.class.type_cast_config_to_integer(@config[:statement_limit])) + end + def can_perform_case_insensitive_comparison_for?(column) @case_insensitive_cache ||= {} @case_insensitive_cache[column.sql_type] ||= begin - sql = <<-end_sql + sql = <<~SQL SELECT exists( SELECT * FROM pg_proc WHERE proname = 'lower' @@ -814,7 +851,7 @@ module ActiveRecord WHERE proname = 'lower' AND castsource = #{quote column.sql_type}::regtype ) - end_sql + SQL execute_and_clear(sql, "SCHEMA", []) do |result| result.getvalue(0, 0) end @@ -829,7 +866,22 @@ module ActiveRecord @connection.type_map_for_queries = map end + def update_typemap_for_default_timezone + if @default_timezone != ActiveRecord::Base.default_timezone && @timestamp_decoder + decoder_class = ActiveRecord::Base.default_timezone == :utc ? + PG::TextDecoder::TimestampUtc : + PG::TextDecoder::TimestampWithoutTimeZone + + @timestamp_decoder = decoder_class.new(@timestamp_decoder.to_h) + @connection.type_map_for_results.add_coder(@timestamp_decoder) + @default_timezone = ActiveRecord::Base.default_timezone + end + end + def add_pg_decoders + @default_timezone = nil + @timestamp_decoder = nil + coders_by_name = { "int2" => PG::TextDecoder::Integer, "int4" => PG::TextDecoder::Integer, @@ -839,8 +891,15 @@ module ActiveRecord "float8" => PG::TextDecoder::Float, "bool" => PG::TextDecoder::Boolean, } + + if defined?(PG::TextDecoder::TimestampUtc) + # Use native PG encoders available since pg-1.1 + coders_by_name["timestamp"] = PG::TextDecoder::TimestampUtc + coders_by_name["timestamptz"] = PG::TextDecoder::TimestampWithTimeZone + end + known_coder_types = coders_by_name.keys.map { |n| quote(n) } - query = <<-SQL % known_coder_types.join(", ") + query = <<~SQL % known_coder_types.join(", ") SELECT t.oid, t.typname FROM pg_type as t WHERE t.typname IN (%s) @@ -854,6 +913,10 @@ module ActiveRecord map = PG::TypeMapByOid.new coders.each { |coder| map.add_coder(coder) } @connection.type_map_for_results = map + + # extract timestamp decoder for use in update_typemap_for_default_timezone + @timestamp_decoder = coders.find { |coder| coder.name == "timestamp" } + update_typemap_for_default_timezone end def construct_coder(row, coder_class) diff --git a/activerecord/lib/active_record/connection_adapters/schema_cache.rb b/activerecord/lib/active_record/connection_adapters/schema_cache.rb index c29cf1f9a1..7d54fcf9a0 100644 --- a/activerecord/lib/active_record/connection_adapters/schema_cache.rb +++ b/activerecord/lib/active_record/connection_adapters/schema_cache.rb @@ -13,6 +13,7 @@ module ActiveRecord @columns_hash = {} @primary_keys = {} @data_sources = {} + @indexes = {} end def initialize_dup(other) @@ -21,22 +22,27 @@ module ActiveRecord @columns_hash = @columns_hash.dup @primary_keys = @primary_keys.dup @data_sources = @data_sources.dup + @indexes = @indexes.dup end def encode_with(coder) - coder["columns"] = @columns - coder["columns_hash"] = @columns_hash - coder["primary_keys"] = @primary_keys - coder["data_sources"] = @data_sources - coder["version"] = connection.migration_context.current_version + coder["columns"] = @columns + coder["primary_keys"] = @primary_keys + coder["data_sources"] = @data_sources + coder["indexes"] = @indexes + coder["version"] = connection.migration_context.current_version + coder["database_version"] = database_version end def init_with(coder) - @columns = coder["columns"] - @columns_hash = coder["columns_hash"] - @primary_keys = coder["primary_keys"] - @data_sources = coder["data_sources"] - @version = coder["version"] + @columns = coder["columns"] + @primary_keys = coder["primary_keys"] + @data_sources = coder["data_sources"] + @indexes = coder["indexes"] || {} + @version = coder["version"] + @database_version = coder["database_version"] + + derive_columns_hash_and_deduplicate_values end def primary_keys(table_name) @@ -57,6 +63,7 @@ module ActiveRecord primary_keys(table_name) columns(table_name) columns_hash(table_name) + indexes(table_name) end end @@ -72,9 +79,20 @@ module ActiveRecord # Get the columns for a table as a hash, key is the column name # value is the column object. def columns_hash(table_name) - @columns_hash[table_name] ||= Hash[columns(table_name).map { |col| - [col.name, col] - }] + @columns_hash[table_name] ||= columns(table_name).index_by(&:name) + end + + # Checks whether the columns hash is already cached for a table. + def columns_hash?(table_name) + @columns_hash.key?(table_name) + end + + def indexes(table_name) + @indexes[table_name] ||= connection.indexes(table_name) + end + + def database_version # :nodoc: + @database_version ||= connection.get_database_version end # Clears out internal caches @@ -83,11 +101,13 @@ module ActiveRecord @columns_hash.clear @primary_keys.clear @data_sources.clear + @indexes.clear @version = nil + @database_version = nil end def size - [@columns, @columns_hash, @primary_keys, @data_sources].map(&:size).inject :+ + [@columns, @columns_hash, @primary_keys, @data_sources].sum(&:size) end # Clear out internal caches for the data source +name+. @@ -96,19 +116,43 @@ module ActiveRecord @columns_hash.delete name @primary_keys.delete name @data_sources.delete name + @indexes.delete name end def marshal_dump # if we get current version during initialization, it happens stack over flow. @version = connection.migration_context.current_version - [@version, @columns, @columns_hash, @primary_keys, @data_sources] + [@version, @columns, {}, @primary_keys, @data_sources, @indexes, database_version] end def marshal_load(array) - @version, @columns, @columns_hash, @primary_keys, @data_sources = array + @version, @columns, _columns_hash, @primary_keys, @data_sources, @indexes, @database_version = array + @indexes ||= {} + + derive_columns_hash_and_deduplicate_values end private + def derive_columns_hash_and_deduplicate_values + @columns = deep_deduplicate(@columns) + @columns_hash = @columns.transform_values { |columns| columns.index_by(&:name) } + @primary_keys = deep_deduplicate(@primary_keys) + @data_sources = deep_deduplicate(@data_sources) + @indexes = deep_deduplicate(@indexes) + end + + def deep_deduplicate(value) + case value + when Hash + value.transform_keys { |k| deep_deduplicate(k) }.transform_values { |v| deep_deduplicate(v) } + when Array + value.map { |i| deep_deduplicate(i) } + when String, Deduplicable + -value + else + value + end + end def prepare_data_sources connection.data_sources.each { |source| @data_sources[source] = true } diff --git a/activerecord/lib/active_record/connection_adapters/sql_type_metadata.rb b/activerecord/lib/active_record/connection_adapters/sql_type_metadata.rb index 8489bcbf1d..969867e70f 100644 --- a/activerecord/lib/active_record/connection_adapters/sql_type_metadata.rb +++ b/activerecord/lib/active_record/connection_adapters/sql_type_metadata.rb @@ -1,9 +1,13 @@ # frozen_string_literal: true +require "active_record/connection_adapters/deduplicable" + module ActiveRecord # :stopdoc: module ConnectionAdapters class SqlTypeMetadata + include Deduplicable + attr_reader :sql_type, :type, :limit, :precision, :scale def initialize(sql_type: nil, type: nil, limit: nil, precision: nil, scale: nil) @@ -16,18 +20,27 @@ module ActiveRecord def ==(other) other.is_a?(SqlTypeMetadata) && - attributes_for_hash == other.attributes_for_hash + sql_type == other.sql_type && + type == other.type && + limit == other.limit && + precision == other.precision && + scale == other.scale end alias eql? == def hash - attributes_for_hash.hash + SqlTypeMetadata.hash ^ + sql_type.hash ^ + type.hash ^ + limit.hash ^ + precision.hash >> 1 ^ + scale.hash >> 2 end - protected - - def attributes_for_hash - [self.class, sql_type, type, limit, precision, scale] + private + def deduplicated + @sql_type = -sql_type + super end end end diff --git a/activerecord/lib/active_record/connection_adapters/sqlite3/database_statements.rb b/activerecord/lib/active_record/connection_adapters/sqlite3/database_statements.rb new file mode 100644 index 0000000000..85053acf91 --- /dev/null +++ b/activerecord/lib/active_record/connection_adapters/sqlite3/database_statements.rb @@ -0,0 +1,122 @@ +# frozen_string_literal: true + +module ActiveRecord + module ConnectionAdapters + module SQLite3 + module DatabaseStatements + READ_QUERY = ActiveRecord::ConnectionAdapters::AbstractAdapter.build_read_query_regexp(:begin, :commit, :explain, :select, :pragma, :release, :savepoint, :rollback) # :nodoc: + private_constant :READ_QUERY + + def write_query?(sql) # :nodoc: + !READ_QUERY.match?(sql) + end + + def explain(arel, binds = []) + sql = "EXPLAIN QUERY PLAN #{to_sql(arel, binds)}" + SQLite3::ExplainPrettyPrinter.new.pp(exec_query(sql, "EXPLAIN", [])) + end + + def execute(sql, name = nil) #:nodoc: + if preventing_writes? && write_query?(sql) + raise ActiveRecord::ReadOnlyError, "Write query attempted while in readonly mode: #{sql}" + end + + materialize_transactions + + log(sql, name) do + ActiveSupport::Dependencies.interlock.permit_concurrent_loads do + @connection.execute(sql) + end + end + end + + def exec_query(sql, name = nil, binds = [], prepare: false) + if preventing_writes? && write_query?(sql) + raise ActiveRecord::ReadOnlyError, "Write query attempted while in readonly mode: #{sql}" + end + + materialize_transactions + + type_casted_binds = type_casted_binds(binds) + + log(sql, name, binds, type_casted_binds) do + ActiveSupport::Dependencies.interlock.permit_concurrent_loads do + # Don't cache statements if they are not prepared + unless prepare + stmt = @connection.prepare(sql) + begin + cols = stmt.columns + unless without_prepared_statement?(binds) + stmt.bind_params(type_casted_binds) + end + records = stmt.to_a + ensure + stmt.close + end + else + stmt = @statements[sql] ||= @connection.prepare(sql) + cols = stmt.columns + stmt.reset! + stmt.bind_params(type_casted_binds) + records = stmt.to_a + end + + ActiveRecord::Result.new(cols, records) + end + end + end + + def exec_delete(sql, name = "SQL", binds = []) + exec_query(sql, name, binds) + @connection.changes + end + alias :exec_update :exec_delete + + def begin_db_transaction #:nodoc: + log("begin transaction", "TRANSACTION") { @connection.transaction } + end + + def commit_db_transaction #:nodoc: + log("commit transaction", "TRANSACTION") { @connection.commit } + end + + def exec_rollback_db_transaction #:nodoc: + log("rollback transaction", "TRANSACTION") { @connection.rollback } + end + + private + def execute_batch(sql, name = nil) + if preventing_writes? && write_query?(sql) + raise ActiveRecord::ReadOnlyError, "Write query attempted while in readonly mode: #{sql}" + end + + materialize_transactions + + log(sql, name) do + ActiveSupport::Dependencies.interlock.permit_concurrent_loads do + @connection.execute_batch2(sql) + end + end + end + + def last_inserted_id(result) + @connection.last_insert_row_id + end + + def build_fixture_statements(fixture_set) + fixture_set.flat_map do |table_name, fixtures| + next if fixtures.empty? + fixtures.map { |fixture| build_fixture_sql([fixture], table_name) } + end.compact + end + + def build_truncate_statements(*table_names) + truncate_tables = table_names.map do |table_name| + "DELETE FROM #{quote_table_name(table_name)}" + end + combine_multi_statements(truncate_tables) + end + end + end + end +end diff --git a/activerecord/lib/active_record/connection_adapters/sqlite3/quoting.rb b/activerecord/lib/active_record/connection_adapters/sqlite3/quoting.rb index b2dcdb5373..9b74a774e5 100644 --- a/activerecord/lib/active_record/connection_adapters/sqlite3/quoting.rb +++ b/activerecord/lib/active_record/connection_adapters/sqlite3/quoting.rb @@ -12,8 +12,12 @@ module ActiveRecord quote_column_name(attr) end + def quote_table_name(name) + self.class.quoted_table_names[name] ||= super.gsub(".", "\".\"").freeze + end + def quote_column_name(name) - @quoted_column_names[name] ||= %Q("#{super.gsub('"', '""')}") + self.class.quoted_column_names[name] ||= %Q("#{super.gsub('"', '""')}") end def quoted_time(value) @@ -26,23 +30,58 @@ module ActiveRecord end def quoted_true - ActiveRecord::ConnectionAdapters::SQLite3Adapter.represent_boolean_as_integer ? "1" : "'t'" + "1" end def unquoted_true - ActiveRecord::ConnectionAdapters::SQLite3Adapter.represent_boolean_as_integer ? 1 : "t" + 1 end def quoted_false - ActiveRecord::ConnectionAdapters::SQLite3Adapter.represent_boolean_as_integer ? "0" : "'f'" + "0" end def unquoted_false - ActiveRecord::ConnectionAdapters::SQLite3Adapter.represent_boolean_as_integer ? 0 : "f" + 0 end - private + def column_name_matcher + COLUMN_NAME + end + + def column_name_with_order_matcher + COLUMN_NAME_WITH_ORDER + end + + COLUMN_NAME = / + \A + ( + (?: + # "table_name"."column_name" | function(one or no argument) + ((?:\w+\.|"\w+"\.)?(?:\w+|"\w+")) | \w+\((?:|\g<2>)\) + ) + (?:(?:\s+AS)?\s+(?:\w+|"\w+"))? + ) + (?:\s*,\s*\g<1>)* + \z + /ix + + COLUMN_NAME_WITH_ORDER = / + \A + ( + (?: + # "table_name"."column_name" | function(one or no argument) + ((?:\w+\.|"\w+"\.)?(?:\w+|"\w+")) | \w+\((?:|\g<2>)\) + ) + (?:\s+ASC|\s+DESC)? + ) + (?:\s*,\s*\g<1>)* + \z + /ix + private_constant :COLUMN_NAME, :COLUMN_NAME_WITH_ORDER + + private def _type_cast(value) case value when BigDecimal diff --git a/activerecord/lib/active_record/connection_adapters/sqlite3/schema_statements.rb b/activerecord/lib/active_record/connection_adapters/sqlite3/schema_statements.rb index 48277f0ae2..e48f59b4f0 100644 --- a/activerecord/lib/active_record/connection_adapters/sqlite3/schema_statements.rb +++ b/activerecord/lib/active_record/connection_adapters/sqlite3/schema_statements.rb @@ -11,7 +11,7 @@ module ActiveRecord # See https://www.sqlite.org/fileformat2.html#intschema next if row["name"].starts_with?("sqlite_") - index_sql = query_value(<<-SQL, "SCHEMA") + index_sql = query_value(<<~SQL, "SCHEMA") SELECT sql FROM sqlite_master WHERE name = #{quote(row['name'])} AND type = 'index' @@ -52,6 +52,32 @@ module ActiveRecord end.compact end + def add_foreign_key(from_table, to_table, **options) + alter_table(from_table) do |definition| + to_table = strip_table_name_prefix_and_suffix(to_table) + definition.foreign_key(to_table, options) + end + end + + def remove_foreign_key(from_table, to_table = nil, **options) + to_table ||= options[:to_table] + options = options.except(:name, :to_table) + foreign_keys = foreign_keys(from_table) + + fkey = foreign_keys.detect do |fk| + table = to_table || begin + table = options[:column].to_s.delete_suffix("_id") + Base.pluralize_table_names ? table.pluralize : table + end + table = strip_table_name_prefix_and_suffix(table) + fk_to_table = strip_table_name_prefix_and_suffix(fk.to_table) + fk_to_table == table && options.all? { |k, v| fk.options[k].to_s == v.to_s } + end || raise(ArgumentError, "Table '#{from_table}' has no foreign key for #{to_table || options}") + + foreign_keys.delete(fkey) + alter_table(from_table, foreign_keys) + end + def create_schema_dumper(options) SQLite3::SchemaDumper.create(self, options) end @@ -62,7 +88,7 @@ module ActiveRecord end def create_table_definition(*args) - SQLite3::TableDefinition.new(*args) + SQLite3::TableDefinition.new(self, *args) end def new_column_from_field(table_name, field) @@ -79,7 +105,7 @@ module ActiveRecord end type_metadata = fetch_type_metadata(field["type"]) - Column.new(field["name"], default, type_metadata, field["notnull"].to_i == 0, table_name, nil, field["collation"]) + Column.new(field["name"], default, type_metadata, field["notnull"].to_i == 0, collation: field["collation"]) end def data_source_sql(name = nil, type: nil) diff --git a/activerecord/lib/active_record/connection_adapters/sqlite3_adapter.rb b/activerecord/lib/active_record/connection_adapters/sqlite3_adapter.rb index 3312c3de01..f4847eb6c0 100644 --- a/activerecord/lib/active_record/connection_adapters/sqlite3_adapter.rb +++ b/activerecord/lib/active_record/connection_adapters/sqlite3_adapter.rb @@ -4,12 +4,13 @@ require "active_record/connection_adapters/abstract_adapter" require "active_record/connection_adapters/statement_pool" require "active_record/connection_adapters/sqlite3/explain_pretty_printer" require "active_record/connection_adapters/sqlite3/quoting" +require "active_record/connection_adapters/sqlite3/database_statements" require "active_record/connection_adapters/sqlite3/schema_creation" require "active_record/connection_adapters/sqlite3/schema_definitions" require "active_record/connection_adapters/sqlite3/schema_dumper" require "active_record/connection_adapters/sqlite3/schema_statements" -gem "sqlite3", "~> 1.3.6" +gem "sqlite3", "~> 1.4" require "sqlite3" module ActiveRecord @@ -36,8 +37,6 @@ module ActiveRecord config.merge(results_as_hash: true) ) - db.busy_timeout(ConnectionAdapters::SQLite3Adapter.type_cast_config_to_integer(config[:timeout])) if config[:timeout] - ConnectionAdapters::SQLite3Adapter.new(db, logger, nil, config) rescue Errno::ENOENT => error if error.message.include?("No such file or directory") @@ -49,8 +48,8 @@ module ActiveRecord end module ConnectionAdapters #:nodoc: - # The SQLite3 adapter works SQLite 3.6.16 or newer - # with the sqlite3-ruby drivers (available as gem from https://rubygems.org/gems/sqlite3). + # The SQLite3 adapter works with the sqlite3-ruby drivers + # (available as gem from https://rubygems.org/gems/sqlite3). # # Options: # @@ -60,6 +59,7 @@ module ActiveRecord include SQLite3::Quoting include SQLite3::SchemaStatements + include SQLite3::DatabaseStatements NATIVE_DATABASE_TYPES = { primary_key: "integer PRIMARY KEY AUTOINCREMENT NOT NULL", @@ -76,22 +76,15 @@ module ActiveRecord json: { name: "json" }, } - ## - # :singleton-method: - # Indicates whether boolean values are stored in sqlite3 databases as 1 - # and 0 or 't' and 'f'. Leaving <tt>ActiveRecord::ConnectionAdapters::SQLite3Adapter.represent_boolean_as_integer</tt> - # set to false is deprecated. SQLite databases have used 't' and 'f' to - # serialize boolean values and must have old data converted to 1 and 0 - # (its native boolean serialization) before setting this flag to true. - # Conversion can be accomplished by setting up a rake task which runs - # - # ExampleModel.where("boolean_column = 't'").update_all(boolean_column: 1) - # ExampleModel.where("boolean_column = 'f'").update_all(boolean_column: 0) - # for all models and all boolean columns, after which the flag must be set - # to true by adding the following to your <tt>application.rb</tt> file: - # - # Rails.application.config.active_record.sqlite3.represent_boolean_as_integer = true - class_attribute :represent_boolean_as_integer, default: false + def self.represent_boolean_as_integer=(value) # :nodoc: + if value == false + raise "`.represent_boolean_as_integer=` is now always true, so make sure your application can work with it and remove this settings." + end + + ActiveSupport::Deprecation.warn( + "`.represent_boolean_as_integer=` is now always true, so setting this is deprecated and will be removed in Rails 6.1." + ) + end class StatementPool < ConnectionAdapters::StatementPool # :nodoc: private @@ -102,12 +95,19 @@ module ActiveRecord def initialize(connection, logger, connection_options, config) super(connection, logger, config) - - @active = true - @statements = StatementPool.new(self.class.type_cast_config_to_integer(config[:statement_limit])) configure_connection end + def self.database_exists?(config) + config = config.symbolize_keys + if config[:database] == ":memory:" + return true + else + database_file = defined?(Rails.root) ? File.expand_path(config[:database], Rails.root) : config[:database] + File.exist?(database_file) + end + end + def supports_ddl_transactions? true end @@ -121,14 +121,14 @@ module ActiveRecord end def supports_expression_index? - sqlite_version >= "3.9.0" + database_version >= "3.9.0" end def requires_reloading? true end - def supports_foreign_keys_in_create? + def supports_foreign_keys? true end @@ -144,23 +144,29 @@ module ActiveRecord true end + def supports_insert_on_conflict? + database_version >= "3.24.0" + end + alias supports_insert_on_duplicate_skip? supports_insert_on_conflict? + alias supports_insert_on_duplicate_update? supports_insert_on_conflict? + alias supports_insert_conflict_target? supports_insert_on_conflict? + def active? - @active + !@connection.closed? + end + + def reconnect! + super + connect if @connection.closed? end # Disconnects from the database if already connected. Otherwise, this # method does nothing. def disconnect! super - @active = false @connection.close rescue nil end - # Clears the prepared statements cache. - def clear_cache! - @statements.clear - end - def supports_index_sort_order? true end @@ -205,79 +211,6 @@ module ActiveRecord end end - #-- - # DATABASE STATEMENTS ====================================== - #++ - - def explain(arel, binds = []) - sql = "EXPLAIN QUERY PLAN #{to_sql(arel, binds)}" - SQLite3::ExplainPrettyPrinter.new.pp(exec_query(sql, "EXPLAIN", [])) - end - - def exec_query(sql, name = nil, binds = [], prepare: false) - materialize_transactions - - type_casted_binds = type_casted_binds(binds) - - log(sql, name, binds, type_casted_binds) do - ActiveSupport::Dependencies.interlock.permit_concurrent_loads do - # Don't cache statements if they are not prepared - unless prepare - stmt = @connection.prepare(sql) - begin - cols = stmt.columns - unless without_prepared_statement?(binds) - stmt.bind_params(type_casted_binds) - end - records = stmt.to_a - ensure - stmt.close - end - else - stmt = @statements[sql] ||= @connection.prepare(sql) - cols = stmt.columns - stmt.reset! - stmt.bind_params(type_casted_binds) - records = stmt.to_a - end - - ActiveRecord::Result.new(cols, records) - end - end - end - - def exec_delete(sql, name = "SQL", binds = []) - exec_query(sql, name, binds) - @connection.changes - end - alias :exec_update :exec_delete - - def last_inserted_id(result) - @connection.last_insert_row_id - end - - def execute(sql, name = nil) #:nodoc: - materialize_transactions - - log(sql, name) do - ActiveSupport::Dependencies.interlock.permit_concurrent_loads do - @connection.execute(sql) - end - end - end - - def begin_db_transaction #:nodoc: - log("begin transaction", nil) { @connection.transaction } - end - - def commit_db_transaction #:nodoc: - log("commit transaction", nil) { @connection.commit } - end - - def exec_rollback_db_transaction #:nodoc: - log("rollback transaction", nil) { @connection.rollback } - end - # SCHEMA STATEMENTS ======================================== def primary_keys(table_name) # :nodoc: @@ -295,15 +228,12 @@ module ActiveRecord # Example: # rename_table('octopuses', 'octopi') def rename_table(table_name, new_name) + schema_cache.clear_data_source_cache!(table_name.to_s) + schema_cache.clear_data_source_cache!(new_name.to_s) exec_query "ALTER TABLE #{quote_table_name(table_name)} RENAME TO #{quote_table_name(new_name)}" rename_table_indexes(table_name, new_name) end - def valid_alter_table_type?(type, options = {}) - !invalid_alter_table_type?(type, options) - end - deprecate :valid_alter_table_type? - def add_column(table_name, column_name, type, options = {}) #:nodoc: if invalid_alter_table_type?(type, options) alter_table(table_name) do |definition| @@ -317,6 +247,9 @@ module ActiveRecord def remove_column(table_name, column_name, type = nil, options = {}) #:nodoc: alter_table(table_name) do |definition| definition.remove_column column_name + definition.foreign_keys.delete_if do |_, fk_options| + fk_options[:column] == column_name.to_s + end end end @@ -375,23 +308,26 @@ module ActiveRecord end end - def insert_fixtures(rows, table_name) - ActiveSupport::Deprecation.warn(<<-MSG.squish) - `insert_fixtures` is deprecated and will be removed in the next version of Rails. - Consider using `insert_fixtures_set` for performance improvement. - MSG - insert_fixtures_set(table_name => rows) + def build_insert_sql(insert) # :nodoc: + sql = +"INSERT #{insert.into} #{insert.values_list}" + + if insert.skip_duplicates? + sql << " ON CONFLICT #{insert.conflict_target} DO NOTHING" + elsif insert.update_duplicates? + sql << " ON CONFLICT #{insert.conflict_target} DO UPDATE SET " + sql << insert.updatable_columns.map { |column| "#{column}=excluded.#{column}" }.join(",") + end + + sql end - def insert_fixtures_set(fixture_set, tables_to_delete = []) - disable_referential_integrity do - transaction(requires_new: true) do - tables_to_delete.each { |table| delete "DELETE FROM #{quote_table_name(table)}", "Fixture Delete" } + def get_database_version # :nodoc: + SQLite3Adapter::Version.new(query_value("SELECT sqlite_version(*)")) + end - fixture_set.each do |table_name, rows| - rows.each { |row| insert_fixture(row, table_name) } - end - end + def check_version # :nodoc: + if database_version < "3.8.0" + raise "Your version of SQLite (#{database_version}) is too old. Active Record supports SQLite >= 3.8." end end @@ -402,12 +338,6 @@ module ActiveRecord 999 end - def check_version - if sqlite_version < "3.8.0" - raise "Your version of SQLite (#{sqlite_version}) is too old. Active Record supports SQLite >= 3.8." - end - end - def initialize_type_map(m = type_map) super register_class_with_limit m, %r(int)i, SQLite3Integer @@ -426,9 +356,8 @@ module ActiveRecord type.to_sym == :primary_key || options[:primary_key] end - def alter_table(table_name, options = {}) + def alter_table(table_name, foreign_keys = foreign_keys(table_name), **options) altered_table_name = "a#{table_name}" - foreign_keys = foreign_keys(table_name) caller = lambda do |definition| rename = options[:rename] || {} @@ -436,7 +365,8 @@ module ActiveRecord if column = rename[fk.options[:column]] fk.options[:column] = column end - definition.foreign_key(fk.to_table, fk.options) + to_table = strip_table_name_prefix_and_suffix(fk.to_table) + definition.foreign_key(to_table, fk.options) end yield definition if block_given? @@ -463,6 +393,7 @@ module ActiveRecord if from_primary_key.is_a?(Array) @definition.primary_keys from_primary_key end + columns(from).each do |column| column_name = options[:rename] ? (options[:rename][column.name] || @@ -525,22 +456,18 @@ module ActiveRecord SELECT #{quoted_from_columns} FROM #{quote_table_name(from)}") end - def sqlite_version - @sqlite_version ||= SQLite3Adapter::Version.new(query_value("SELECT sqlite_version(*)")) - end - - def translate_exception(exception, message) + def translate_exception(exception, message:, sql:, binds:) case exception.message # SQLite 3.8.2 returns a newly formatted error message: # UNIQUE constraint failed: *table_name*.*column_name* # Older versions of SQLite return: # column *column_name* is not unique when /column(s)? .* (is|are) not unique/, /UNIQUE constraint failed: .*/ - RecordNotUnique.new(message) + RecordNotUnique.new(message, sql: sql, binds: binds) when /.* may not be NULL/, /NOT NULL constraint failed: .*/ - NotNullViolation.new(message) + NotNullViolation.new(message, sql: sql, binds: binds) when /FOREIGN KEY constraint failed/i - InvalidForeignKey.new(message) + InvalidForeignKey.new(message, sql: sql, binds: binds) else super end @@ -550,7 +477,7 @@ module ActiveRecord def table_structure_with_collation(table_name, basic_structure) collation_hash = {} - sql = <<-SQL + sql = <<~SQL SELECT sql FROM (SELECT * FROM sqlite_master UNION ALL SELECT * FROM sqlite_temp_master) @@ -563,9 +490,9 @@ module ActiveRecord result = exec_query(sql, "SCHEMA").first if result - # Splitting with left parentheses and picking up last will return all + # Splitting with left parentheses and discarding the first part will return all # columns separated with comma(,). - columns_string = result["sql"].split("(").last + columns_string = result["sql"].split("(", 2).last columns_string.split(",").each do |column_string| # This regex will match the column name and collation type and will save @@ -591,7 +518,21 @@ module ActiveRecord Arel::Visitors::SQLite.new(self) end + def build_statement_pool + StatementPool.new(self.class.type_cast_config_to_integer(@config[:statement_limit])) + end + + def connect + @connection = ::SQLite3::Database.new( + @config[:database].to_s, + @config.merge(results_as_hash: true) + ) + configure_connection + end + def configure_connection + @connection.busy_timeout(self.class.type_cast_config_to_integer(@config[:timeout])) if @config[:timeout] + execute("PRAGMA foreign_keys = ON", "SCHEMA") end diff --git a/activerecord/lib/active_record/connection_adapters/statement_pool.rb b/activerecord/lib/active_record/connection_adapters/statement_pool.rb index 46bd831da7..0960feed84 100644 --- a/activerecord/lib/active_record/connection_adapters/statement_pool.rb +++ b/activerecord/lib/active_record/connection_adapters/statement_pool.rb @@ -48,7 +48,6 @@ module ActiveRecord end private - def cache @cache[Process.pid] end |