diff options
Diffstat (limited to 'activerecord/lib/active_record/connection_adapters')
50 files changed, 1668 insertions, 1086 deletions
diff --git a/activerecord/lib/active_record/connection_adapters/abstract/connection_pool.rb b/activerecord/lib/active_record/connection_adapters/abstract/connection_pool.rb index d99dc9a5db..6535121075 100644 --- a/activerecord/lib/active_record/connection_adapters/abstract/connection_pool.rb +++ b/activerecord/lib/active_record/connection_adapters/abstract/connection_pool.rb @@ -1,7 +1,6 @@ require 'thread' require 'thread_safe' require 'monitor' -require 'set' module ActiveRecord # Raised when a connection could not be obtained within the connection @@ -10,6 +9,12 @@ module ActiveRecord class ConnectionTimeoutError < ConnectionNotEstablished end + # Raised when a pool was unable to get ahold of all its connections + # to perform a "group" action such as +ConnectionPool#disconnect!+ + # or +ConnectionPool#clear_reloadable_connections!+. + class ExclusiveConnectionTimeoutError < ConnectionTimeoutError + end + module ConnectionAdapters # Connection pool base class for managing Active Record database # connections. @@ -63,6 +68,15 @@ module ActiveRecord # connection at the end of a thread or a thread dies unexpectedly. # Regardless of this setting, the Reaper will be invoked before every # blocking wait. (Default nil, which means don't schedule the Reaper). + # + #-- + # Synchronization policy: + # * all public methods can be called outside +synchronize+ + # * access to these i-vars needs to be in +synchronize+: + # * @connections + # * @now_connecting + # * private methods that require being called in a +synchronize+ blocks + # are now explicitly documented class ConnectionPool # Threadsafe, fair, FIFO queue. Meant to be used by ConnectionPool # with which it shares a Monitor. But could be a generic Queue. @@ -129,17 +143,15 @@ module ActiveRecord # - ConnectionTimeoutError if +timeout+ is given and no element # becomes available within +timeout+ seconds, def poll(timeout = nil) - synchronize do - if timeout - no_wait_poll || wait_poll(timeout) - else - no_wait_poll - end - end + synchronize { internal_poll(timeout) } end private + def internal_poll(timeout) + no_wait_poll || (timeout && wait_poll(timeout)) + end + def synchronize(&block) @lock.synchronize(&block) end @@ -193,6 +205,80 @@ module ActiveRecord end end + # Adds the ability to turn a basic fair FIFO queue into one + # biased to some thread. + module BiasableQueue # :nodoc: + class BiasedConditionVariable # :nodoc: + # semantics of condition variables guarantee that +broadcast+, +broadcast_on_biased+, + # +signal+ and +wait+ methods are only called while holding a lock + def initialize(lock, other_cond, preferred_thread) + @real_cond = lock.new_cond + @other_cond = other_cond + @preferred_thread = preferred_thread + @num_waiting_on_real_cond = 0 + end + + def broadcast + broadcast_on_biased + @other_cond.broadcast + end + + def broadcast_on_biased + @num_waiting_on_real_cond = 0 + @real_cond.broadcast + end + + def signal + if @num_waiting_on_real_cond > 0 + @num_waiting_on_real_cond -= 1 + @real_cond + else + @other_cond + end.signal + end + + def wait(timeout) + if Thread.current == @preferred_thread + @num_waiting_on_real_cond += 1 + @real_cond + else + @other_cond + end.wait(timeout) + end + end + + def with_a_bias_for(thread) + previous_cond = nil + new_cond = nil + synchronize do + previous_cond = @cond + @cond = new_cond = BiasedConditionVariable.new(@lock, @cond, thread) + end + yield + ensure + synchronize do + @cond = previous_cond if previous_cond + new_cond.broadcast_on_biased if new_cond # wake up any remaining sleepers + end + end + end + + # Connections must be leased while holding the main pool mutex. This is + # an internal subclass that also +.leases+ returned connections while + # still in queue's critical section (queue synchronizes with the same + # +@lock+ as the main pool) so that a returned connection is already + # leased and there is no need to re-enter synchronized block. + class ConnectionLeasingQueue < Queue # :nodoc: + include BiasableQueue + + private + def internal_poll(timeout) + conn = super + conn.lease if conn + conn + end + end + # Every +frequency+ seconds, the reaper will call +reap+ on +pool+. # A reaper instantiated with a nil frequency will never reap the # connection pool. @@ -220,7 +306,7 @@ module ActiveRecord include MonitorMixin - attr_accessor :automatic_reconnect, :checkout_timeout + attr_accessor :automatic_reconnect, :checkout_timeout, :schema_cache attr_reader :spec, :connections, :size, :reaper # Creates a new ConnectionPool object. +spec+ is a ConnectionSpecification @@ -241,56 +327,75 @@ module ActiveRecord # default max pool size to 5 @size = (spec.config[:pool] && spec.config[:pool].to_i) || 5 - # The cache of reserved connections mapped to threads - @reserved_connections = ThreadSafe::Cache.new(:initial_capacity => @size) + # The cache of threads mapped to reserved connections, the sole purpose + # of the cache is to speed-up +connection+ method, it is not the authoritative + # registry of which thread owns which connection, that is tracked by + # +connection.owner+ attr on each +connection+ instance. + # The invariant works like this: if there is mapping of +thread => conn+, + # then that +thread+ does indeed own that +conn+, however an absence of a such + # mapping does not mean that the +thread+ doesn't own the said connection, in + # that case +conn.owner+ attr should be consulted. + # Access and modification of +@thread_cached_conns+ does not require + # synchronization. + @thread_cached_conns = ThreadSafe::Cache.new(:initial_capacity => @size) @connections = [] @automatic_reconnect = true - @available = Queue.new self + # Connection pool allows for concurrent (outside the main `synchronize` section) + # establishment of new connections. This variable tracks the number of threads + # currently in the process of independently establishing connections to the DB. + @now_connecting = 0 + + # A boolean toggle that allows/disallows new connections. + @new_cons_enabled = true + + @available = ConnectionLeasingQueue.new self end # Retrieve the connection associated with the current thread, or call # #checkout to obtain one if necessary. # # #connection can be called any number of times; the connection is - # held in a hash keyed by the thread id. + # held in a cache keyed by a thread. def connection - # this is correctly done double-checked locking - # (ThreadSafe::Cache's lookups have volatile semantics) - @reserved_connections[current_connection_id] || synchronize do - @reserved_connections[current_connection_id] ||= checkout - end + @thread_cached_conns[connection_cache_key(Thread.current)] ||= checkout end # Is there an open connection that is being used for the current thread? + # + # This method only works for connections that have been abtained through + # #connection or #with_connection methods, connections obtained through + # #checkout will not be detected by #active_connection? def active_connection? - synchronize do - @reserved_connections.fetch(current_connection_id) { - return false - }.in_use? - end + @thread_cached_conns[connection_cache_key(Thread.current)] end # Signal that the thread is finished with the current connection. # #release_connection releases the connection-thread association # and returns the connection to the pool. - def release_connection(with_id = current_connection_id) - synchronize do - conn = @reserved_connections.delete(with_id) - checkin conn if conn + # + # This method only works for connections that have been obtained through + # #connection or #with_connection methods, connections obtained through + # #checkout will not be automatically released. + def release_connection(owner_thread = Thread.current) + if conn = @thread_cached_conns.delete(connection_cache_key(owner_thread)) + checkin conn end end - # If a connection already exists yield it to the block. If no connection + # If a connection obtained through #connection or #with_connection methods + # already exists yield it to the block. If no such connection # exists checkout a connection, yield it to the block, and checkin the # connection when finished. def with_connection - connection_id = current_connection_id - fresh_connection = true unless active_connection? - yield connection + unless conn = @thread_cached_conns[connection_cache_key(Thread.current)] + conn = connection + fresh_connection = true + end + yield conn ensure - release_connection(connection_id) if fresh_connection + release_connection if fresh_connection end # Returns true if a connection has already been opened. @@ -299,32 +404,81 @@ module ActiveRecord end # Disconnects all connections in the pool, and clears the pool. - def disconnect! - synchronize do - @reserved_connections.clear - @connections.each do |conn| - checkin conn - conn.disconnect! + # + # Raises: + # - +ExclusiveConnectionTimeoutError+ if unable to gain ownership of all + # connections in the pool within a timeout interval (default duration is + # +spec.config[:checkout_timeout] * 2+ seconds). + def disconnect(raise_on_acquisition_timeout = true) + with_exclusively_acquired_all_connections(raise_on_acquisition_timeout) do + synchronize do + @connections.each do |conn| + checkin conn + conn.disconnect! + end + @connections = [] + @available.clear end - @connections = [] - @available.clear end end - # Clears the cache which maps classes. - def clear_reloadable_connections! - synchronize do - @reserved_connections.clear - @connections.each do |conn| - checkin conn - conn.disconnect! if conn.requires_reloading? - end - @connections.delete_if(&:requires_reloading?) - @available.clear - @connections.each do |conn| - @available.add conn + # Disconnects all connections in the pool, and clears the pool. + # + # The pool first tries to gain ownership of all connections, if unable to + # do so within a timeout interval (default duration is + # +spec.config[:checkout_timeout] * 2+ seconds), the pool is forcefully + # disconneted wihout any regard for other connection owning threads. + def disconnect! + disconnect(false) + end + + # Clears the cache which maps classes and re-connects connections that + # require reloading. + # + # Raises: + # - +ExclusiveConnectionTimeoutError+ if unable to gain ownership of all + # connections in the pool within a timeout interval (default duration is + # +spec.config[:checkout_timeout] * 2+ seconds). + def clear_reloadable_connections(raise_on_acquisition_timeout = true) + num_new_conns_required = 0 + + with_exclusively_acquired_all_connections(raise_on_acquisition_timeout) do + synchronize do + @connections.each do |conn| + checkin conn + conn.disconnect! if conn.requires_reloading? + end + @connections.delete_if(&:requires_reloading?) + + @available.clear + + if @connections.size < @size + # because of the pruning done by this method, we might be running + # low on connections, while threads stuck in queue are helpless + # (not being able to establish new connections for themselves), + # see also more detailed explanation in +remove+ + num_new_conns_required = num_waiting_in_queue - @connections.size + end + + @connections.each do |conn| + @available.add conn + end end end + + bulk_make_new_connections(num_new_conns_required) if num_new_conns_required > 0 + end + + # Clears the cache which maps classes and re-connects connections that + # require reloading. + # + # The pool first tries to gain ownership of all connections, if unable to + # do so within a timeout interval (default duration is + # +spec.config[:checkout_timeout] * 2+ seconds), the pool forcefully + # clears the cache and reloads connections without any regard for other + # connection owning threads. + def clear_reloadable_connections! + clear_reloadable_connections(false) end # Check-out a database connection from the pool, indicating that you want @@ -341,12 +495,8 @@ module ActiveRecord # # Raises: # - ConnectionTimeoutError: no connection can be obtained from the pool. - def checkout - synchronize do - conn = acquire_connection - conn.lease - checkout_and_verify(conn) - end + def checkout(checkout_timeout = @checkout_timeout) + checkout_and_verify(acquire_connection(checkout_timeout)) end # Check-in a database connection back into the pool, indicating that you @@ -356,14 +506,12 @@ module ActiveRecord # calling +checkout+ on this pool. def checkin(conn) synchronize do - owner = conn.owner + remove_connection_from_thread_cache conn - conn._run_checkin_callbacks do + conn.run_callbacks :checkin do conn.expire end - release conn, owner - @available.add conn end end @@ -371,14 +519,32 @@ module ActiveRecord # Remove a connection from the connection pool. The connection will # remain open and active but will no longer be managed by this pool. def remove(conn) + needs_new_connection = false + synchronize do + remove_connection_from_thread_cache conn + @connections.delete conn @available.delete conn - release conn, conn.owner - - @available.add checkout_new_connection if @available.any_waiting? + # @available.any_waiting? => true means that prior to removing this + # conn, the pool was at its max size (@connections.size == @size) + # this would mean that any threads stuck waiting in the queue wouldn't + # know they could checkout_new_connection, so let's do it for them. + # Because condition-wait loop is encapsulated in the Queue class + # (that in turn is oblivious to ConnectionPool implementation), threads + # that are "stuck" there are helpless, they have no way of creating + # new connections and are completely reliant on us feeding available + # connections into the Queue. + needs_new_connection = @available.any_waiting? end + + # This is intentionally done outside of the synchronized section as we + # would like not to hold the main mutex while checking out new connections, + # thus there is some chance that needs_new_connection information is now + # stale, we can live with that (bulk_make_new_connections will make + # sure not to exceed the pool's @size limit). + bulk_make_new_connections(1) if needs_new_connection end # Recover lost connections for the pool. A lost connection can occur if @@ -403,7 +569,118 @@ module ActiveRecord end end + def num_waiting_in_queue # :nodoc: + @available.num_waiting + end + private + #-- + # this is unfortunately not concurrent + def bulk_make_new_connections(num_new_conns_needed) + num_new_conns_needed.times do + # try_to_checkout_new_connection will not exceed pool's @size limit + if new_conn = try_to_checkout_new_connection + # make the new_conn available to the starving threads stuck @available Queue + checkin(new_conn) + end + end + end + + #-- + # From the discussion on Github: + # https://github.com/rails/rails/pull/14938#commitcomment-6601951 + # This hook-in method allows for easier monkey-patching fixes needed by + # JRuby users that use Fibers. + def connection_cache_key(thread) + thread + end + + # Take control of all existing connections so a "group" action such as + # reload/disconnect can be performed safely. It is no longer enough to + # wrap it in +synchronize+ because some pool's actions are allowed + # to be performed outside of the main +synchronize+ block. + def with_exclusively_acquired_all_connections(raise_on_acquisition_timeout = true) + with_new_connections_blocked do + attempt_to_checkout_all_existing_connections(raise_on_acquisition_timeout) + yield + end + end + + def attempt_to_checkout_all_existing_connections(raise_on_acquisition_timeout = true) + collected_conns = synchronize do + # account for our own connections + @connections.select {|conn| conn.owner == Thread.current} + end + + newly_checked_out = [] + timeout_time = Time.now + (@checkout_timeout * 2) + + @available.with_a_bias_for(Thread.current) do + while true + synchronize do + return if collected_conns.size == @connections.size && @now_connecting == 0 + remaining_timeout = timeout_time - Time.now + remaining_timeout = 0 if remaining_timeout < 0 + conn = checkout_for_exclusive_access(remaining_timeout) + collected_conns << conn + newly_checked_out << conn + end + end + end + rescue ExclusiveConnectionTimeoutError + # `raise_on_acquisition_timeout == false` means we are directed to ignore any + # timeouts and are expected to just give up: we've obtained as many connections + # as possible, note that in a case like that we don't return any of the + # `newly_checked_out` connections. + + if raise_on_acquisition_timeout + release_newly_checked_out = true + raise + end + rescue Exception # if something else went wrong + # this can't be a "naked" rescue, because we have should return conns + # even for non-StandardErrors + release_newly_checked_out = true + raise + ensure + if release_newly_checked_out && newly_checked_out + # releasing only those conns that were checked out in this method, conns + # checked outside this method (before it was called) are not for us to release + newly_checked_out.each {|conn| checkin(conn)} + end + end + + #-- + # Must be called in a synchronize block. + def checkout_for_exclusive_access(checkout_timeout) + checkout(checkout_timeout) + rescue ConnectionTimeoutError + # this block can't be easily moved into attempt_to_checkout_all_existing_connections's + # rescue block, because doing so would put it outside of synchronize section, without + # being in a critical section thread_report might become inaccurate + msg = "could not obtain ownership of all database connections in #{checkout_timeout} seconds" + + thread_report = [] + @connections.each do |conn| + unless conn.owner == Thread.current + thread_report << "#{conn} is owned by #{conn.owner}" + end + end + + msg << " (#{thread_report.join(', ')})" if thread_report.any? + + raise ExclusiveConnectionTimeoutError, msg + end + + def with_new_connections_blocked + previous_value = nil + synchronize do + previous_value, @new_cons_enabled = @new_cons_enabled, false + end + yield + ensure + synchronize { @new_cons_enabled = previous_value } + end # Acquire a connection by one of 1) immediately removing one # from the queue of available connections, 2) creating a new @@ -412,44 +689,82 @@ module ActiveRecord # # Raises: # - ConnectionTimeoutError if a connection could not be acquired - def acquire_connection - if conn = @available.poll + # + #-- + # Implementation detail: the connection returned by +acquire_connection+ + # will already be "+connection.lease+ -ed" to the current thread. + def acquire_connection(checkout_timeout) + # NOTE: we rely on `@available.poll` and `try_to_checkout_new_connection` to + # `conn.lease` the returned connection (and to do this in a `synchronized` + # section), this is not the cleanest implementation, as ideally we would + # `synchronize { conn.lease }` in this method, but by leaving it to `@available.poll` + # and `try_to_checkout_new_connection` we can piggyback on `synchronize` sections + # of the said methods and avoid an additional `synchronize` overhead. + if conn = @available.poll || try_to_checkout_new_connection conn - elsif @connections.size < @size - checkout_new_connection else reap - @available.poll(@checkout_timeout) + @available.poll(checkout_timeout) end end - def release(conn, owner) - thread_id = owner.object_id - - if @reserved_connections[thread_id] == conn - @reserved_connections.delete thread_id - end + #-- + # if owner_thread param is omitted, this must be called in synchronize block + def remove_connection_from_thread_cache(conn, owner_thread = conn.owner) + @thread_cached_conns.delete_pair(connection_cache_key(owner_thread), conn) end + alias_method :release, :remove_connection_from_thread_cache def new_connection - Base.send(spec.adapter_method, spec.config) + Base.send(spec.adapter_method, spec.config).tap do |conn| + conn.schema_cache = schema_cache.dup if schema_cache + end + end + + # If the pool is not at a +@size+ limit, establish new connection. Connecting + # to the DB is done outside main synchronized section. + #-- + # Implementation constraint: a newly established connection returned by this + # method must be in the +.leased+ state. + def try_to_checkout_new_connection + # first in synchronized section check if establishing new conns is allowed + # and increment @now_connecting, to prevent overstepping this pool's @size + # constraint + do_checkout = synchronize do + if @new_cons_enabled && (@connections.size + @now_connecting) < @size + @now_connecting += 1 + end + end + if do_checkout + begin + # if successfully incremented @now_connecting establish new connection + # outside of synchronized section + conn = checkout_new_connection + ensure + synchronize do + if conn + adopt_connection(conn) + # returned conn needs to be already leased + conn.lease + end + @now_connecting -= 1 + end + end + end end - def current_connection_id #:nodoc: - Base.connection_id ||= Thread.current.object_id + def adopt_connection(conn) + conn.pool = self + @connections << conn end def checkout_new_connection raise ConnectionNotEstablished unless @automatic_reconnect - - c = new_connection - c.pool = self - @connections << c - c + new_connection end def checkout_and_verify(c) - c._run_checkout_callbacks do + c.run_callbacks :checkout do c.verify! end c @@ -618,7 +933,9 @@ module ActiveRecord # A connection was established in an ancestor process that must have # subsequently forked. We can't reuse the connection, but we can copy # the specification and establish a new connection with it. - establish_connection owner, ancestor_pool.spec + establish_connection(owner, ancestor_pool.spec).tap do |pool| + pool.schema_cache = ancestor_pool.schema_cache if ancestor_pool.schema_cache + end else owner_to_pool[owner.name] = nil end diff --git a/activerecord/lib/active_record/connection_adapters/abstract/database_limits.rb b/activerecord/lib/active_record/connection_adapters/abstract/database_limits.rb index c0a2111571..30b2fca2ca 100644 --- a/activerecord/lib/active_record/connection_adapters/abstract/database_limits.rb +++ b/activerecord/lib/active_record/connection_adapters/abstract/database_limits.rb @@ -18,9 +18,9 @@ module ActiveRecord end # Returns the maximum allowed length for an index name. This - # limit is enforced by rails and Is less than or equal to + # limit is enforced by \Rails and is less than or equal to # <tt>index_name_length</tt>. The gap between - # <tt>index_name_length</tt> is to allow internal rails + # <tt>index_name_length</tt> is to allow internal \Rails # operations to use prefixes in temporary operations. def allowed_index_name_length index_name_length diff --git a/activerecord/lib/active_record/connection_adapters/abstract/database_statements.rb b/activerecord/lib/active_record/connection_adapters/abstract/database_statements.rb index 787d07c4c2..38dd9578fe 100644 --- a/activerecord/lib/active_record/connection_adapters/abstract/database_statements.rb +++ b/activerecord/lib/active_record/connection_adapters/abstract/database_statements.rb @@ -40,8 +40,9 @@ module ActiveRecord # Returns a single value from a record def select_value(arel, name = nil, binds = []) - if result = select_one(arel, name, binds) - result.values.first + arel, binds = binds_from_relation arel, binds + if result = select_rows(to_sql(arel, binds), name, binds).first + result.first end end @@ -136,7 +137,7 @@ module ActiveRecord # # In order to get around this problem, #transaction will emulate the effect # of nested transactions, by using savepoints: - # http://dev.mysql.com/doc/refman/5.0/en/savepoint.html + # http://dev.mysql.com/doc/refman/5.6/en/savepoint.html # Savepoints are supported by MySQL and PostgreSQL. SQLite3 version >= '3.6.8' # supports savepoints. # @@ -188,8 +189,8 @@ module ActiveRecord # You should consult the documentation for your database to understand the # semantics of these different levels: # - # * http://www.postgresql.org/docs/9.1/static/transaction-iso.html - # * https://dev.mysql.com/doc/refman/5.0/en/set-transaction.html + # * http://www.postgresql.org/docs/current/static/transaction-iso.html + # * https://dev.mysql.com/doc/refman/5.6/en/set-transaction.html # # An <tt>ActiveRecord::TransactionIsolationError</tt> will be raised if: # @@ -201,16 +202,14 @@ module ActiveRecord # isolation level. However, support is disabled for MySQL versions below 5, # because they are affected by a bug[http://bugs.mysql.com/bug.php?id=39170] # which means the isolation level gets persisted outside the transaction. - def transaction(options = {}) - options.assert_valid_keys :requires_new, :joinable, :isolation - - if !options[:requires_new] && current_transaction.joinable? - if options[:isolation] + def transaction(requires_new: nil, isolation: nil, joinable: true) + if !requires_new && current_transaction.joinable? + if isolation raise ActiveRecord::TransactionIsolationError, "cannot set isolation when joining a transaction" end yield else - transaction_manager.within_new_transaction(options) { yield } + transaction_manager.within_new_transaction(isolation: isolation, joinable: joinable) { yield } end rescue ActiveRecord::Rollback # rollbacks are silently swallowed @@ -234,6 +233,10 @@ module ActiveRecord current_transaction.add_record(record) end + def transaction_state + current_transaction.state + end + # Begins the transaction (and turns off auto-committing). def begin_db_transaction() end @@ -286,10 +289,11 @@ module ActiveRecord columns = schema_cache.columns_hash(table_name) binds = fixture.map do |name, value| - [columns[name], value] + type = lookup_cast_type_from_column(columns[name]) + Relation::QueryAttribute.new(name, value, type) end key_list = fixture.keys.map { |name| quote_column_name(name) } - value_list = prepare_binds_for_database(binds).map do |_, value| + value_list = prepare_binds_for_database(binds).map do |value| begin quote(value) rescue TypeError @@ -381,7 +385,7 @@ module ActiveRecord def binds_from_relation(relation, binds) if relation.is_a?(Relation) && binds.empty? - relation, binds = relation.arel, relation.bind_values + relation, binds = relation.arel, relation.bound_attributes end [relation, binds] end diff --git a/activerecord/lib/active_record/connection_adapters/abstract/quoting.rb b/activerecord/lib/active_record/connection_adapters/abstract/quoting.rb index c18caa2a2f..2c7409b2dc 100644 --- a/activerecord/lib/active_record/connection_adapters/abstract/quoting.rb +++ b/activerecord/lib/active_record/connection_adapters/abstract/quoting.rb @@ -16,7 +16,7 @@ module ActiveRecord https://github.com/rails/arel/commit/6160bfbda1d1781c3b08a33ec4955f170e95be11 for more information. MSG - value = column.cast_type.type_cast_for_database(value) + value = type_cast_from_column(column, value) end _quote(value) @@ -31,7 +31,7 @@ module ActiveRecord end if column - value = column.cast_type.type_cast_for_database(value) + value = type_cast_from_column(column, value) end _type_cast(value) @@ -40,10 +40,44 @@ module ActiveRecord raise TypeError, "can't cast #{value.class}#{to_type}" end + # If you are having to call this function, you are likely doing something + # wrong. The column does not have sufficient type information if the user + # provided a custom type on the class level either explicitly (via + # `attribute`) or implicitly (via `serialize`, + # `time_zone_aware_attributes`). In almost all cases, the sql type should + # only be used to change quoting behavior, when the primitive to + # represent the type doesn't sufficiently reflect the differences + # (varchar vs binary) for example. The type used to get this primitive + # should have been provided before reaching the connection adapter. + def type_cast_from_column(column, value) # :nodoc: + if column + type = lookup_cast_type_from_column(column) + type.serialize(value) + else + value + end + end + + # See docs for +type_cast_from_column+ + def lookup_cast_type_from_column(column) # :nodoc: + lookup_cast_type(column.sql_type) + end + + def fetch_type_metadata(sql_type) + cast_type = lookup_cast_type(sql_type) + SqlTypeMetadata.new( + sql_type: sql_type, + type: cast_type.type, + limit: cast_type.limit, + precision: cast_type.precision, + scale: cast_type.scale, + ) + end + # Quotes a string, escaping any ' (single quote) and \ (backslash) # characters. def quote_string(s) - s.gsub(/\\/, '\&\&').gsub(/'/, "''") # ' (for ruby-mode) + s.gsub('\\'.freeze, '\&\&'.freeze).gsub("'".freeze, "''".freeze) # ' (for ruby-mode) end # Quotes the column name. Defaults to no quoting. @@ -68,6 +102,11 @@ module ActiveRecord quote_table_name("#{table}.#{attr}") end + def quote_default_expression(value, column) #:nodoc: + value = lookup_cast_type(column.sql_type).serialize(value) + quote(value) + end + def quoted_true "'t'" end @@ -84,6 +123,8 @@ module ActiveRecord 'f' end + # Quote date/time values for use in SQL input. Includes microseconds + # if the value is a Time responding to usec. def quoted_date(value) if value.acts_like?(:time) zone_conversion_method = ActiveRecord::Base.default_timezone == :utc ? :getutc : :getlocal @@ -93,17 +134,16 @@ module ActiveRecord end end - value.to_s(:db) + result = value.to_s(:db) + if value.respond_to?(:usec) && value.usec > 0 + "#{result}.#{sprintf("%06d", value.usec)}" + else + result + end end def prepare_binds_for_database(binds) # :nodoc: - binds.map do |column, value| - if column - column_name = column.name - value = column.cast_type.type_cast_for_database(value) - end - [column_name, value] - end + binds.map(&:value_for_database) end private diff --git a/activerecord/lib/active_record/connection_adapters/abstract/schema_creation.rb b/activerecord/lib/active_record/connection_adapters/abstract/schema_creation.rb index db20b60d60..18d943f452 100644 --- a/activerecord/lib/active_record/connection_adapters/abstract/schema_creation.rb +++ b/activerecord/lib/active_record/connection_adapters/abstract/schema_creation.rb @@ -14,26 +14,29 @@ module ActiveRecord send m, o end - def visit_AddColumn(o) - "ADD #{accept(o)}" - end + delegate :quote_column_name, :quote_table_name, :quote_default_expression, :type_to_sql, to: :@conn + private :quote_column_name, :quote_table_name, :quote_default_expression, :type_to_sql private def visit_AlterTable(o) sql = "ALTER TABLE #{quote_table_name(o.name)} " - sql << o.adds.map { |col| visit_AddColumn col }.join(' ') + sql << o.adds.map { |col| accept col }.join(' ') sql << o.foreign_key_adds.map { |fk| visit_AddForeignKey fk }.join(' ') sql << o.foreign_key_drops.map { |fk| visit_DropForeignKey fk }.join(' ') end def visit_ColumnDefinition(o) - o.sql_type = type_to_sql(o.type, o.limit, o.precision, o.scale) + o.sql_type ||= type_to_sql(o.type, o.limit, o.precision, o.scale) column_sql = "#{quote_column_name(o.name)} #{o.sql_type}" add_column_options!(column_sql, column_options(o)) unless o.type == :primary_key column_sql end + def visit_AddColumnDefinition(o) + "ADD #{accept(o.column)}" + end + def visit_TableDefinition(o) create_sql = "CREATE#{' TEMPORARY' if o.temporary} TABLE " create_sql << "#{quote_table_name(o.name)} " @@ -67,21 +70,10 @@ module ActiveRecord column_options[:after] = o.after column_options[:auto_increment] = o.auto_increment column_options[:primary_key] = o.primary_key + column_options[:collation] = o.collation column_options end - def quote_column_name(name) - @conn.quote_column_name name - end - - def quote_table_name(name) - @conn.quote_table_name name - end - - def type_to_sql(type, limit, precision, scale) - @conn.type_to_sql type.to_sym, limit, precision, scale - end - def add_column_options!(sql, options) sql << " DEFAULT #{quote_default_expression(options[:default], options[:column])}" if options_include_default?(options) # must explicitly check for :null to allow change_column to work on migrations @@ -97,11 +89,6 @@ module ActiveRecord sql end - def quote_default_expression(value, column) - value = type_for_column(column).type_cast_for_database(value) - @conn.quote(value) - end - def options_include_default?(options) options.include?(:default) && !(options[:null] == false && options[:default].nil?) end @@ -118,10 +105,6 @@ module ActiveRecord MSG end end - - def type_for_column(column) - @conn.lookup_cast_type(column.sql_type) - end end end end diff --git a/activerecord/lib/active_record/connection_adapters/abstract/schema_definitions.rb b/activerecord/lib/active_record/connection_adapters/abstract/schema_definitions.rb index 7eaa89c9a7..158b773e11 100644 --- a/activerecord/lib/active_record/connection_adapters/abstract/schema_definitions.rb +++ b/activerecord/lib/active_record/connection_adapters/abstract/schema_definitions.rb @@ -1,8 +1,3 @@ -require 'date' -require 'set' -require 'bigdecimal' -require 'bigdecimal/util' - module ActiveRecord module ConnectionAdapters #:nodoc: # Abstract representation of an index definition on a table. Instances of @@ -15,13 +10,16 @@ module ActiveRecord # are typically created by methods in TableDefinition, and added to the # +columns+ attribute of said TableDefinition object, in order to be used # for generating a number of table creation or table changing SQL statements. - class ColumnDefinition < Struct.new(:name, :type, :limit, :precision, :scale, :default, :null, :first, :after, :auto_increment, :primary_key, :sql_type, :cast_type) #:nodoc: + class ColumnDefinition < Struct.new(:name, :type, :limit, :precision, :scale, :default, :null, :first, :after, :auto_increment, :primary_key, :collation, :sql_type) #:nodoc: def primary_key? primary_key || type.to_sym == :primary_key end end + class AddColumnDefinition < Struct.new(:column) # :nodoc: + end + class ChangeColumnDefinition < Struct.new(:column, :name) #:nodoc: end @@ -50,6 +48,14 @@ module ActiveRecord options[:primary_key] != default_primary_key end + def defined_for?(options_or_to_table = {}) + if options_or_to_table.is_a?(Hash) + options_or_to_table.all? {|key, value| options[key].to_s == value.to_s } + else + to_table == options_or_to_table.to_s + end + end + private def default_primary_key "id" @@ -130,7 +136,42 @@ module ActiveRecord end def foreign_table_name - name.to_s.pluralize + Base.pluralize_table_names ? name.to_s.pluralize : name + end + end + + module ColumnMethods + # Appends a primary key definition to the table definition. + # Can be called multiple times, but this is probably not a good idea. + def primary_key(name, type = :primary_key, **options) + column(name, type, options.merge(primary_key: true)) + end + + # Appends a column or columns of a specified type. + # + # t.string(:goat) + # t.string(:goat, :sheep) + # + # See TableDefinition#column + [ + :bigint, + :binary, + :boolean, + :date, + :datetime, + :decimal, + :float, + :integer, + :string, + :text, + :time, + :timestamp, + ].each do |column_type| + module_eval <<-CODE, __FILE__, __LINE__ + 1 + def #{column_type}(*args, **options) + args.each { |name| column(name, :#{column_type}, options) } + end + CODE end end @@ -155,6 +196,8 @@ module ActiveRecord # The table definitions # The Columns are stored as a ColumnDefinition in the +columns+ attribute. class TableDefinition + include ColumnMethods + # An array of ColumnDefinition objects, representing the column changes # that have been defined. attr_accessor :indexes @@ -173,12 +216,6 @@ module ActiveRecord def columns; @columns_hash.values; end - # Appends a primary key definition to the table definition. - # Can be called multiple times, but this is probably not a good idea. - def primary_key(name, type = :primary_key, options = {}) - column(name, type, options.merge(:primary_key => true)) - end - # Returns a ColumnDefinition for the column with name +name+. def [](name) @columns_hash[name.to_s] @@ -188,7 +225,7 @@ module ActiveRecord # The +type+ parameter is normally one of the migrations native types, # which is one of the following: # <tt>:primary_key</tt>, <tt>:string</tt>, <tt>:text</tt>, - # <tt>:integer</tt>, <tt>:float</tt>, <tt>:decimal</tt>, + # <tt>:integer</tt>, <tt>:bigint</tt>, <tt>:float</tt>, <tt>:decimal</tt>, # <tt>:datetime</tt>, <tt>:time</tt>, <tt>:date</tt>, # <tt>:binary</tt>, <tt>:boolean</tt>. # @@ -320,6 +357,7 @@ module ActiveRecord def column(name, type, options = {}) name = name.to_s type = type.to_sym + options = options.dup if @columns_hash[name] && @columns_hash[name].primary_key? raise ArgumentError, "you can't redefine the primary key column '#{name}'. To define a custom primary key, pass { id: false } to create_table." @@ -335,14 +373,6 @@ module ActiveRecord @columns_hash.delete name.to_s end - [:string, :text, :integer, :bigint, :float, :decimal, :datetime, :timestamp, :time, :date, :binary, :boolean].each do |column_type| - define_method column_type do |*args| - options = args.extract_options! - column_names = args - column_names.each { |name| column(name, column_type, options) } - end - end - # Adds index options to the indexes hash, keyed by column name # This is primarily used to track indexes that need to be created after the table # @@ -368,17 +398,12 @@ module ActiveRecord column(:updated_at, :datetime, options) end - # Adds a reference. Optionally adds a +type+ column, if the - # +:polymorphic+ option is provided. +references+ and +belongs_to+ - # are acceptable. The reference column will be an +integer+ by default, - # the +:type+ option can be used to specify a different type. A foreign - # key will be created if the +:foreign_key+ option is passed. + # Adds a reference. # # t.references(:user) - # t.references(:user, type: "string") - # t.belongs_to(:supplier, polymorphic: true) + # t.belongs_to(:supplier, foreign_key: true) # - # See SchemaStatements#add_reference + # See SchemaStatements#add_reference for details of the options you can use. def references(*args, **options) args.each do |col| ReferenceDefinition.new(col, **options).add_to(self) @@ -402,6 +427,7 @@ module ActiveRecord column.after = options[:after] column.auto_increment = options[:auto_increment] column.primary_key = type == :primary_key || options[:primary_key] + column.collation = options[:collation] column end @@ -444,7 +470,7 @@ module ActiveRecord def add_column(name, type, options) name = name.to_s type = type.to_sym - @adds << @td.new_column_definition(name, type, options) + @adds << AddColumnDefinition.new(@td.new_column_definition(name, type, options)) end end @@ -454,6 +480,7 @@ module ActiveRecord # Available transformations are: # # change_table :table do |t| + # t.primary_key # t.column # t.index # t.rename_index @@ -466,6 +493,7 @@ module ActiveRecord # t.string # t.text # t.integer + # t.bigint # t.float # t.decimal # t.datetime @@ -482,6 +510,8 @@ module ActiveRecord # end # class Table + include ColumnMethods + attr_reader :name def initialize(table_name, base) @@ -500,6 +530,8 @@ module ActiveRecord # Checks to see if a column exists. # + # t.string(:name) unless t.column_exists?(:name, :string) + # # See SchemaStatements#column_exists? def column_exists?(column_name, type = nil, options = {}) @base.column_exists?(name, column_name, type, options) @@ -519,6 +551,10 @@ module ActiveRecord # Checks to see if an index exists. # + # unless t.index_exists?(:branch_id) + # t.index(:branch_id) + # end + # # See SchemaStatements#index_exists? def index_exists?(column_name, options = {}) @base.index_exists?(name, column_name, options) @@ -601,15 +637,12 @@ module ActiveRecord @base.rename_column(name, column_name, new_column_name) end - # Adds a reference. Optionally adds a +type+ column, if - # <tt>:polymorphic</tt> option is provided. + # Adds a reference. # # t.references(:user) - # t.references(:user, type: "string") - # t.belongs_to(:supplier, polymorphic: true) # t.belongs_to(:supplier, foreign_key: true) # - # See SchemaStatements#add_reference + # See SchemaStatements#add_reference for details of the options you can use. def references(*args) options = args.extract_options! args.each do |ref_name| @@ -632,25 +665,24 @@ module ActiveRecord end alias :remove_belongs_to :remove_references - # Adds a column or columns of a specified type. + # Adds a foreign key. # - # t.string(:goat) - # t.string(:goat, :sheep) + # t.foreign_key(:authors) # - # See SchemaStatements#add_column - [:string, :text, :integer, :float, :decimal, :datetime, :timestamp, :time, :date, :binary, :boolean].each do |column_type| - define_method column_type do |*args| - options = args.extract_options! - args.each do |column_name| - @base.add_column(name, column_name, column_type, options) - end - end - end - + # See SchemaStatements#add_foreign_key def foreign_key(*args) # :nodoc: @base.add_foreign_key(name, *args) end + # Checks to see if a foreign key exists. + # + # t.foreign_key(:authors) unless t.foreign_key_exists?(:authors) + # + # See SchemaStatements#foreign_key_exists? + def foreign_key_exists?(*args) # :nodoc: + @base.foreign_key_exists?(name, *args) + end + private def native @base.native_database_types diff --git a/activerecord/lib/active_record/connection_adapters/abstract/schema_dumper.rb b/activerecord/lib/active_record/connection_adapters/abstract/schema_dumper.rb index 42ea599a74..b944a8631c 100644 --- a/activerecord/lib/active_record/connection_adapters/abstract/schema_dumper.rb +++ b/activerecord/lib/active_record/connection_adapters/abstract/schema_dumper.rb @@ -24,33 +24,66 @@ module ActiveRecord def prepare_column_options(column) spec = {} spec[:name] = column.name.inspect - spec[:type] = column.type.to_s + spec[:type] = schema_type(column) spec[:null] = 'false' unless column.null - limit = column.limit || native_database_types[column.type][:limit] - spec[:limit] = limit.inspect if limit - spec[:precision] = column.precision.inspect if column.precision - spec[:scale] = column.scale.inspect if column.scale + if limit = schema_limit(column) + spec[:limit] = limit + end + + if precision = schema_precision(column) + spec[:precision] = precision + end + + if scale = schema_scale(column) + spec[:scale] = scale + end default = schema_default(column) if column.has_default? spec[:default] = default unless default.nil? + if collation = schema_collation(column) + spec[:collation] = collation + end + spec end # Lists the valid migration options def migration_keys - [:name, :limit, :precision, :scale, :default, :null] + [:name, :limit, :precision, :scale, :default, :null, :collation] end private + def schema_type(column) + column.type.to_s + end + + def schema_limit(column) + limit = column.limit || native_database_types[column.type][:limit] + limit.inspect if limit + end + + def schema_precision(column) + column.precision.inspect if column.precision + end + + def schema_scale(column) + column.scale.inspect if column.scale + end + def schema_default(column) - default = column.type_cast_from_database(column.default) + type = lookup_cast_type_from_column(column) + default = type.deserialize(column.default) unless default.nil? - column.type_cast_for_schema(default) + type.type_cast_for_schema(default) end end + + def schema_collation(column) + column.collation.inspect if column.collation + end end end end diff --git a/activerecord/lib/active_record/connection_adapters/abstract/schema_statements.rb b/activerecord/lib/active_record/connection_adapters/abstract/schema_statements.rb index 0f44c332ae..ed19819d63 100644 --- a/activerecord/lib/active_record/connection_adapters/abstract/schema_statements.rb +++ b/activerecord/lib/active_record/connection_adapters/abstract/schema_statements.rb @@ -1,4 +1,6 @@ require 'active_record/migration/join_table' +require 'active_support/core_ext/string/access' +require 'digest' module ActiveRecord module ConnectionAdapters # :nodoc: @@ -12,6 +14,10 @@ module ActiveRecord {} end + def table_options(table_name) + nil + end + # Truncates a table alias according to the limits of the current adapter. def table_alias_for(table_name) table_name[0...table_alias_length].tr('.', '_') @@ -118,6 +124,8 @@ module ActiveRecord # [<tt>:id</tt>] # Whether to automatically add a primary key column. Defaults to true. # Join tables for +has_and_belongs_to_many+ should set it to false. + # + # A Symbol can be used to specify the type of the generated primary key column. # [<tt>:primary_key</tt>] # The name of the primary key, if one is to be added automatically. # Defaults to +id+. If <tt>:id</tt> is false this option is ignored. @@ -161,6 +169,19 @@ module ActiveRecord # name varchar(80) # ) # + # ====== Change the primary key column type + # + # create_table(:tags, id: :string) do |t| + # t.column :label, :string + # end + # + # generates: + # + # CREATE TABLE tags ( + # id varchar PRIMARY KEY, + # label varchar + # ) + # # ====== Do not add a primary key column # # create_table(:categories_suppliers, id: false) do |t| @@ -375,16 +396,23 @@ module ActiveRecord # [<tt>:force</tt>] # Set to +:cascade+ to drop dependent objects as well. # Defaults to false. + # [<tt>:if_exists</tt>] + # Set to +true+ to only drop the table if it exists. + # Defaults to false. # # Although this command ignores most +options+ and the block if one is given, # it can be helpful to provide these in a migration's +change+ method so it can be reverted. # In that case, +options+ and the block will be used by create_table. def drop_table(table_name, options = {}) - execute "DROP TABLE #{quote_table_name(table_name)}" + execute "DROP TABLE#{' IF EXISTS' if options[:if_exists]} #{quote_table_name(table_name)}" end # Adds a new column to the named table. # See TableDefinition#column for details of the options you can use. + # + # Note: Not all options will be available, generally this command should + # ignore most of them. In favor of doing a low-level call to simply + # create a column. def add_column(table_name, column_name, type, options = {}) at = create_alter_table table_name at.add_column(column_name, type, options) @@ -528,6 +556,8 @@ module ActiveRecord # # CREATE UNIQUE INDEX index_accounts_on_branch_id_and_party_id ON accounts(branch_id, party_id) WHERE active # + # Note: Partial indexes are only supported for PostgreSQL and SQLite 3.8.0+. + # # ====== Creating an index with a specific method # # add_index(:developers, :name, using: 'btree') @@ -619,11 +649,21 @@ module ActiveRecord indexes(table_name).detect { |i| i.name == index_name } end - # Adds a reference. Optionally adds a +type+ column, if <tt>:polymorphic</tt> option is provided. - # The reference column is an +integer+ by default, the <tt>:type</tt> option can be used to specify - # a different type. + # Adds a reference. The reference column is an integer by default, + # the <tt>:type</tt> option can be used to specify a different type. + # Optionally adds a +_type+ column, if <tt>:polymorphic</tt> option is provided. # <tt>add_reference</tt> and <tt>add_belongs_to</tt> are acceptable. # + # The +options+ hash can include the following keys: + # [<tt>:type</tt>] + # The reference column type. Defaults to +:integer+. + # [<tt>:index</tt>] + # Add an appropriate index. Defaults to false. + # [<tt>:foreign_key</tt>] + # Add an appropriate foreign key. Defaults to false. + # [<tt>:polymorphic</tt>] + # Wether an additional +_type+ column should be added. Defaults to false. + # # ====== Create a user_id integer column # # add_reference(:products, :user) @@ -632,10 +672,6 @@ module ActiveRecord # # add_reference(:products, :user, type: :string) # - # ====== Create a supplier_id and supplier_type columns - # - # add_belongs_to(:products, :supplier, polymorphic: true) - # # ====== Create supplier_id, supplier_type columns and appropriate index # # add_reference(:products, :supplier, polymorphic: true, index: true) @@ -650,7 +686,7 @@ module ActiveRecord alias :add_belongs_to :add_reference # Removes the reference(s). Also removes a +type+ column if one exists. - # <tt>remove_reference</tt>, <tt>remove_references</tt> and <tt>remove_belongs_to</tt> are acceptable. + # <tt>remove_reference</tt> and <tt>remove_belongs_to</tt> are acceptable. # # ====== Remove the reference # @@ -660,7 +696,16 @@ module ActiveRecord # # remove_reference(:products, :supplier, polymorphic: true) # + # ====== Remove the reference with a foreign key + # + # remove_reference(:products, :user, index: true, foreign_key: true) + # def remove_reference(table_name, ref_name, options = {}) + if options[:foreign_key] + reference_name = Base.pluralize_table_names ? ref_name.to_s.pluralize : ref_name + remove_foreign_key(table_name, reference_name) + end + remove_column(table_name, "#{ref_name}_id") remove_column(table_name, "#{ref_name}_type") if options[:polymorphic] end @@ -676,8 +721,8 @@ module ActiveRecord # +to_table+ contains the referenced primary key. # # The foreign key will be named after the following pattern: <tt>fk_rails_<identifier></tt>. - # +identifier+ is a 10 character long random string. A custom name can be specified with - # the <tt>:name</tt> option. + # +identifier+ is a 10 character long string which is deterministically generated from the + # +from_table+ and +column+. A custom name can be specified with the <tt>:name</tt> option. # # ====== Creating a simple foreign key # @@ -749,21 +794,7 @@ module ActiveRecord def remove_foreign_key(from_table, options_or_to_table = {}) return unless supports_foreign_keys? - if options_or_to_table.is_a?(Hash) - options = options_or_to_table - else - options = { column: foreign_key_column_for(options_or_to_table) } - end - - fk_name_to_delete = options.fetch(:name) do - fk_to_delete = foreign_keys(from_table).detect {|fk| fk.column == options[:column].to_s } - - if fk_to_delete - fk_to_delete.name - else - raise ArgumentError, "Table '#{from_table}' has no foreign key on column '#{options[:column]}'" - end - end + fk_name_to_delete = foreign_key_for!(from_table, options_or_to_table).name at = create_alter_table from_table at.drop_foreign_key fk_name_to_delete @@ -771,6 +802,31 @@ module ActiveRecord execute schema_creation.accept(at) end + # Checks to see if a foreign key exists on a table for a given foreign key definition. + # + # # Check a foreign key exists + # foreign_key_exists?(:accounts, :branches) + # + # # Check a foreign key on a specified column exists + # foreign_key_exists?(:accounts, column: :owner_id) + # + # # Check a foreign key with a custom name exists + # foreign_key_exists?(:accounts, name: "special_fk_name") + # + def foreign_key_exists?(from_table, options_or_to_table = {}) + foreign_key_for(from_table, options_or_to_table).present? + end + + def foreign_key_for(from_table, options_or_to_table = {}) # :nodoc: + return unless supports_foreign_keys? + foreign_keys(from_table).detect {|fk| fk.defined_for? options_or_to_table } + end + + def foreign_key_for!(from_table, options_or_to_table = {}) # :nodoc: + foreign_key_for(from_table, options_or_to_table) or \ + raise ArgumentError, "Table '#{from_table}' has no foreign key for #{options_or_to_table}" + end + def foreign_key_column_for(table_name) # :nodoc: "#{table_name.to_s.singularize}_id" end @@ -832,6 +888,12 @@ module ActiveRecord raise ArgumentError, "Error adding decimal column: precision cannot be empty if scale is specified" end + elsif [:datetime, :time].include?(type) && precision ||= native[:precision] + if (0..6) === precision + column_type_sql << "(#{precision})" + else + raise(ActiveRecordError, "No #{native[:name]} type has precision of #{precision}. The allowed range of precision is from 0 to 6") + end elsif (type != :primary_key) && (limit ||= native.is_a?(Hash) && native[:limit]) column_type_sql << "(#{limit})" end @@ -878,13 +940,13 @@ module ActiveRecord def add_index_options(table_name, column_name, options = {}) #:nodoc: column_names = Array(column_name) - index_name = index_name(table_name, column: column_names) options.assert_valid_keys(:unique, :order, :name, :where, :length, :internal, :using, :algorithm, :type) - index_type = options[:unique] ? "UNIQUE" : "" index_type = options[:type].to_s if options.key?(:type) + index_type ||= options[:unique] ? "UNIQUE" : "" index_name = options[:name].to_s if options.key?(:name) + index_name ||= index_name(table_name, column: column_names) max_index_length = options.fetch(:internal, false) ? index_name_length : allowed_index_name_length if options.key?(:algorithm) @@ -990,12 +1052,14 @@ module ActiveRecord end def foreign_key_name(table_name, options) # :nodoc: + identifier = "#{table_name}_#{options.fetch(:column)}_fk" + hashed_identifier = Digest::SHA256.hexdigest(identifier).first(10) options.fetch(:name) do - "fk_rails_#{SecureRandom.hex(5)}" + "fk_rails_#{hashed_identifier}" end end - def validate_index_length!(table_name, new_name) + def validate_index_length!(table_name, new_name) # :nodoc: if new_name.length > allowed_index_name_length raise ArgumentError, "Index name '#{new_name}' on table '#{table_name}' is too long; the limit is #{allowed_index_name_length} characters" end diff --git a/activerecord/lib/active_record/connection_adapters/abstract/transaction.rb b/activerecord/lib/active_record/connection_adapters/abstract/transaction.rb index 7535e9147a..295a7bed87 100644 --- a/activerecord/lib/active_record/connection_adapters/abstract/transaction.rb +++ b/activerecord/lib/active_record/connection_adapters/abstract/transaction.rb @@ -1,13 +1,10 @@ module ActiveRecord module ConnectionAdapters class TransactionState - attr_reader :parent - VALID_STATES = Set.new([:committed, :rolledback, nil]) def initialize(state = nil) @state = state - @parent = nil end def finalized? @@ -27,7 +24,7 @@ module ActiveRecord end def set_state(state) - if !VALID_STATES.include?(state) + unless VALID_STATES.include?(state) raise ArgumentError, "Invalid transaction state: #{state}" end @state = state @@ -47,19 +44,16 @@ module ActiveRecord attr_reader :connection, :state, :records, :savepoint_name attr_writer :joinable - def initialize(connection, options) + def initialize(connection, options, run_commit_callbacks: false) @connection = connection @state = TransactionState.new @records = [] @joinable = options.fetch(:joinable, true) + @run_commit_callbacks = run_commit_callbacks end def add_record(record) - if record.has_transactional_callbacks? - records << record - else - record.set_transaction_state(@state) - end + records << record end def rollback @@ -81,15 +75,22 @@ module ActiveRecord @state.set_state(:committed) end + def before_commit_records + records.uniq.each(&:before_committed!) if @run_commit_callbacks + end + def commit_records ite = records.uniq while record = ite.shift - record.committed! + if @run_commit_callbacks + record.committed! + else + # if not running callbacks, only adds the record to the parent transaction + record.add_to_transaction + end end ensure - ite.each do |i| - i.committed!(should_run_callbacks: false) - end + ite.each { |i| i.committed!(should_run_callbacks: false) } end def full_rollback?; true; end @@ -100,8 +101,8 @@ module ActiveRecord class SavepointTransaction < Transaction - def initialize(connection, savepoint_name, options) - super(connection, options) + def initialize(connection, savepoint_name, options, *args) + super(connection, options, *args) if options[:isolation] raise ActiveRecord::TransactionIsolationError, "cannot set transaction isolation in a nested transaction" end @@ -111,7 +112,6 @@ module ActiveRecord def rollback connection.rollback_to_savepoint(savepoint_name) super - rollback_records end def commit @@ -124,7 +124,7 @@ module ActiveRecord class RealTransaction < Transaction - def initialize(connection, options) + def initialize(connection, options, *args) super if options[:isolation] connection.begin_isolated_db_transaction(options[:isolation]) @@ -136,13 +136,11 @@ module ActiveRecord def rollback connection.rollback_db_transaction super - rollback_records end def commit connection.commit_db_transaction super - commit_records end end @@ -153,24 +151,31 @@ module ActiveRecord end def begin_transaction(options = {}) + run_commit_callbacks = !current_transaction.joinable? transaction = if @stack.empty? - RealTransaction.new(@connection, options) + RealTransaction.new(@connection, options, run_commit_callbacks: run_commit_callbacks) else - SavepointTransaction.new(@connection, "active_record_#{@stack.size}", options) + SavepointTransaction.new(@connection, "active_record_#{@stack.size}", options, + run_commit_callbacks: run_commit_callbacks) end + @stack.push(transaction) transaction end def commit_transaction - transaction = @stack.pop + transaction = @stack.last + transaction.before_commit_records + @stack.pop transaction.commit - transaction.records.each { |r| current_transaction.add_record(r) } + transaction.commit_records end - def rollback_transaction - @stack.pop.rollback + def rollback_transaction(transaction = nil) + transaction ||= @stack.pop + transaction.rollback + transaction.rollback_records end def within_new_transaction(options = {}) @@ -182,12 +187,12 @@ module ActiveRecord ensure unless error if Thread.current.status == 'aborting' - rollback_transaction + rollback_transaction if transaction else begin commit_transaction rescue Exception - transaction.rollback unless transaction.state.completed? + rollback_transaction(transaction) unless transaction.state.completed? raise end end diff --git a/activerecord/lib/active_record/connection_adapters/abstract_adapter.rb b/activerecord/lib/active_record/connection_adapters/abstract_adapter.rb index c941c9f1eb..6d3a21a3dc 100644 --- a/activerecord/lib/active_record/connection_adapters/abstract_adapter.rb +++ b/activerecord/lib/active_record/connection_adapters/abstract_adapter.rb @@ -1,12 +1,9 @@ -require 'date' -require 'bigdecimal' -require 'bigdecimal/util' require 'active_record/type' require 'active_support/core_ext/benchmark' require 'active_record/connection_adapters/schema_cache' +require 'active_record/connection_adapters/sql_type_metadata' require 'active_record/connection_adapters/abstract/schema_dumper' require 'active_record/connection_adapters/abstract/schema_creation' -require 'monitor' require 'arel/collectors/bind' require 'arel/collectors/sql_string' @@ -69,7 +66,6 @@ module ActiveRecord include DatabaseLimits include QueryCache include ActiveSupport::Callbacks - include MonitorMixin include ColumnDumper SIMPLE_INT = /\A\d+\z/ @@ -114,7 +110,7 @@ module ActiveRecord class BindCollector < Arel::Collectors::Bind def compile(bvs, conn) casted_binds = conn.prepare_binds_for_database(bvs) - super(casted_binds.map { |_, value| conn.quote(value) }) + super(casted_binds.map { |value| conn.quote(value) }) end end @@ -140,12 +136,20 @@ module ActiveRecord SchemaCreation.new self end + # this method must only be called while holding connection pool's mutex def lease - synchronize do - unless in_use? - @owner = Thread.current + if in_use? + msg = 'Cannot lease connection, ' + if @owner == Thread.current + msg << 'it is already leased by the current thread.' + else + msg << "it is already in use by a different thread: #{@owner}. " << + "Current thread: #{Thread.current}." end + raise ActiveRecordError, msg end + + @owner = Thread.current end def schema_cache=(cache) @@ -153,6 +157,7 @@ module ActiveRecord @schema_cache = cache end + # this method must only be called while holding connection pool's mutex def expire @owner = nil end @@ -244,6 +249,11 @@ module ActiveRecord false end + # Does this adapter support datetime with precision? + def supports_datetime_with_precision? + false + end + # This is meant to be implemented by the adapters that support extensions def disable_extension(name) end @@ -262,18 +272,6 @@ module ActiveRecord {} end - # QUOTING ================================================== - - # Quote date/time values for use in SQL input. Includes microseconds - # if the value is a Time responding to usec. - def quoted_date(value) #:nodoc: - if value.acts_like?(:time) && value.respond_to?(:usec) - "#{super}.#{sprintf("%06d", value.usec)}" - else - super - end - end - # Returns a bind substitution value given a bind +column+ # NOTE: The column param is currently being used by the sqlserver-adapter def substitute_at(column, _unused = 0) @@ -366,9 +364,18 @@ module ActiveRecord end def case_insensitive_comparison(table, attribute, column, value) - table[attribute].lower.eq(table.lower(value)) + if can_perform_case_insensitive_comparison_for?(column) + table[attribute].lower.eq(table.lower(value)) + else + case_sensitive_comparison(table, attribute, column, value) + end end + def can_perform_case_insensitive_comparison_for?(column) + true + end + private :can_perform_case_insensitive_comparison_for? + def current_savepoint_name current_transaction.savepoint_name end @@ -384,8 +391,8 @@ module ActiveRecord end end - def new_column(name, default, cast_type, sql_type = nil, null = true) - Column.new(name, default, cast_type, sql_type, null) + def new_column(name, default, sql_type_metadata = nil, null = true, default_function = nil, collation = nil) + Column.new(name, default, sql_type_metadata, null, default_function, collation) end def lookup_cast_type(sql_type) # :nodoc: @@ -399,15 +406,15 @@ module ActiveRecord protected def initialize_type_map(m) # :nodoc: - register_class_with_limit m, %r(boolean)i, Type::Boolean - register_class_with_limit m, %r(char)i, Type::String - register_class_with_limit m, %r(binary)i, Type::Binary - register_class_with_limit m, %r(text)i, Type::Text - register_class_with_limit m, %r(date)i, Type::Date - register_class_with_limit m, %r(time)i, Type::Time - register_class_with_limit m, %r(datetime)i, Type::DateTime - register_class_with_limit m, %r(float)i, Type::Float - register_class_with_limit m, %r(int)i, Type::Integer + register_class_with_limit m, %r(boolean)i, Type::Boolean + register_class_with_limit m, %r(char)i, Type::String + register_class_with_limit m, %r(binary)i, Type::Binary + register_class_with_limit m, %r(text)i, Type::Text + register_class_with_precision m, %r(date)i, Type::Date + register_class_with_precision m, %r(time)i, Type::Time + register_class_with_precision m, %r(datetime)i, Type::DateTime + register_class_with_limit m, %r(float)i, Type::Float + register_class_with_limit m, %r(int)i, Type::Integer m.alias_type %r(blob)i, 'binary' m.alias_type %r(clob)i, 'text' @@ -441,6 +448,13 @@ module ActiveRecord end end + def register_class_with_precision(mapping, key, klass) # :nodoc: + mapping.register_type(key) do |*args| + precision = extract_precision(args.last) + klass.new(precision: precision) + end + end + def extract_scale(sql_type) # :nodoc: case sql_type when /\((\d+)\)/ then 0 @@ -453,7 +467,12 @@ module ActiveRecord end def extract_limit(sql_type) # :nodoc: - $1.to_i if sql_type =~ /\((.*)\)/ + case sql_type + when /^bigint/i + 8 + when /\((.*)\)/ + $1.to_i + end end def translate_exception_class(e, sql) @@ -463,7 +482,6 @@ module ActiveRecord message = "#{e.class.name}: #{e.message.force_encoding sql.encoding}: #{sql}" end - @logger.error message if @logger exception = translate_exception(e, message) exception.set_backtrace e.backtrace exception diff --git a/activerecord/lib/active_record/connection_adapters/abstract_mysql_adapter.rb b/activerecord/lib/active_record/connection_adapters/abstract_mysql_adapter.rb index e9a3c26c32..00e3d2965b 100644 --- a/activerecord/lib/active_record/connection_adapters/abstract_mysql_adapter.rb +++ b/activerecord/lib/active_record/connection_adapters/abstract_mysql_adapter.rb @@ -1,4 +1,3 @@ -require 'arel/visitors/bind_visitor' require 'active_support/core_ext/string/strip' module ActiveRecord @@ -6,20 +5,45 @@ module ActiveRecord class AbstractMysqlAdapter < AbstractAdapter include Savepoints - class TableDefinition < ActiveRecord::ConnectionAdapters::TableDefinition - def primary_key(name, type = :primary_key, options = {}) - options[:auto_increment] ||= type == :bigint + module ColumnMethods + def primary_key(name, type = :primary_key, **options) + options[:auto_increment] = true if type == :bigint super end end - class SchemaCreation < AbstractAdapter::SchemaCreation - def visit_AddColumn(o) - add_column_position!(super, column_options(o)) + class ColumnDefinition < ActiveRecord::ConnectionAdapters::ColumnDefinition + attr_accessor :charset + end + + class TableDefinition < ActiveRecord::ConnectionAdapters::TableDefinition + include ColumnMethods + + def new_column_definition(name, type, options) # :nodoc: + column = super + case column.type + when :primary_key + column.type = :integer + column.auto_increment = true + end + column.charset = options[:charset] + column end private + def create_column_definition(name, type) + ColumnDefinition.new(name, type) + end + end + + class Table < ActiveRecord::ConnectionAdapters::Table + include ColumnMethods + end + + class SchemaCreation < AbstractAdapter::SchemaCreation + private + def visit_DropForeignKey(name) "DROP FOREIGN KEY #{name}" end @@ -37,11 +61,31 @@ module ActiveRecord create_sql end + def visit_AddColumnDefinition(o) + add_column_position!(super, column_options(o.column)) + end + def visit_ChangeColumnDefinition(o) change_column_sql = "CHANGE #{quote_column_name(o.name)} #{accept(o.column)}" add_column_position!(change_column_sql, column_options(o.column)) end + def column_options(o) + column_options = super + column_options[:charset] = o.charset + column_options + end + + def add_column_options!(sql, options) + if options[:charset] + sql << " CHARACTER SET #{options[:charset]}" + end + if options[:collation] + sql << " COLLATE #{options[:collation]}" + end + super + end + def add_column_position!(sql, options) if options[:first] sql << " FIRST" @@ -52,20 +96,24 @@ module ActiveRecord end def index_in_create(table_name, column_name, options) - index_name, index_type, index_columns, index_options, index_algorithm, index_using = @conn.add_index_options(table_name, column_name, options) - "#{index_type} INDEX #{quote_column_name(index_name)} #{index_using} (#{index_columns})#{index_options} #{index_algorithm}" + index_name, index_type, index_columns, _, _, index_using = @conn.add_index_options(table_name, column_name, options) + "#{index_type} INDEX #{quote_column_name(index_name)} #{index_using} (#{index_columns}) " end end + def update_table_definition(table_name, base) # :nodoc: + Table.new(table_name, base) + end + def schema_creation SchemaCreation.new self end def column_spec_for_primary_key(column) spec = {} - if column.extra == 'auto_increment' - return unless column.limit == 8 - spec[:id] = ':bigint' + if column.auto_increment? + spec[:id] = ':bigint' if column.bigint? + return if spec.empty? else spec[:id] = column.type.inspect spec.merge!(prepare_column_options(column).delete_if { |key, _| [:name, :type, :null].include?(key) }) @@ -73,14 +121,31 @@ module ActiveRecord spec end + private + + def schema_limit(column) + super unless column.type == :boolean + end + + def schema_precision(column) + super unless /time/ === column.sql_type && column.precision == 0 + end + + def schema_collation(column) + if column.collation && table_name = column.instance_variable_get(:@table_name) + @collation_cache ||= {} + @collation_cache[table_name] ||= select_one("SHOW TABLE STATUS LIKE '#{table_name}'")["Collation"] + column.collation.inspect if column.collation != @collation_cache[table_name] + end + end + + public + class Column < ConnectionAdapters::Column # :nodoc: - attr_reader :collation, :strict, :extra + delegate :strict, :extra, to: :sql_type_metadata, allow_nil: true - def initialize(name, default, cast_type, sql_type = nil, null = true, collation = nil, strict = false, extra = "") - @strict = strict - @collation = collation - @extra = extra - super(name, default, cast_type, sql_type, null) + def initialize(*) + super assert_valid_default(default) extract_default end @@ -88,7 +153,7 @@ module ActiveRecord def extract_default if blob_or_text_column? @default = null || strict ? nil : '' - elsif missing_default_forged_as_empty_string?(@default) + elsif missing_default_forged_as_empty_string?(default) @default = nil end end @@ -106,11 +171,8 @@ module ActiveRecord collation && !collation.match(/_ci$/) end - def ==(other) - super && - collation == other.collation && - strict == other.strict && - extra == other.extra + def auto_increment? + extra == 'auto_increment' end private @@ -131,9 +193,32 @@ module ActiveRecord raise ArgumentError, "#{type} columns cannot have a default value: #{default.inspect}" end end + end + + class MysqlTypeMetadata < DelegateClass(SqlTypeMetadata) # :nodoc: + attr_reader :extra, :strict + + def initialize(type_metadata, extra: "", strict: false) + super(type_metadata) + @type_metadata = type_metadata + @extra = extra + @strict = strict + end + + def ==(other) + other.is_a?(MysqlTypeMetadata) && + attributes_for_hash == other.attributes_for_hash + end + alias eql? == + + def hash + attributes_for_hash.hash + end + + protected def attributes_for_hash - super + [collation, strict, extra] + [self.class, @type_metadata, extra, strict] end end @@ -188,6 +273,16 @@ module ActiveRecord end end + MAX_INDEX_LENGTH_FOR_CHARSETS_OF_4BYTES_MAXLEN = 191 + CHARSETS_OF_4BYTES_MAXLEN = ['utf8mb4', 'utf16', 'utf16le', 'utf32'] + def initialize_schema_migrations_table + if CHARSETS_OF_4BYTES_MAXLEN.include?(charset) + ActiveRecord::SchemaMigration.create_table(MAX_INDEX_LENGTH_FOR_CHARSETS_OF_4BYTES_MAXLEN) + else + ActiveRecord::SchemaMigration.create_table + end + end + # Returns true, since this connection adapter supports migrations. def supports_migrations? true @@ -227,6 +322,10 @@ module ActiveRecord version[0] >= 5 end + def supports_datetime_with_precision? + (version[0] == 5 && version[1] >= 6) || version[0] >= 6 + end + def native_database_types NATIVE_DATABASE_TYPES end @@ -243,8 +342,8 @@ module ActiveRecord raise NotImplementedError end - def new_column(field, default, cast_type, sql_type = nil, null = true, collation = "", extra = "") # :nodoc: - Column.new(field, default, cast_type, sql_type, null, collation, strict_mode?, extra) + def new_column(field, default, sql_type_metadata = nil, null = true, default_function = nil, collation = nil) # :nodoc: + Column.new(field, default, sql_type_metadata, null, default_function, collation) end # Must return the MySQL error number from the exception, if the exception has an @@ -467,8 +566,8 @@ module ActiveRecord each_hash(result).map do |field| field_name = set_field_encoding(field[:Field]) sql_type = field[:Type] - cast_type = lookup_cast_type(sql_type) - new_column(field_name, field[:Default], cast_type, sql_type, field[:Null] == "YES", field[:Collation], field[:Extra]) + type_metadata = fetch_type_metadata(sql_type, field[:Extra]) + new_column(field_name, field[:Default], type_metadata, field[:Null] == "YES", nil, field[:Collation]) end end end @@ -501,8 +600,23 @@ module ActiveRecord rename_table_indexes(table_name, new_name) end + # Drops a table from the database. + # + # [<tt>:force</tt>] + # Set to +:cascade+ to drop dependent objects as well. + # Defaults to false. + # [<tt>:if_exists</tt>] + # Set to +true+ to only drop the table if it exists. + # Defaults to false. + # [<tt>:temporary</tt>] + # Set to +true+ to drop temporary table. + # Defaults to false. + # + # Although this command ignores most +options+ and the block if one is given, + # it can be helpful to provide these in a migration's +change+ method so it can be reverted. + # In that case, +options+ and the block will be used by create_table. def drop_table(table_name, options = {}) - execute "DROP#{' TEMPORARY' if options[:temporary]} TABLE #{quote_table_name(table_name)}#{' CASCADE' if options[:force] == :cascade}" + execute "DROP#{' TEMPORARY' if options[:temporary]} TABLE#{' IF EXISTS' if options[:if_exists]} #{quote_table_name(table_name)}#{' CASCADE' if options[:force] == :cascade}" end def rename_index(table_name, old_name, new_name) @@ -520,7 +634,7 @@ module ActiveRecord change_column table_name, column_name, column.sql_type, :default => default end - def change_column_null(table_name, column_name, null, default = nil) + def change_column_null(table_name, column_name, null, default = nil) #:nodoc: column = column_for(table_name, column_name) unless null || default.nil? @@ -540,8 +654,8 @@ module ActiveRecord end def add_index(table_name, column_name, options = {}) #:nodoc: - index_name, index_type, index_columns, index_options, index_algorithm, index_using = add_index_options(table_name, column_name, options) - execute "CREATE #{index_type} INDEX #{quote_column_name(index_name)} #{index_using} ON #{quote_table_name(table_name)} (#{index_columns})#{index_options} #{index_algorithm}" + index_name, index_type, index_columns, _, index_algorithm, index_using = add_index_options(table_name, column_name, options) + execute "CREATE #{index_type} INDEX #{quote_column_name(index_name)} #{index_using} ON #{quote_table_name(table_name)} (#{index_columns}) #{index_algorithm}" end def foreign_keys(table_name) @@ -572,40 +686,25 @@ module ActiveRecord end end + def table_options(table_name) + create_table_info = select_one("SHOW CREATE TABLE #{quote_table_name(table_name)}")["Create Table"] + + # strip create_definitions and partition_options + raw_table_options = create_table_info.sub(/\A.*\n\) /m, '').sub(/\n\/\*!.*\*\/\n\z/m, '').strip + + # strip AUTO_INCREMENT + raw_table_options.sub(/(ENGINE=\w+)(?: AUTO_INCREMENT=\d+)/, '\1') + end + # Maps logical Rails types to MySQL-specific data types. def type_to_sql(type, limit = nil, precision = nil, scale = nil) case type.to_s when 'binary' - case limit - when 0..0xfff; "varbinary(#{limit})" - when nil; "blob" - when 0x1000..0xffffffff; "blob(#{limit})" - else raise(ActiveRecordError, "No binary type has character length #{limit}") - end + binary_to_sql(limit) when 'integer' - case limit - when 1; 'tinyint' - when 2; 'smallint' - when 3; 'mediumint' - when nil, 4, 11; 'int(11)' # compatibility with MySQL default - when 5..8; 'bigint' - else raise(ActiveRecordError, "No integer type has byte size #{limit}") - end + integer_to_sql(limit) when 'text' - case limit - when 0..0xff; 'tinytext' - when nil, 0x100..0xffff; 'text' - when 0x10000..0xffffff; 'mediumtext' - when 0x1000000..0xffffffff; 'longtext' - else raise(ActiveRecordError, "No text type has character length #{limit}") - end - when 'datetime' - return super unless precision - - case precision - when 0..6; "datetime(#{precision})" - else raise(ActiveRecordError, "No datetime type has precision of #{precision}. The allowed range of precision is from 0 to 6.") - end + text_to_sql(limit) else super end @@ -694,11 +793,6 @@ module ActiveRecord m.alias_type %r(year)i, 'integer' m.alias_type %r(bit)i, 'binary' - m.register_type(%r(datetime)i) do |sql_type| - precision = extract_precision(sql_type) - MysqlDateTime.new(precision: precision) - end - m.register_type(%r(enum)i) do |sql_type| limit = sql_type[/^enum\((.+)\)/i, 1] .split(',').map{|enum| enum.strip.length - 2}.max @@ -716,15 +810,16 @@ module ActiveRecord end end - # MySQL is too stupid to create a temporary table for use subquery, so we have - # to give it some prompting in the form of a subsubquery. Ugh! - def subquery_for(key, select) - subsubselect = select.clone - subsubselect.projections = [key] + def extract_precision(sql_type) + if /time/ === sql_type + super || 0 + else + super + end + end - subselect = Arel::SelectManager.new(select.engine) - subselect.project Arel.sql(key.name) - subselect.from subsubselect.as('__active_record_temp') + def fetch_type_metadata(sql_type, extra = "") + MysqlTypeMetadata.new(super(sql_type), extra: extra, strict: strict_mode?) end def add_index_length(option_strings, column_names, options = {}) @@ -766,7 +861,7 @@ module ActiveRecord def add_column_sql(table_name, column_name, type, options = {}) td = create_table_definition(table_name) cd = td.new_column_definition(column_name, type, options) - schema_creation.visit_AddColumn cd + schema_creation.accept(AddColumnDefinition.new(cd)) end def change_column_sql(table_name, column_name, type, options = {}) @@ -790,7 +885,7 @@ module ActiveRecord options = { default: column.default, null: column.null, - auto_increment: column.extra == "auto_increment" + auto_increment: column.auto_increment? } current_type = select_one("SHOW COLUMNS FROM #{quote_table_name(table_name)} LIKE '#{column_name}'", 'SCHEMA')["Type"] @@ -808,8 +903,9 @@ module ActiveRecord end def add_index_sql(table_name, column_name, options = {}) - index_name, index_type, index_columns = add_index_options(table_name, column_name, options) - "ADD #{index_type} INDEX #{index_name} (#{index_columns})" + index_name, index_type, index_columns, _, index_algorithm, index_using = add_index_options(table_name, column_name, options) + index_algorithm[0, 0] = ", " if index_algorithm.present? + "ADD #{index_type} INDEX #{quote_column_name(index_name)} #{index_using} (#{index_columns})#{index_algorithm}" end def remove_index_sql(table_name, options = {}) @@ -827,6 +923,19 @@ module ActiveRecord private + # MySQL is too stupid to create a temporary table for use subquery, so we have + # to give it some prompting in the form of a subsubquery. Ugh! + def subquery_for(key, select) + subsubselect = select.clone + subsubselect.projections = [key] + + subselect = Arel::SelectManager.new(select.engine) + subselect.project Arel.sql(key.name) + # Materialized subquery by adding distinct + # to work with MySQL 5.7.6 which sets optimizer_switch='derived_merge=on' + subselect.from subsubselect.distinct.as('__active_record_temp') + end + def version @version ||= full_version.scan(/^(\d+)\.(\d+)\.(\d+)/).flatten.map(&:to_i) end @@ -842,8 +951,7 @@ module ActiveRecord def configure_connection variables = @config.fetch(:variables, {}).stringify_keys - # By default, MySQL 'where id is null' selects the last inserted id. - # Turn this off. http://dev.rubyonrails.org/ticket/6778 + # By default, MySQL 'where id is null' selects the last inserted id; Turn this off. variables['sql_auto_is_null'] = 0 # Increase timeout so the server doesn't disconnect us. @@ -851,15 +959,17 @@ module ActiveRecord wait_timeout = 2147483 unless wait_timeout.is_a?(Fixnum) variables['wait_timeout'] = self.class.type_cast_config_to_integer(wait_timeout) + defaults = [':default', :default].to_set + # Make MySQL reject illegal values rather than truncating or blanking them, see - # http://dev.mysql.com/doc/refman/5.0/en/server-sql-mode.html#sqlmode_strict_all_tables + # http://dev.mysql.com/doc/refman/5.6/en/sql-mode.html#sqlmode_strict_all_tables # If the user has provided another value for sql_mode, don't replace it. - unless variables.has_key?('sql_mode') + unless variables.has_key?('sql_mode') || defaults.include?(@config[:strict]) variables['sql_mode'] = strict_mode? ? 'STRICT_ALL_TABLES' : '' end # NAMES does not have an equals sign, see - # http://dev.mysql.com/doc/refman/5.0/en/set-statement.html#id944430 + # http://dev.mysql.com/doc/refman/5.6/en/set-statement.html#id944430 # (trailing comma because variable_assignments will always have content) if @config[:encoding] encoding = "NAMES #{@config[:encoding]}" @@ -869,7 +979,7 @@ module ActiveRecord # Gather up all of the SET variables... variable_assignments = variables.map do |k, v| - if v == ':default' || v == :default + if defaults.include?(v) "@@SESSION.#{k} = DEFAULT" # Sets the value to the global or compile default elsif !v.nil? "@@SESSION.#{k} = #{quote(v)}" @@ -894,24 +1004,38 @@ module ActiveRecord TableDefinition.new(native_database_types, name, temporary, options, as) end - class MysqlDateTime < Type::DateTime # :nodoc: - def type_cast_for_database(value) - if value.acts_like?(:time) && value.respond_to?(:usec) - result = super.to_s(:db) - case precision - when 1..6 - "#{result}.#{sprintf("%0#{precision}d", value.usec / 10 ** (6 - precision))}" - else - result - end - else - super - end + def binary_to_sql(limit) # :nodoc: + case limit + when 0..0xfff; "varbinary(#{limit})" + when nil; "blob" + when 0x1000..0xffffffff; "blob(#{limit})" + else raise(ActiveRecordError, "No binary type has character length #{limit}") + end + end + + def integer_to_sql(limit) # :nodoc: + case limit + when 1; 'tinyint' + when 2; 'smallint' + when 3; 'mediumint' + when nil, 4, 11; 'int(11)' # compatibility with MySQL default + when 5..8; 'bigint' + else raise(ActiveRecordError, "No integer type has byte size #{limit}") + end + end + + def text_to_sql(limit) # :nodoc: + case limit + when 0..0xff; 'tinytext' + when nil, 0x100..0xffff; 'text' + when 0x10000..0xffffff; 'mediumtext' + when 0x1000000..0xffffffff; 'longtext' + else raise(ActiveRecordError, "No text type has character length #{limit}") end end class MysqlString < Type::String # :nodoc: - def type_cast_for_database(value) + def serialize(value) case value when true then "1" when false then "0" @@ -929,6 +1053,9 @@ module ActiveRecord end end end + + ActiveRecord::Type.register(:string, MysqlString, adapter: :mysql) + ActiveRecord::Type.register(:string, MysqlString, adapter: :mysql2) end end end diff --git a/activerecord/lib/active_record/connection_adapters/column.rb b/activerecord/lib/active_record/connection_adapters/column.rb index e74de60a83..4b95b0681d 100644 --- a/activerecord/lib/active_record/connection_adapters/column.rb +++ b/activerecord/lib/active_record/connection_adapters/column.rb @@ -12,34 +12,32 @@ module ActiveRecord ISO_DATETIME = /\A(\d{4})-(\d\d)-(\d\d) (\d\d):(\d\d):(\d\d)(\.\d+)?\z/ end - attr_reader :name, :cast_type, :null, :sql_type, :default, :default_function + attr_reader :name, :null, :sql_type_metadata, :default, :default_function, :collation - delegate :type, :precision, :scale, :limit, :klass, :accessor, - :text?, :number?, :binary?, :changed?, - :type_cast_from_user, :type_cast_from_database, :type_cast_for_database, - :type_cast_for_schema, - to: :cast_type + delegate :precision, :scale, :limit, :type, :sql_type, to: :sql_type_metadata, allow_nil: true # Instantiates a new column in the table. # # +name+ is the column's name, such as <tt>supplier_id</tt> in <tt>supplier_id int(11)</tt>. # +default+ is the type-casted default value, such as +new+ in <tt>sales_stage varchar(20) default 'new'</tt>. - # +cast_type+ is the object used for type casting and type information. - # +sql_type+ is used to extract the column's length, if necessary. For example +60+ in - # <tt>company_name varchar(60)</tt>. - # It will be mapped to one of the standard Rails SQL types in the <tt>type</tt> attribute. + # +sql_type_metadata+ is various information about the type of the column # +null+ determines if this column allows +NULL+ values. - def initialize(name, default, cast_type, sql_type = nil, null = true, default_function = nil) - @name = name - @cast_type = cast_type - @sql_type = sql_type - @null = null - @default = default + def initialize(name, default, sql_type_metadata = nil, null = true, default_function = nil, collation = nil) + @name = name + @sql_type_metadata = sql_type_metadata + @null = null + @default = default @default_function = default_function + @collation = collation + @table_name = nil end def has_default? - !default.nil? + !default.nil? || default_function + end + + def bigint? + /bigint/ === sql_type end # Returns the human name of the column name. @@ -50,19 +48,9 @@ module ActiveRecord Base.human_attribute_name(@name) end - def with_type(type) - dup.tap do |clone| - clone.instance_variable_set('@cast_type', type) - end - end - def ==(other) - other.name == name && - other.default == default && - other.cast_type == cast_type && - other.sql_type == sql_type && - other.null == null && - other.default_function == default_function + other.is_a?(Column) && + attributes_for_hash == other.attributes_for_hash end alias :eql? :== @@ -70,16 +58,16 @@ module ActiveRecord attributes_for_hash.hash end - private + protected def attributes_for_hash - [self.class, name, default, cast_type, sql_type, null, default_function] + [self.class, name, default, sql_type_metadata, null, default_function, collation] end end class NullColumn < Column def initialize(name) - super name, nil, Type::Value.new + super(name, nil) end end end diff --git a/activerecord/lib/active_record/connection_adapters/mysql2_adapter.rb b/activerecord/lib/active_record/connection_adapters/mysql2_adapter.rb index 75f244b3f3..e97e82f056 100644 --- a/activerecord/lib/active_record/connection_adapters/mysql2_adapter.rb +++ b/activerecord/lib/active_record/connection_adapters/mysql2_adapter.rb @@ -1,6 +1,6 @@ require 'active_record/connection_adapters/abstract_mysql_adapter' -gem 'mysql2', '~> 0.3.13' +gem 'mysql2', '~> 0.3.18' require 'mysql2' module ActiveRecord @@ -37,15 +37,6 @@ module ActiveRecord configure_connection end - MAX_INDEX_LENGTH_FOR_UTF8MB4 = 191 - def initialize_schema_migrations_table - if @config[:encoding] == 'utf8mb4' - ActiveRecord::SchemaMigration.create_table(MAX_INDEX_LENGTH_FOR_UTF8MB4) - else - ActiveRecord::SchemaMigration.create_table - end - end - def supports_explain? true end diff --git a/activerecord/lib/active_record/connection_adapters/mysql_adapter.rb b/activerecord/lib/active_record/connection_adapters/mysql_adapter.rb index 23d8389abb..2ae462d773 100644 --- a/activerecord/lib/active_record/connection_adapters/mysql_adapter.rb +++ b/activerecord/lib/active_record/connection_adapters/mysql_adapter.rb @@ -5,8 +5,10 @@ require 'active_support/core_ext/hash/keys' gem 'mysql', '~> 2.9' require 'mysql' -class Mysql +class Mysql # :nodoc: all class Time + # Used for casting DateTime fields to a MySQL friendly Time. + # This was documented in 48498da0dfed5239ea1eafb243ce47d7e3ce9e8e def to_date Date.new(year, month, day) end @@ -56,9 +58,9 @@ module ActiveRecord # * <tt>:password</tt> - Defaults to nothing. # * <tt>:database</tt> - The name of the database. No default, must be provided. # * <tt>:encoding</tt> - (Optional) Sets the client encoding by executing "SET NAMES <encoding>" after connection. - # * <tt>:reconnect</tt> - Defaults to false (See MySQL documentation: http://dev.mysql.com/doc/refman/5.0/en/auto-reconnect.html). - # * <tt>:strict</tt> - Defaults to true. Enable STRICT_ALL_TABLES. (See MySQL documentation: http://dev.mysql.com/doc/refman/5.0/en/server-sql-mode.html) - # * <tt>:variables</tt> - (Optional) A hash session variables to send as <tt>SET @@SESSION.key = value</tt> on each database connection. Use the value +:default+ to set a variable to its DEFAULT value. (See MySQL documentation: http://dev.mysql.com/doc/refman/5.0/en/set-statement.html). + # * <tt>:reconnect</tt> - Defaults to false (See MySQL documentation: http://dev.mysql.com/doc/refman/5.6/en/auto-reconnect.html). + # * <tt>:strict</tt> - Defaults to true. Enable STRICT_ALL_TABLES. (See MySQL documentation: http://dev.mysql.com/doc/refman/5.6/en/sql-mode.html) + # * <tt>:variables</tt> - (Optional) A hash session variables to send as <tt>SET @@SESSION.key = value</tt> on each database connection. Use the value +:default+ to set a variable to its DEFAULT value. (See MySQL documentation: http://dev.mysql.com/doc/refman/5.6/en/set-statement.html). # * <tt>:sslca</tt> - Necessary to use MySQL with an SSL connection. # * <tt>:sslkey</tt> - Necessary to use MySQL with an SSL connection. # * <tt>:sslcert</tt> - Necessary to use MySQL with an SSL connection. @@ -69,34 +71,10 @@ module ActiveRecord ADAPTER_NAME = 'MySQL'.freeze class StatementPool < ConnectionAdapters::StatementPool - def initialize(connection, max = 1000) - super - @cache = Hash.new { |h,pid| h[pid] = {} } - end - - def each(&block); cache.each(&block); end - def key?(key); cache.key?(key); end - def [](key); cache[key]; end - def length; cache.length; end - def delete(key); cache.delete(key); end - - def []=(sql, key) - while @max <= cache.size - cache.shift.last[:stmt].close - end - cache[sql] = key - end - - def clear - cache.each_value do |hash| - hash[:stmt].close - end - cache.clear - end - private - def cache - @cache[Process.pid] + + def dealloc(stmt) + stmt[:stmt].close end end @@ -328,8 +306,8 @@ module ActiveRecord def initialize_type_map(m) # :nodoc: super - m.register_type %r(datetime)i, Fields::DateTime.new - m.register_type %r(time)i, Fields::Time.new + register_class_with_precision m, %r(datetime)i, Fields::DateTime + register_class_with_precision m, %r(time)i, Fields::Time end def exec_without_stmt(sql, name = 'SQL') # :nodoc: @@ -395,11 +373,9 @@ module ActiveRecord def exec_stmt(sql, name, binds) cache = {} - type_casted_binds = binds.map { |col, val| - [col, type_cast(val, col)] - } + type_casted_binds = binds.map { |attr| type_cast(attr.value_for_database) } - log(sql, name, type_casted_binds) do + log(sql, name, binds) do if binds.empty? stmt = @connection.prepare(sql) else @@ -410,14 +386,17 @@ module ActiveRecord end begin - stmt.execute(*type_casted_binds.map { |_, val| val }) + stmt.execute(*type_casted_binds) rescue Mysql::Error => e # Older versions of MySQL leave the prepared statement in a bad # place when an error occurs. To support older MySQL versions, we # need to close the statement and delete the statement from the # cache. - stmt.close - @statements.delete sql + if binds.empty? + stmt.close + else + @statements.delete sql + end raise e end diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/array_parser.rb b/activerecord/lib/active_record/connection_adapters/postgresql/array_parser.rb deleted file mode 100644 index 1b74c039ce..0000000000 --- a/activerecord/lib/active_record/connection_adapters/postgresql/array_parser.rb +++ /dev/null @@ -1,93 +0,0 @@ -module ActiveRecord - module ConnectionAdapters - module PostgreSQL - module ArrayParser # :nodoc: - - DOUBLE_QUOTE = '"' - BACKSLASH = "\\" - COMMA = ',' - BRACKET_OPEN = '{' - BRACKET_CLOSE = '}' - - def parse_pg_array(string) # :nodoc: - local_index = 0 - array = [] - while(local_index < string.length) - case string[local_index] - when BRACKET_OPEN - local_index,array = parse_array_contents(array, string, local_index + 1) - when BRACKET_CLOSE - return array - end - local_index += 1 - end - - array - end - - private - - def parse_array_contents(array, string, index) - is_escaping = false - is_quoted = false - was_quoted = false - current_item = '' - - local_index = index - while local_index - token = string[local_index] - if is_escaping - current_item << token - is_escaping = false - else - if is_quoted - case token - when DOUBLE_QUOTE - is_quoted = false - was_quoted = true - when BACKSLASH - is_escaping = true - else - current_item << token - end - else - case token - when BACKSLASH - is_escaping = true - when COMMA - add_item_to_array(array, current_item, was_quoted) - current_item = '' - was_quoted = false - when DOUBLE_QUOTE - is_quoted = true - when BRACKET_OPEN - internal_items = [] - local_index,internal_items = parse_array_contents(internal_items, string, local_index + 1) - array.push(internal_items) - when BRACKET_CLOSE - add_item_to_array(array, current_item, was_quoted) - return local_index,array - else - current_item << token - end - end - end - - local_index += 1 - end - return local_index,array - end - - def add_item_to_array(array, current_item, quoted) - return if !quoted && current_item.length == 0 - - if !quoted && current_item == 'NULL' - array.push nil - else - array.push current_item - end - end - end - end - end -end diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/column.rb b/activerecord/lib/active_record/connection_adapters/postgresql/column.rb index acb1278499..bfa03fa136 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql/column.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql/column.rb @@ -2,21 +2,14 @@ module ActiveRecord module ConnectionAdapters # PostgreSQL-specific extensions to column definitions in a table. class PostgreSQLColumn < Column #:nodoc: - attr_reader :array + delegate :array, :oid, :fmod, to: :sql_type_metadata alias :array? :array - def initialize(name, default, cast_type, sql_type = nil, null = true, default_function = nil) - if sql_type =~ /\[\]$/ - @array = true - sql_type = sql_type[0..sql_type.length - 3] - else - @array = false - end - super - end - def serial? - default_function && default_function =~ /\Anextval\(.*\)\z/ + return unless default_function + + table_name = @table_name || '(?<table_name>.+)' + %r{\Anextval\('"?#{table_name}_#{name}_seq"?'::regclass\)\z} === default_function end end end diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/oid.rb b/activerecord/lib/active_record/connection_adapters/postgresql/oid.rb index d28a2b4fa0..92349e2f9b 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql/oid.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql/oid.rb @@ -1,25 +1,19 @@ -require 'active_record/connection_adapters/postgresql/oid/infinity' - require 'active_record/connection_adapters/postgresql/oid/array' require 'active_record/connection_adapters/postgresql/oid/bit' require 'active_record/connection_adapters/postgresql/oid/bit_varying' require 'active_record/connection_adapters/postgresql/oid/bytea' require 'active_record/connection_adapters/postgresql/oid/cidr' -require 'active_record/connection_adapters/postgresql/oid/date' require 'active_record/connection_adapters/postgresql/oid/date_time' require 'active_record/connection_adapters/postgresql/oid/decimal' require 'active_record/connection_adapters/postgresql/oid/enum' -require 'active_record/connection_adapters/postgresql/oid/float' require 'active_record/connection_adapters/postgresql/oid/hstore' require 'active_record/connection_adapters/postgresql/oid/inet' -require 'active_record/connection_adapters/postgresql/oid/integer' require 'active_record/connection_adapters/postgresql/oid/json' require 'active_record/connection_adapters/postgresql/oid/jsonb' require 'active_record/connection_adapters/postgresql/oid/money' require 'active_record/connection_adapters/postgresql/oid/point' require 'active_record/connection_adapters/postgresql/oid/range' require 'active_record/connection_adapters/postgresql/oid/specialized_string' -require 'active_record/connection_adapters/postgresql/oid/time' require 'active_record/connection_adapters/postgresql/oid/uuid' require 'active_record/connection_adapters/postgresql/oid/vector' require 'active_record/connection_adapters/postgresql/oid/xml' diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/oid/array.rb b/activerecord/lib/active_record/connection_adapters/postgresql/oid/array.rb index e45a2f59d9..3de794f797 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql/oid/array.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql/oid/array.rb @@ -3,51 +3,48 @@ module ActiveRecord module PostgreSQL module OID # :nodoc: class Array < Type::Value # :nodoc: - include Type::Mutable - - # Loads pg_array_parser if available. String parsing can be - # performed quicker by a native extension, which will not create - # a large amount of Ruby objects that will need to be garbage - # collected. pg_array_parser has a C and Java extension - begin - require 'pg_array_parser' - include PgArrayParser - rescue LoadError - require 'active_record/connection_adapters/postgresql/array_parser' - include PostgreSQL::ArrayParser - end + include Type::Helpers::Mutable attr_reader :subtype, :delimiter - delegate :type, :user_input_in_time_zone, to: :subtype + delegate :type, :user_input_in_time_zone, :limit, to: :subtype def initialize(subtype, delimiter = ',') @subtype = subtype @delimiter = delimiter + + @pg_encoder = PG::TextEncoder::Array.new name: "#{type}[]", delimiter: delimiter + @pg_decoder = PG::TextDecoder::Array.new name: "#{type}[]", delimiter: delimiter end - def type_cast_from_database(value) + def deserialize(value) if value.is_a?(::String) - type_cast_array(parse_pg_array(value), :type_cast_from_database) + type_cast_array(@pg_decoder.decode(value), :deserialize) else super end end - def type_cast_from_user(value) + def cast(value) if value.is_a?(::String) - value = parse_pg_array(value) + value = @pg_decoder.decode(value) end - type_cast_array(value, :type_cast_from_user) + type_cast_array(value, :cast) end - def type_cast_for_database(value) + def serialize(value) if value.is_a?(::Array) - cast_value_for_database(value) + @pg_encoder.encode(type_cast_array(value, :serialize)) else super end end + def ==(other) + other.is_a?(Array) && + subtype == other.subtype && + delimiter == other.delimiter + end + private def type_cast_array(value, method) @@ -57,41 +54,6 @@ module ActiveRecord @subtype.public_send(method, value) end end - - def cast_value_for_database(value) - if value.is_a?(::Array) - casted_values = value.map { |item| cast_value_for_database(item) } - "{#{casted_values.join(delimiter)}}" - else - quote_and_escape(subtype.type_cast_for_database(value)) - end - end - - ARRAY_ESCAPE = "\\" * 2 * 2 # escape the backslash twice for PG arrays - - def quote_and_escape(value) - case value - when ::String - if string_requires_quoting?(value) - value = value.gsub(/\\/, ARRAY_ESCAPE) - value.gsub!(/"/,"\\\"") - %("#{value}") - else - value - end - when nil then "NULL" - else value - end - end - - # See http://www.postgresql.org/docs/9.2/static/arrays.html#ARRAYS-IO - # for a list of all cases in which strings will be quoted. - def string_requires_quoting?(string) - string.empty? || - string == "NULL" || - string =~ /[\{\}"\\\s]/ || - string.include?(delimiter) - end end end end diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/oid/bit.rb b/activerecord/lib/active_record/connection_adapters/postgresql/oid/bit.rb index 1dbb40ca1d..ea0fa2517f 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql/oid/bit.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql/oid/bit.rb @@ -7,7 +7,7 @@ module ActiveRecord :bit end - def type_cast(value) + def cast(value) if ::String === value case value when /^0x/i @@ -20,7 +20,7 @@ module ActiveRecord end end - def type_cast_for_database(value) + def serialize(value) Data.new(super) if value end diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/oid/bytea.rb b/activerecord/lib/active_record/connection_adapters/postgresql/oid/bytea.rb index 6bd1b8ecae..8f9d6e7f9b 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql/oid/bytea.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql/oid/bytea.rb @@ -3,7 +3,7 @@ module ActiveRecord module PostgreSQL module OID # :nodoc: class Bytea < Type::Binary # :nodoc: - def type_cast_from_database(value) + def deserialize(value) return if value.nil? return value.to_s if value.is_a?(Type::Binary::Data) PGconn.unescape_bytea(super) diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/oid/cidr.rb b/activerecord/lib/active_record/connection_adapters/postgresql/oid/cidr.rb index 222f10fa8f..eeccb09bdf 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql/oid/cidr.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql/oid/cidr.rb @@ -18,7 +18,7 @@ module ActiveRecord end end - def type_cast_for_database(value) + def serialize(value) if IPAddr === value "#{value}/#{value.instance_variable_get(:@mask_addr).to_s(2).count('1')}" else diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/oid/date.rb b/activerecord/lib/active_record/connection_adapters/postgresql/oid/date.rb deleted file mode 100644 index 1d8d264530..0000000000 --- a/activerecord/lib/active_record/connection_adapters/postgresql/oid/date.rb +++ /dev/null @@ -1,11 +0,0 @@ -module ActiveRecord - module ConnectionAdapters - module PostgreSQL - module OID # :nodoc: - class Date < Type::Date # :nodoc: - include Infinity - end - end - end - end -end diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/oid/date_time.rb b/activerecord/lib/active_record/connection_adapters/postgresql/oid/date_time.rb index b9e7894e5c..2c04c46131 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql/oid/date_time.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql/oid/date_time.rb @@ -3,8 +3,6 @@ module ActiveRecord module PostgreSQL module OID # :nodoc: class DateTime < Type::DateTime # :nodoc: - include Infinity - def cast_value(value) if value.is_a?(::String) case value diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/oid/enum.rb b/activerecord/lib/active_record/connection_adapters/postgresql/oid/enum.rb index 77d5038efd..91d339f32c 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql/oid/enum.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql/oid/enum.rb @@ -7,7 +7,9 @@ module ActiveRecord :enum end - def type_cast(value) + private + + def cast_value(value) value.to_s end end diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/oid/float.rb b/activerecord/lib/active_record/connection_adapters/postgresql/oid/float.rb deleted file mode 100644 index 78ef94b912..0000000000 --- a/activerecord/lib/active_record/connection_adapters/postgresql/oid/float.rb +++ /dev/null @@ -1,21 +0,0 @@ -module ActiveRecord - module ConnectionAdapters - module PostgreSQL - module OID # :nodoc: - class Float < Type::Float # :nodoc: - include Infinity - - def cast_value(value) - case value - when ::Float then value - when 'Infinity' then ::Float::INFINITY - when '-Infinity' then -::Float::INFINITY - when 'NaN' then ::Float::NAN - else value.to_f - end - end - end - end - end - end -end diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/oid/hstore.rb b/activerecord/lib/active_record/connection_adapters/postgresql/oid/hstore.rb index be4525c94f..9270fc9f21 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql/oid/hstore.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql/oid/hstore.rb @@ -3,13 +3,13 @@ module ActiveRecord module PostgreSQL module OID # :nodoc: class Hstore < Type::Value # :nodoc: - include Type::Mutable + include Type::Helpers::Mutable def type :hstore end - def type_cast_from_database(value) + def deserialize(value) if value.is_a?(::String) ::Hash[value.scan(HstorePair).map { |k, v| v = v.upcase == 'NULL' ? nil : v.gsub(/\A"(.*)"\Z/m,'\1').gsub(/\\(.)/, '\1') @@ -21,7 +21,7 @@ module ActiveRecord end end - def type_cast_for_database(value) + def serialize(value) if value.is_a?(::Hash) value.map { |k, v| "#{escape_hstore(k)}=>#{escape_hstore(v)}" }.join(', ') else diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/oid/infinity.rb b/activerecord/lib/active_record/connection_adapters/postgresql/oid/infinity.rb deleted file mode 100644 index e47780399a..0000000000 --- a/activerecord/lib/active_record/connection_adapters/postgresql/oid/infinity.rb +++ /dev/null @@ -1,13 +0,0 @@ -module ActiveRecord - module ConnectionAdapters - module PostgreSQL - module OID # :nodoc: - module Infinity # :nodoc: - def infinity(options = {}) - options[:negative] ? -::Float::INFINITY : ::Float::INFINITY - end - end - end - end - end -end diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/oid/integer.rb b/activerecord/lib/active_record/connection_adapters/postgresql/oid/integer.rb deleted file mode 100644 index 59abdc0009..0000000000 --- a/activerecord/lib/active_record/connection_adapters/postgresql/oid/integer.rb +++ /dev/null @@ -1,11 +0,0 @@ -module ActiveRecord - module ConnectionAdapters - module PostgreSQL - module OID # :nodoc: - class Integer < Type::Integer # :nodoc: - include Infinity - end - end - end - end -end diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/oid/json.rb b/activerecord/lib/active_record/connection_adapters/postgresql/oid/json.rb index e12ddd9901..8e1256baad 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql/oid/json.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql/oid/json.rb @@ -3,25 +3,25 @@ module ActiveRecord module PostgreSQL module OID # :nodoc: class Json < Type::Value # :nodoc: - include Type::Mutable + include Type::Helpers::Mutable def type :json end - def type_cast_from_database(value) + def deserialize(value) if value.is_a?(::String) - ::ActiveSupport::JSON.decode(value) + ::ActiveSupport::JSON.decode(value) rescue nil else - super + value end end - def type_cast_for_database(value) + def serialize(value) if value.is_a?(::Array) || value.is_a?(::Hash) ::ActiveSupport::JSON.encode(value) else - super + value end end diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/oid/jsonb.rb b/activerecord/lib/active_record/connection_adapters/postgresql/oid/jsonb.rb index 380c50fc14..87391b5dc7 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql/oid/jsonb.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql/oid/jsonb.rb @@ -9,11 +9,11 @@ module ActiveRecord def changed_in_place?(raw_old_value, new_value) # Postgres does not preserve insignificant whitespaces when - # roundtripping jsonb columns. This causes some false positives for + # round-tripping jsonb columns. This causes some false positives for # the comparison here. Therefore, we need to parse and re-dump the # raw value here to ensure the insignificant whitespaces are # consistent with our encoder's output. - raw_old_value = type_cast_for_database(type_cast_from_database(raw_old_value)) + raw_old_value = serialize(deserialize(raw_old_value)) super(raw_old_value, new_value) end end diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/oid/money.rb b/activerecord/lib/active_record/connection_adapters/postgresql/oid/money.rb index df890c2ed6..2163674019 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql/oid/money.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql/oid/money.rb @@ -3,8 +3,6 @@ module ActiveRecord module PostgreSQL module OID # :nodoc: class Money < Type::Decimal # :nodoc: - include Infinity - class_attribute :precision def type diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/oid/point.rb b/activerecord/lib/active_record/connection_adapters/postgresql/oid/point.rb index bac8b01d6b..bf565bcf47 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql/oid/point.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql/oid/point.rb @@ -3,19 +3,19 @@ module ActiveRecord module PostgreSQL module OID # :nodoc: class Point < Type::Value # :nodoc: - include Type::Mutable + include Type::Helpers::Mutable def type :point end - def type_cast(value) + def cast(value) case value when ::String if value[0] == '(' && value[-1] == ')' value = value[1...-1] end - type_cast(value.split(',')) + cast(value.split(',')) when ::Array value.map { |v| Float(v) } else @@ -23,7 +23,7 @@ module ActiveRecord end end - def type_cast_for_database(value) + def serialize(value) if value.is_a?(::Array) "(#{number_for_point(value[0])},#{number_for_point(value[1])})" else diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/oid/range.rb b/activerecord/lib/active_record/connection_adapters/postgresql/oid/range.rb index 3adfb8b9d8..fc201f8fb9 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql/oid/range.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql/oid/range.rb @@ -7,7 +7,7 @@ module ActiveRecord class Range < Type::Value # :nodoc: attr_reader :subtype, :type - def initialize(subtype, type) + def initialize(subtype, type = :range) @subtype = subtype @type = type end @@ -30,7 +30,7 @@ module ActiveRecord ::Range.new(from, to, extracted[:exclude_end]) end - def type_cast_for_database(value) + def serialize(value) if value.is_a?(::Range) from = type_cast_single_for_database(value.begin) to = type_cast_single_for_database(value.end) @@ -40,26 +40,42 @@ module ActiveRecord end end + def ==(other) + other.is_a?(Range) && + other.subtype == subtype && + other.type == type + end + private def type_cast_single(value) - infinity?(value) ? value : @subtype.type_cast_from_database(value) + infinity?(value) ? value : @subtype.deserialize(value) end def type_cast_single_for_database(value) - infinity?(value) ? '' : @subtype.type_cast_for_database(value) + infinity?(value) ? '' : @subtype.serialize(value) end def extract_bounds(value) from, to = value[1..-2].split(',') { - from: (value[1] == ',' || from == '-infinity') ? @subtype.infinity(negative: true) : from, - to: (value[-2] == ',' || to == 'infinity') ? @subtype.infinity : to, + from: (value[1] == ',' || from == '-infinity') ? infinity(negative: true) : from, + to: (value[-2] == ',' || to == 'infinity') ? infinity : to, exclude_start: (value[0] == '('), exclude_end: (value[-1] == ')') } end + def infinity(negative: false) + if subtype.respond_to?(:infinity) + subtype.infinity(negative: negative) + elsif negative + -::Float::INFINITY + else + ::Float::INFINITY + end + end + def infinity?(value) value.respond_to?(:infinite?) && value.infinite? end diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/oid/specialized_string.rb b/activerecord/lib/active_record/connection_adapters/postgresql/oid/specialized_string.rb index b2a42e9ebb..2d2fede4e8 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql/oid/specialized_string.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql/oid/specialized_string.rb @@ -8,10 +8,6 @@ module ActiveRecord def initialize(type) @type = type end - - def text? - false - end end end end diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/oid/time.rb b/activerecord/lib/active_record/connection_adapters/postgresql/oid/time.rb deleted file mode 100644 index 8f0246eddb..0000000000 --- a/activerecord/lib/active_record/connection_adapters/postgresql/oid/time.rb +++ /dev/null @@ -1,11 +0,0 @@ -module ActiveRecord - module ConnectionAdapters - module PostgreSQL - module OID # :nodoc: - class Time < Type::Time # :nodoc: - include Infinity - end - end - end - end -end diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/oid/type_map_initializer.rb b/activerecord/lib/active_record/connection_adapters/postgresql/oid/type_map_initializer.rb index 9b3de41fab..191c828e60 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql/oid/type_map_initializer.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql/oid/type_map_initializer.rb @@ -15,11 +15,11 @@ module ActiveRecord def run(records) nodes = records.reject { |row| @store.key? row['oid'].to_i } mapped, nodes = nodes.partition { |row| @store.key? row['typname'] } - ranges, nodes = nodes.partition { |row| row['typtype'] == 'r' } - enums, nodes = nodes.partition { |row| row['typtype'] == 'e' } - domains, nodes = nodes.partition { |row| row['typtype'] == 'd' } - arrays, nodes = nodes.partition { |row| row['typinput'] == 'array_in' } - composites, nodes = nodes.partition { |row| row['typelem'] != '0' } + ranges, nodes = nodes.partition { |row| row['typtype'] == 'r'.freeze } + enums, nodes = nodes.partition { |row| row['typtype'] == 'e'.freeze } + domains, nodes = nodes.partition { |row| row['typtype'] == 'd'.freeze } + arrays, nodes = nodes.partition { |row| row['typinput'] == 'array_in'.freeze } + composites, nodes = nodes.partition { |row| row['typelem'].to_i != 0 } mapped.each { |row| register_mapped_type(row) } enums.each { |row| register_enum_type(row) } @@ -29,6 +29,18 @@ module ActiveRecord composites.each { |row| register_composite_type(row) } end + def query_conditions_for_initial_load(type_map) + known_type_names = type_map.keys.map { |n| "'#{n}'" } + known_type_types = %w('r' 'e' 'd') + <<-SQL % [known_type_names.join(", "), known_type_types.join(", ")] + WHERE + t.typname IN (%s) + OR t.typtype IN (%s) + OR t.typinput::varchar = 'array_in' + OR t.typelem != 0 + SQL + end + private def register_mapped_type(row) alias_type row['oid'], row['typname'] diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/oid/uuid.rb b/activerecord/lib/active_record/connection_adapters/postgresql/oid/uuid.rb index 97b4fd3d08..5e839228e9 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql/oid/uuid.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql/oid/uuid.rb @@ -5,13 +5,13 @@ module ActiveRecord class Uuid < Type::Value # :nodoc: ACCEPTABLE_UUID = %r{\A\{?([a-fA-F0-9]{4}-?){8}\}?\z}x - alias_method :type_cast_for_database, :type_cast_from_database + alias_method :serialize, :deserialize def type :uuid end - def type_cast(value) + def cast(value) value.to_s[ACCEPTABLE_UUID, 0] end end diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/oid/vector.rb b/activerecord/lib/active_record/connection_adapters/postgresql/oid/vector.rb index de4187b028..b26e876b54 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql/oid/vector.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql/oid/vector.rb @@ -16,7 +16,7 @@ module ActiveRecord # FIXME: this should probably split on +delim+ and use +subtype+ # to cast the values. Unfortunately, the current Rails behavior # is to just return the string. - def type_cast(value) + def cast(value) value end end diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/oid/xml.rb b/activerecord/lib/active_record/connection_adapters/postgresql/oid/xml.rb index 334af7c598..d40d837cee 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql/oid/xml.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql/oid/xml.rb @@ -7,7 +7,7 @@ module ActiveRecord :xml end - def type_cast_for_database(value) + def serialize(value) return unless value Data.new(super) end diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/quoting.rb b/activerecord/lib/active_record/connection_adapters/postgresql/quoting.rb index 9de9e2c7dc..f175730551 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql/quoting.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql/quoting.rb @@ -40,8 +40,7 @@ module ActiveRecord PGconn.quote_ident(name.to_s) end - # Quote date/time values for use in SQL input. Includes microseconds - # if the value is a Time responding to usec. + # Quote date/time values for use in SQL input. def quoted_date(value) #:nodoc: if value.year <= 0 bce_year = format("%04d", -value.year + 1) @@ -52,15 +51,21 @@ module ActiveRecord end # Does not quote function default values for UUID columns - def quote_default_value(value, column) #:nodoc: + def quote_default_expression(value, column) #:nodoc: if column.type == :uuid && value =~ /\(\)/ value - else - value = column.cast_type.type_cast_for_database(value) + elsif column.respond_to?(:array?) + value = type_cast_from_column(column, value) quote(value) + else + super end end + def lookup_cast_type_from_column(column) # :nodoc: + type_map.lookup(column.oid, column.fmod, column.sql_type) + end + private def _quote(value) diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/referential_integrity.rb b/activerecord/lib/active_record/connection_adapters/postgresql/referential_integrity.rb index 52b307c432..44a7338bf5 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql/referential_integrity.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql/referential_integrity.rb @@ -8,20 +8,39 @@ module ActiveRecord def disable_referential_integrity # :nodoc: if supports_disable_referential_integrity? + original_exception = nil + begin - execute(tables.collect { |name| "ALTER TABLE #{quote_table_name(name)} DISABLE TRIGGER ALL" }.join(";")) - rescue - execute(tables.collect { |name| "ALTER TABLE #{quote_table_name(name)} DISABLE TRIGGER USER" }.join(";")) + transaction(requires_new: true) do + execute(tables.collect { |name| "ALTER TABLE #{quote_table_name(name)} DISABLE TRIGGER ALL" }.join(";")) + end + rescue ActiveRecord::ActiveRecordError => e + original_exception = e end - end - yield - ensure - if supports_disable_referential_integrity? + + begin + yield + rescue ActiveRecord::InvalidForeignKey => e + warn <<-WARNING +WARNING: Rails was not able to disable referential integrity. + +This is most likely caused due to missing permissions. +Rails needs superuser privileges to disable referential integrity. + + cause: #{original_exception.try(:message)} + + WARNING + raise e + end + begin - execute(tables.collect { |name| "ALTER TABLE #{quote_table_name(name)} ENABLE TRIGGER ALL" }.join(";")) - rescue - execute(tables.collect { |name| "ALTER TABLE #{quote_table_name(name)} ENABLE TRIGGER USER" }.join(";")) + transaction(requires_new: true) do + execute(tables.collect { |name| "ALTER TABLE #{quote_table_name(name)} ENABLE TRIGGER ALL" }.join(";")) + end + rescue ActiveRecord::ActiveRecordError end + else + yield end end end diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/schema_definitions.rb b/activerecord/lib/active_record/connection_adapters/postgresql/schema_definitions.rb index a9522e152f..022dbdfa27 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql/schema_definitions.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql/schema_definitions.rb @@ -2,90 +2,129 @@ module ActiveRecord module ConnectionAdapters module PostgreSQL module ColumnMethods - def xml(*args) - options = args.extract_options! - column(args[0], :xml, options) + # Defines the primary key field. + # Use of the native PostgreSQL UUID type is supported, and can be used + # by defining your tables as such: + # + # create_table :stuffs, id: :uuid do |t| + # t.string :content + # t.timestamps + # end + # + # By default, this will use the +uuid_generate_v4()+ function from the + # +uuid-ossp+ extension, which MUST be enabled on your database. To enable + # the +uuid-ossp+ extension, you can use the +enable_extension+ method in your + # migrations. To use a UUID primary key without +uuid-ossp+ enabled, you can + # set the +:default+ option to +nil+: + # + # create_table :stuffs, id: false do |t| + # t.primary_key :id, :uuid, default: nil + # t.uuid :foo_id + # t.timestamps + # end + # + # You may also pass a different UUID generation function from +uuid-ossp+ + # or another library. + # + # Note that setting the UUID primary key default value to +nil+ will + # require you to assure that you always provide a UUID value before saving + # a record (as primary keys cannot be +nil+). This might be done via the + # +SecureRandom.uuid+ method and a +before_save+ callback, for instance. + def primary_key(name, type = :primary_key, **options) + options[:default] = options.fetch(:default, 'uuid_generate_v4()') if type == :uuid + super + end + + def bigserial(*args, **options) + args.each { |name| column(name, :bigserial, options) } + end + + def bit(*args, **options) + args.each { |name| column(name, :bit, options) } end - def tsvector(*args) - options = args.extract_options! - column(args[0], :tsvector, options) + def bit_varying(*args, **options) + args.each { |name| column(name, :bit_varying, options) } end - def int4range(name, options = {}) - column(name, :int4range, options) + def cidr(*args, **options) + args.each { |name| column(name, :cidr, options) } end - def int8range(name, options = {}) - column(name, :int8range, options) + def citext(*args, **options) + args.each { |name| column(name, :citext, options) } end - def tsrange(name, options = {}) - column(name, :tsrange, options) + def daterange(*args, **options) + args.each { |name| column(name, :daterange, options) } end - def tstzrange(name, options = {}) - column(name, :tstzrange, options) + def hstore(*args, **options) + args.each { |name| column(name, :hstore, options) } end - def numrange(name, options = {}) - column(name, :numrange, options) + def inet(*args, **options) + args.each { |name| column(name, :inet, options) } end - def daterange(name, options = {}) - column(name, :daterange, options) + def int4range(*args, **options) + args.each { |name| column(name, :int4range, options) } end - def hstore(name, options = {}) - column(name, :hstore, options) + def int8range(*args, **options) + args.each { |name| column(name, :int8range, options) } end - def ltree(name, options = {}) - column(name, :ltree, options) + def json(*args, **options) + args.each { |name| column(name, :json, options) } end - def inet(name, options = {}) - column(name, :inet, options) + def jsonb(*args, **options) + args.each { |name| column(name, :jsonb, options) } end - def cidr(name, options = {}) - column(name, :cidr, options) + def ltree(*args, **options) + args.each { |name| column(name, :ltree, options) } end - def macaddr(name, options = {}) - column(name, :macaddr, options) + def macaddr(*args, **options) + args.each { |name| column(name, :macaddr, options) } end - def uuid(name, options = {}) - column(name, :uuid, options) + def money(*args, **options) + args.each { |name| column(name, :money, options) } end - def json(name, options = {}) - column(name, :json, options) + def numrange(*args, **options) + args.each { |name| column(name, :numrange, options) } end - def jsonb(name, options = {}) - column(name, :jsonb, options) + def point(*args, **options) + args.each { |name| column(name, :point, options) } end - def citext(name, options = {}) - column(name, :citext, options) + def serial(*args, **options) + args.each { |name| column(name, :serial, options) } end - def point(name, options = {}) - column(name, :point, options) + def tsrange(*args, **options) + args.each { |name| column(name, :tsrange, options) } end - def bit(name, options) - column(name, :bit, options) + def tstzrange(*args, **options) + args.each { |name| column(name, :tstzrange, options) } end - def bit_varying(name, options) - column(name, :bit_varying, options) + def tsvector(*args, **options) + args.each { |name| column(name, :tsvector, options) } end - def money(name, options) - column(name, :money, options) + def uuid(*args, **options) + args.each { |name| column(name, :uuid, options) } + end + + def xml(*args, **options) + args.each { |name| column(name, :xml, options) } end end @@ -96,39 +135,6 @@ module ActiveRecord class TableDefinition < ActiveRecord::ConnectionAdapters::TableDefinition include ColumnMethods - # Defines the primary key field. - # Use of the native PostgreSQL UUID type is supported, and can be used - # by defining your tables as such: - # - # create_table :stuffs, id: :uuid do |t| - # t.string :content - # t.timestamps - # end - # - # By default, this will use the +uuid_generate_v4()+ function from the - # +uuid-ossp+ extension, which MUST be enabled on your database. To enable - # the +uuid-ossp+ extension, you can use the +enable_extension+ method in your - # migrations. To use a UUID primary key without +uuid-ossp+ enabled, you can - # set the +:default+ option to +nil+: - # - # create_table :stuffs, id: false do |t| - # t.primary_key :id, :uuid, default: nil - # t.uuid :foo_id - # t.timestamps - # end - # - # You may also pass a different UUID generation function from +uuid-ossp+ - # or another library. - # - # Note that setting the UUID primary key default value to +nil+ will - # require you to assure that you always provide a UUID value before saving - # a record (as primary keys cannot be +nil+). This might be done via the - # +SecureRandom.uuid+ method and a +before_save+ callback, for instance. - def primary_key(name, type = :primary_key, options = {}) - options[:default] = options.fetch(:default, 'uuid_generate_v4()') if type == :uuid - super - end - def new_column_definition(name, type, options) # :nodoc: column = super column.array = options[:array] diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/schema_statements.rb b/activerecord/lib/active_record/connection_adapters/postgresql/schema_statements.rb index a90adcf4aa..595c635fc0 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql/schema_statements.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql/schema_statements.rb @@ -4,34 +4,17 @@ module ActiveRecord class SchemaCreation < AbstractAdapter::SchemaCreation private - def column_options(o) - column_options = super - column_options[:array] = o.array - column_options + def visit_ColumnDefinition(o) + o.sql_type = type_to_sql(o.type, o.limit, o.precision, o.scale, o.array) + super end def add_column_options!(sql, options) - if options[:array] - sql << '[]' + if options[:collation] + sql << " COLLATE \"#{options[:collation]}\"" end super end - - def quote_default_expression(value, column) - if column.type == :uuid && value =~ /\(\)/ - value - else - super - end - end - - def type_for_column(column) - if column.array - @conn.lookup_cast_type("#{column.sql_type}[]") - else - super - end - end end module SchemaStatements @@ -87,11 +70,7 @@ module ActiveRecord # Returns the list of all tables in the schema search path or a specified schema. def tables(name = nil) - query(<<-SQL, 'SCHEMA').map { |row| row[0] } - SELECT tablename - FROM pg_tables - WHERE schemaname = ANY (current_schemas(false)) - SQL + select_values("SELECT tablename FROM pg_tables WHERE schemaname = ANY(current_schemas(false))", 'SCHEMA') end # Returns true if table exists. @@ -101,7 +80,7 @@ module ActiveRecord name = Utils.extract_schema_qualified_name(name.to_s) return false unless name.identifier - exec_query(<<-SQL, 'SCHEMA').rows.first[0].to_i > 0 + select_value(<<-SQL, 'SCHEMA').to_i > 0 SELECT COUNT(*) FROM pg_class c LEFT JOIN pg_namespace n ON n.oid = c.relnamespace @@ -111,21 +90,18 @@ module ActiveRecord SQL end - def drop_table(table_name, options = {}) - execute "DROP TABLE #{quote_table_name(table_name)}#{' CASCADE' if options[:force] == :cascade}" + def drop_table(table_name, options = {}) # :nodoc: + execute "DROP TABLE#{' IF EXISTS' if options[:if_exists]} #{quote_table_name(table_name)}#{' CASCADE' if options[:force] == :cascade}" end # Returns true if schema exists. def schema_exists?(name) - exec_query(<<-SQL, 'SCHEMA').rows.first[0].to_i > 0 - SELECT COUNT(*) - FROM pg_namespace - WHERE nspname = '#{name}' - SQL + select_value("SELECT COUNT(*) FROM pg_namespace WHERE nspname = '#{name}'", 'SCHEMA').to_i > 0 end + # Verifies existence of an index with a given name. def index_name_exists?(table_name, index_name, default) - exec_query(<<-SQL, 'SCHEMA').rows.first[0].to_i > 0 + select_value(<<-SQL, 'SCHEMA').to_i > 0 SELECT COUNT(*) FROM pg_class t INNER JOIN pg_index d ON t.oid = d.indrelid @@ -153,8 +129,8 @@ module ActiveRecord result.map do |row| index_name = row[0] - unique = row[1] == 't' - indkey = row[2].split(" ") + unique = row[1] + indkey = row[2].split(" ").map(&:to_i) inddef = row[3] oid = row[4] @@ -182,53 +158,48 @@ module ActiveRecord # Returns the list of all column definitions for a table. def columns(table_name) # Limit, precision, and scale are all handled by the superclass. - column_definitions(table_name).map do |column_name, type, default, notnull, oid, fmod| - oid = get_oid_type(oid.to_i, fmod.to_i, column_name, type) - default_value = extract_value_from_default(oid, default) + column_definitions(table_name).map do |column_name, type, default, notnull, oid, fmod, collation| + oid = oid.to_i + fmod = fmod.to_i + type_metadata = fetch_type_metadata(column_name, type, oid, fmod) + default_value = extract_value_from_default(default) default_function = extract_default_function(default_value, default) - new_column(column_name, default_value, oid, type, notnull == 'f', default_function) + new_column(column_name, default_value, type_metadata, !notnull, default_function, collation) end end - def new_column(name, default, cast_type, sql_type = nil, null = true, default_function = nil) # :nodoc: - PostgreSQLColumn.new(name, default, cast_type, sql_type, null, default_function) + def new_column(name, default, sql_type_metadata = nil, null = true, default_function = nil, collation = nil) # :nodoc: + PostgreSQLColumn.new(name, default, sql_type_metadata, null, default_function, collation) end # Returns the current database name. def current_database - query('select current_database()', 'SCHEMA')[0][0] + select_value('select current_database()', 'SCHEMA') end # Returns the current schema name. def current_schema - query('SELECT current_schema', 'SCHEMA')[0][0] + select_value('SELECT current_schema', 'SCHEMA') end # Returns the current database encoding format. def encoding - query(<<-end_sql, 'SCHEMA')[0][0] - SELECT pg_encoding_to_char(pg_database.encoding) FROM pg_database - WHERE pg_database.datname LIKE '#{current_database}' - end_sql + select_value("SELECT pg_encoding_to_char(encoding) FROM pg_database WHERE datname LIKE '#{current_database}'", 'SCHEMA') end # Returns the current database collation. def collation - query(<<-end_sql, 'SCHEMA')[0][0] - SELECT pg_database.datcollate FROM pg_database WHERE pg_database.datname LIKE '#{current_database}' - end_sql + select_value("SELECT datcollate FROM pg_database WHERE datname LIKE '#{current_database}'", 'SCHEMA') end # Returns the current database ctype. def ctype - query(<<-end_sql, 'SCHEMA')[0][0] - SELECT pg_database.datctype FROM pg_database WHERE pg_database.datname LIKE '#{current_database}' - end_sql + select_value("SELECT datctype FROM pg_database WHERE datname LIKE '#{current_database}'", 'SCHEMA') end # Returns an array of schema names. def schema_names - query(<<-SQL, 'SCHEMA').flatten + select_values(<<-SQL, 'SCHEMA') SELECT nspname FROM pg_namespace WHERE nspname !~ '^pg_.*' @@ -261,12 +232,12 @@ module ActiveRecord # Returns the active schema search path. def schema_search_path - @schema_search_path ||= query('SHOW search_path', 'SCHEMA')[0][0] + @schema_search_path ||= select_value('SHOW search_path', 'SCHEMA') end # Returns the current client message level. def client_min_messages - query('SHOW client_min_messages', 'SCHEMA')[0][0] + select_value('SHOW client_min_messages', 'SCHEMA') end # Set the client message level. @@ -284,10 +255,7 @@ module ActiveRecord end def serial_sequence(table, column) - result = exec_query(<<-eosql, 'SCHEMA') - SELECT pg_get_serial_sequence('#{table}', '#{column}') - eosql - result.rows.first.first + select_value("SELECT pg_get_serial_sequence('#{table}', '#{column}')", 'SCHEMA') end # Sets the sequence of a table's primary key to the specified value. @@ -298,9 +266,7 @@ module ActiveRecord if sequence quoted_sequence = quote_table_name(sequence) - select_value <<-end_sql, 'SCHEMA' - SELECT setval('#{quoted_sequence}', #{value}) - end_sql + select_value("SELECT setval('#{quoted_sequence}', #{value})", 'SCHEMA') else @logger.warn "#{table} has primary key #{pk} with no default sequence" if @logger end @@ -323,7 +289,7 @@ module ActiveRecord if pk && sequence quoted_sequence = quote_table_name(sequence) - select_value <<-end_sql, 'SCHEMA' + select_value(<<-end_sql, 'SCHEMA') SELECT setval('#{quoted_sequence}', (SELECT COALESCE(MAX(#{quote_column_name pk})+(SELECT increment_by FROM #{quoted_sequence}), (SELECT min_value FROM #{quoted_sequence})) FROM #{quote_table_name(table)}), false) end_sql end @@ -385,7 +351,7 @@ module ActiveRecord # Returns just a table's primary key def primary_key(table) - pks = exec_query(<<-end_sql, 'SCHEMA').rows + pks = query(<<-end_sql, 'SCHEMA') SELECT attr.attname FROM pg_attribute attr INNER JOIN pg_constraint cons ON attr.attrelid = cons.conrelid AND attr.attnum = any(cons.conkey) @@ -417,23 +383,25 @@ module ActiveRecord rename_table_indexes(table_name, new_name) end - # Adds a new column to the named table. - # See TableDefinition#column for details of the options you can use. - def add_column(table_name, column_name, type, options = {}) + def add_column(table_name, column_name, type, options = {}) #:nodoc: clear_cache! super end - # Changes the column of a table. - def change_column(table_name, column_name, type, options = {}) + def change_column(table_name, column_name, type, options = {}) #:nodoc: clear_cache! quoted_table_name = quote_table_name(table_name) - sql_type = type_to_sql(type, options[:limit], options[:precision], options[:scale]) - sql_type << "[]" if options[:array] - sql = "ALTER TABLE #{quoted_table_name} ALTER COLUMN #{quote_column_name(column_name)} TYPE #{sql_type}" - sql << " USING #{options[:using]}" if options[:using] - if options[:cast_as] - sql << " USING CAST(#{quote_column_name(column_name)} AS #{type_to_sql(options[:cast_as], options[:limit], options[:precision], options[:scale])})" + quoted_column_name = quote_column_name(column_name) + sql_type = type_to_sql(type, options[:limit], options[:precision], options[:scale], options[:array]) + sql = "ALTER TABLE #{quoted_table_name} ALTER COLUMN #{quoted_column_name} TYPE #{sql_type}" + if options[:collation] + sql << " COLLATE \"#{options[:collation]}\"" + end + if options[:using] + sql << " USING #{options[:using]}" + elsif options[:cast_as] + cast_as_type = type_to_sql(options[:cast_as], options[:limit], options[:precision], options[:scale], options[:array]) + sql << " USING CAST(#{quoted_column_name} AS #{cast_as_type})" end execute sql @@ -442,7 +410,7 @@ module ActiveRecord end # Changes the default value of a table column. - def change_column_default(table_name, column_name, default) + def change_column_default(table_name, column_name, default) # :nodoc: clear_cache! column = column_for(table_name, column_name) return unless column @@ -453,21 +421,21 @@ module ActiveRecord # cast the default to the columns type, which leaves us with a default like "default NULL::character varying". execute alter_column_query % "DROP DEFAULT" else - execute alter_column_query % "SET DEFAULT #{quote_default_value(default, column)}" + execute alter_column_query % "SET DEFAULT #{quote_default_expression(default, column)}" end end - def change_column_null(table_name, column_name, null, default = nil) + def change_column_null(table_name, column_name, null, default = nil) #:nodoc: clear_cache! unless null || default.nil? column = column_for(table_name, column_name) - execute("UPDATE #{quote_table_name(table_name)} SET #{quote_column_name(column_name)}=#{quote_default_value(default, column)} WHERE #{quote_column_name(column_name)} IS NULL") if column + execute("UPDATE #{quote_table_name(table_name)} SET #{quote_column_name(column_name)}=#{quote_default_expression(default, column)} WHERE #{quote_column_name(column_name)} IS NULL") if column end execute("ALTER TABLE #{quote_table_name(table_name)} ALTER #{quote_column_name(column_name)} #{null ? 'DROP' : 'SET'} NOT NULL") end # Renames a column in a table. - def rename_column(table_name, column_name, new_column_name) + def rename_column(table_name, column_name, new_column_name) #:nodoc: clear_cache! execute "ALTER TABLE #{quote_table_name(table_name)} RENAME COLUMN #{quote_column_name(column_name)} TO #{quote_column_name(new_column_name)}" rename_column_indexes(table_name, column_name, new_column_name) @@ -482,6 +450,8 @@ module ActiveRecord execute "DROP INDEX #{quote_table_name(index_name)}" end + # Renames an index of a table. Raises error if length of new + # index name is greater than allowed limit. def rename_index(table_name, old_name, new_name) validate_index_length!(table_name, new_name) @@ -530,41 +500,35 @@ module ActiveRecord end # Maps logical Rails types to PostgreSQL-specific data types. - def type_to_sql(type, limit = nil, precision = nil, scale = nil) - case type.to_s + def type_to_sql(type, limit = nil, precision = nil, scale = nil, array = nil) + sql = case type.to_s when 'binary' # PostgreSQL doesn't support limits on binary (bytea) columns. - # The hard limit is 1Gb, because of a 32-bit size field, and TOAST. + # The hard limit is 1GB, because of a 32-bit size field, and TOAST. case limit when nil, 0..0x3fffffff; super(type) else raise(ActiveRecordError, "No binary type has byte size #{limit}.") end when 'text' # PostgreSQL doesn't support limits on text columns. - # The hard limit is 1Gb, according to section 8.3 in the manual. + # The hard limit is 1GB, according to section 8.3 in the manual. case limit when nil, 0..0x3fffffff; super(type) else raise(ActiveRecordError, "The limit on text can be at most 1GB - 1byte.") end when 'integer' - return 'integer' unless limit - case limit - when 1, 2; 'smallint' - when 3, 4; 'integer' - when 5..8; 'bigint' - else raise(ActiveRecordError, "No integer type has byte size #{limit}. Use a numeric with precision 0 instead.") - end - when 'datetime' - return super unless precision - - case precision - when 0..6; "timestamp(#{precision})" - else raise(ActiveRecordError, "No timestamp type has precision of #{precision}. The allowed range of precision is from 0 to 6") + when 1, 2; 'smallint' + when nil, 3, 4; 'integer' + when 5..8; 'bigint' + else raise(ActiveRecordError, "No integer type has byte size #{limit}. Use a numeric with precision 0 instead.") end else - super + super(type, limit, precision, scale) end + + sql << '[]' if array && type != :primary_key + sql end # PostgreSQL requires the ORDER BY columns in the select list for distinct queries, and @@ -580,6 +544,18 @@ module ActiveRecord [super, *order_columns].join(', ') end + + def fetch_type_metadata(column_name, sql_type, oid, fmod) + cast_type = get_oid_type(oid, fmod, column_name, sql_type) + simple_type = SqlTypeMetadata.new( + sql_type: sql_type, + type: cast_type.type, + limit: cast_type.limit, + precision: cast_type.precision, + scale: cast_type.scale, + ) + PostgreSQLTypeMetadata.new(simple_type, oid: oid, fmod: fmod) + end end end end diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/type_metadata.rb b/activerecord/lib/active_record/connection_adapters/postgresql/type_metadata.rb new file mode 100644 index 0000000000..58715978f7 --- /dev/null +++ b/activerecord/lib/active_record/connection_adapters/postgresql/type_metadata.rb @@ -0,0 +1,35 @@ +module ActiveRecord + module ConnectionAdapters + class PostgreSQLTypeMetadata < DelegateClass(SqlTypeMetadata) + attr_reader :oid, :fmod, :array + + def initialize(type_metadata, oid: nil, fmod: nil) + super(type_metadata) + @type_metadata = type_metadata + @oid = oid + @fmod = fmod + @array = /\[\]$/ === type_metadata.sql_type + end + + def sql_type + super.gsub(/\[\]$/, "") + end + + def ==(other) + other.is_a?(PostgreSQLTypeMetadata) && + attributes_for_hash == other.attributes_for_hash + end + alias eql? == + + def hash + attributes_for_hash.hash + end + + protected + + def attributes_for_hash + [self.class, @type_metadata, oid, fmod] + end + end + end +end diff --git a/activerecord/lib/active_record/connection_adapters/postgresql_adapter.rb b/activerecord/lib/active_record/connection_adapters/postgresql_adapter.rb index f4f9747359..93fa3984e6 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql_adapter.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql_adapter.rb @@ -1,21 +1,19 @@ -require 'active_record/connection_adapters/abstract_adapter' -require 'active_record/connection_adapters/statement_pool' - -require 'active_record/connection_adapters/postgresql/utils' -require 'active_record/connection_adapters/postgresql/column' -require 'active_record/connection_adapters/postgresql/oid' -require 'active_record/connection_adapters/postgresql/quoting' -require 'active_record/connection_adapters/postgresql/referential_integrity' -require 'active_record/connection_adapters/postgresql/schema_definitions' -require 'active_record/connection_adapters/postgresql/schema_statements' -require 'active_record/connection_adapters/postgresql/database_statements' - -require 'arel/visitors/bind_visitor' - -# Make sure we're using pg high enough for PGResult#values -gem 'pg', '~> 0.15' +# Make sure we're using pg high enough for type casts and Ruby 2.2+ compatibility +gem 'pg', '~> 0.18' require 'pg' +require "active_record/connection_adapters/abstract_adapter" +require "active_record/connection_adapters/postgresql/column" +require "active_record/connection_adapters/postgresql/database_statements" +require "active_record/connection_adapters/postgresql/oid" +require "active_record/connection_adapters/postgresql/quoting" +require "active_record/connection_adapters/postgresql/referential_integrity" +require "active_record/connection_adapters/postgresql/schema_definitions" +require "active_record/connection_adapters/postgresql/schema_statements" +require "active_record/connection_adapters/postgresql/type_metadata" +require "active_record/connection_adapters/postgresql/utils" +require "active_record/connection_adapters/statement_pool" + require 'ipaddr' module ActiveRecord @@ -68,11 +66,11 @@ module ActiveRecord # defaults to true. # # Any further options are used as connection parameters to libpq. See - # http://www.postgresql.org/docs/9.1/static/libpq-connect.html for the + # http://www.postgresql.org/docs/current/static/libpq-connect.html for the # list of parameters. # # In addition, default connection parameters of libpq can be set per environment variables. - # See http://www.postgresql.org/docs/9.1/static/libpq-envars.html . + # See http://www.postgresql.org/docs/current/static/libpq-envars.html . class PostgreSQLAdapter < AbstractAdapter ADAPTER_NAME = 'PostgreSQL'.freeze @@ -128,7 +126,7 @@ module ActiveRecord def column_spec_for_primary_key(column) spec = {} if column.serial? - return unless column.sql_type == 'bigint' + return unless column.bigint? spec[:id] = ':bigserial' elsif column.type == :uuid spec[:id] = ':uuid' @@ -145,7 +143,6 @@ module ActiveRecord def prepare_column_options(column) # :nodoc: spec = super spec[:array] = 'true' if column.array? - spec[:default] = "\"#{column.default_function}\"" if column.default_function spec end @@ -154,6 +151,26 @@ module ActiveRecord super + [:array] end + def schema_type(column) + return super unless column.serial? + + if column.bigint? + 'bigserial' + else + 'serial' + end + end + private :schema_type + + def schema_default(column) + if column.default_function + column.default_function.inspect unless column.serial? + else + super + end + end + private :schema_default + # Returns +true+, since this connection adapter supports prepared statement # caching. def supports_statement_cache? @@ -180,6 +197,10 @@ module ActiveRecord true end + def supports_datetime_with_precision? + true + end + def index_algorithms { concurrently: 'CONCURRENTLY' } end @@ -188,44 +209,18 @@ module ActiveRecord def initialize(connection, max) super @counter = 0 - @cache = Hash.new { |h,pid| h[pid] = {} } end - def each(&block); cache.each(&block); end - def key?(key); cache.key?(key); end - def [](key); cache[key]; end - def length; cache.length; end - def next_key "a#{@counter + 1}" end def []=(sql, key) - while @max <= cache.size - dealloc(cache.shift.last) - end - @counter += 1 - cache[sql] = key - end - - def clear - cache.each_value do |stmt_key| - dealloc stmt_key - end - cache.clear - end - - def delete(sql_key) - dealloc cache[sql_key] - cache.delete sql_key + super.tap { @counter += 1 } end private - def cache - @cache[Process.pid] - end - def dealloc(key) @connection.query "DEALLOCATE #{key}" if connection_active? end @@ -255,6 +250,7 @@ module ActiveRecord @table_alias_length = nil connect + add_pg_encoders @statements = StatementPool.new @connection, self.class.type_cast_config_to_integer(config.fetch(:statement_limit) { 1000 }) @@ -262,6 +258,8 @@ module ActiveRecord raise "Your version of PostgreSQL (#{postgresql_version}) is too old, please upgrade!" end + add_pg_decoders + @type_map = Type::HashLookupTypeMap.new initialize_type_map(type_map) @local_tz = execute('SHOW TIME ZONE', 'SCHEMA').first["TimeZone"] @@ -426,7 +424,7 @@ module ActiveRecord @connection.server_version end - # See http://www.postgresql.org/docs/9.1/static/errcodes-appendix.html + # See http://www.postgresql.org/docs/current/static/errcodes-appendix.html FOREIGN_KEY_VIOLATION = "23503" UNIQUE_VIOLATION = "23505" @@ -459,11 +457,11 @@ module ActiveRecord end def initialize_type_map(m) # :nodoc: - register_class_with_limit m, 'int2', OID::Integer - register_class_with_limit m, 'int4', OID::Integer - register_class_with_limit m, 'int8', OID::Integer + register_class_with_limit m, 'int2', Type::Integer + register_class_with_limit m, 'int4', Type::Integer + register_class_with_limit m, 'int8', Type::Integer m.alias_type 'oid', 'int2' - m.register_type 'float4', OID::Float.new + m.register_type 'float4', Type::Float.new m.alias_type 'float8', 'float4' m.register_type 'text', Type::Text.new register_class_with_limit m, 'varchar', Type::String @@ -474,8 +472,7 @@ module ActiveRecord register_class_with_limit m, 'bit', OID::Bit register_class_with_limit m, 'varbit', OID::BitVarying m.alias_type 'timestamptz', 'timestamp' - m.register_type 'date', OID::Date.new - m.register_type 'time', OID::Time.new + m.register_type 'date', Type::Date.new m.register_type 'money', OID::Money.new m.register_type 'bytea', OID::Bytea.new @@ -501,10 +498,8 @@ module ActiveRecord m.alias_type 'lseg', 'varchar' m.alias_type 'box', 'varchar' - m.register_type 'timestamp' do |_, _, sql_type| - precision = extract_precision(sql_type) - OID::DateTime.new(precision: precision) - end + register_class_with_precision m, 'time', Type::Time + register_class_with_precision m, 'timestamp', OID::DateTime m.register_type 'numeric' do |_, fmod, sql_type| precision = extract_precision(sql_type) @@ -541,13 +536,13 @@ module ActiveRecord end # Extracts the value from a PostgreSQL column default definition. - def extract_value_from_default(oid, default) # :nodoc: + def extract_value_from_default(default) # :nodoc: case default # Quoted types when /\A[\(B]?'(.*)'::/m - $1.gsub(/''/, "'") + $1.gsub("''".freeze, "'".freeze) # Boolean types - when 'true', 'false' + when 'true'.freeze, 'false'.freeze default # Numeric types when /\A\(?(-?\d+(\.\d*)?)\)?(::bigint)?\z/ @@ -571,6 +566,8 @@ module ActiveRecord end def load_additional_types(type_map, oids = nil) # :nodoc: + initializer = OID::TypeMapInitializer.new(type_map) + if supports_ranges? query = <<-SQL SELECT t.oid, t.typname, t.typelem, t.typdelim, t.typinput, r.rngsubtype, t.typtype, t.typbasetype @@ -586,11 +583,13 @@ module ActiveRecord if oids query += "WHERE t.oid::integer IN (%s)" % oids.join(", ") + else + query += initializer.query_conditions_for_initial_load(type_map) end - initializer = OID::TypeMapInitializer.new(type_map) - records = execute(query, 'SCHEMA') - initializer.run(records) + execute_and_clear(query, 'SCHEMA', []) do |records| + initializer.run(records) + end end FEATURE_NOT_SUPPORTED = "0A000" #:nodoc: @@ -609,12 +608,10 @@ module ActiveRecord def exec_cache(sql, name, binds) stmt_key = prepare_statement(sql) - type_casted_binds = binds.map { |col, val| - [col, type_cast(val, col)] - } + type_casted_binds = binds.map { |attr| type_cast(attr.value_for_database) } - log(sql, name, type_casted_binds, stmt_key) do - @connection.exec_prepared(stmt_key, type_casted_binds.map { |_, val| val }) + log(sql, name, binds, stmt_key) do + @connection.exec_prepared(stmt_key, type_casted_binds) end rescue ActiveRecord::StatementInvalid => e pgerror = e.original_exception @@ -701,7 +698,7 @@ module ActiveRecord end # SET statements from :variables config hash - # http://www.postgresql.org/docs/8.3/static/sql-set.html + # http://www.postgresql.org/docs/current/static/sql-set.html variables = @config[:variables] || {} variables.map do |k, v| if v == ':default' || v == :default @@ -745,9 +742,11 @@ module ActiveRecord # - format_type includes the column size constraint, e.g. varchar(50) # - ::regclass is a function that gives the id for a table name def column_definitions(table_name) # :nodoc: - exec_query(<<-end_sql, 'SCHEMA').rows + query(<<-end_sql, 'SCHEMA') SELECT a.attname, format_type(a.atttypid, a.atttypmod), - pg_get_expr(d.adbin, d.adrelid), a.attnotnull, a.atttypid, a.atttypmod + pg_get_expr(d.adbin, d.adrelid), a.attnotnull, a.atttypid, a.atttypmod, + (SELECT c.collname FROM pg_collation c, pg_type t + WHERE c.oid = a.attcollation AND t.oid = a.atttypid AND a.attcollation <> t.typcollation) FROM pg_attribute a LEFT JOIN pg_attrdef d ON a.attrelid = d.adrelid AND a.attnum = d.adnum WHERE a.attrelid = '#{quote_table_name(table_name)}'::regclass @@ -764,6 +763,84 @@ module ActiveRecord def create_table_definition(name, temporary = false, options = nil, as = nil) # :nodoc: PostgreSQL::TableDefinition.new native_database_types, name, temporary, options, as end + + def can_perform_case_insensitive_comparison_for?(column) + @case_insensitive_cache ||= {} + @case_insensitive_cache[column.sql_type] ||= begin + sql = <<-end_sql + SELECT exists( + SELECT * FROM pg_proc + INNER JOIN pg_cast + ON casttarget::text::oidvector = proargtypes + WHERE proname = 'lower' + AND castsource = '#{column.sql_type}'::regtype::oid + ) + end_sql + execute_and_clear(sql, "SCHEMA", []) do |result| + result.getvalue(0, 0) + end + end + end + + def add_pg_encoders + map = PG::TypeMapByClass.new + map[Integer] = PG::TextEncoder::Integer.new + map[TrueClass] = PG::TextEncoder::Boolean.new + map[FalseClass] = PG::TextEncoder::Boolean.new + map[Float] = PG::TextEncoder::Float.new + @connection.type_map_for_queries = map + end + + def add_pg_decoders + coders_by_name = { + 'int2' => PG::TextDecoder::Integer, + 'int4' => PG::TextDecoder::Integer, + 'int8' => PG::TextDecoder::Integer, + 'oid' => PG::TextDecoder::Integer, + 'float4' => PG::TextDecoder::Float, + 'float8' => PG::TextDecoder::Float, + 'bool' => PG::TextDecoder::Boolean, + } + known_coder_types = coders_by_name.keys.map { |n| quote(n) } + query = <<-SQL % known_coder_types.join(", ") + SELECT t.oid, t.typname + FROM pg_type as t + WHERE t.typname IN (%s) + SQL + coders = execute_and_clear(query, "SCHEMA", []) do |result| + result + .map { |row| construct_coder(row, coders_by_name[row['typname']]) } + .compact + end + + map = PG::TypeMapByOid.new + coders.each { |coder| map.add_coder(coder) } + @connection.type_map_for_results = map + end + + def construct_coder(row, coder_class) + return unless coder_class + coder_class.new(oid: row['oid'].to_i, name: row['typname']) + end + + ActiveRecord::Type.add_modifier({ array: true }, OID::Array, adapter: :postgresql) + ActiveRecord::Type.add_modifier({ range: true }, OID::Range, adapter: :postgresql) + ActiveRecord::Type.register(:bit, OID::Bit, adapter: :postgresql) + ActiveRecord::Type.register(:bit_varying, OID::BitVarying, adapter: :postgresql) + ActiveRecord::Type.register(:binary, OID::Bytea, adapter: :postgresql) + ActiveRecord::Type.register(:cidr, OID::Cidr, adapter: :postgresql) + ActiveRecord::Type.register(:date_time, OID::DateTime, adapter: :postgresql) + ActiveRecord::Type.register(:decimal, OID::Decimal, adapter: :postgresql) + ActiveRecord::Type.register(:enum, OID::Enum, adapter: :postgresql) + ActiveRecord::Type.register(:hstore, OID::Hstore, adapter: :postgresql) + ActiveRecord::Type.register(:inet, OID::Inet, adapter: :postgresql) + ActiveRecord::Type.register(:json, OID::Json, adapter: :postgresql) + ActiveRecord::Type.register(:jsonb, OID::Jsonb, adapter: :postgresql) + ActiveRecord::Type.register(:money, OID::Money, adapter: :postgresql) + ActiveRecord::Type.register(:point, OID::Point, adapter: :postgresql) + ActiveRecord::Type.register(:uuid, OID::Uuid, adapter: :postgresql) + ActiveRecord::Type.register(:vector, OID::Vector, adapter: :postgresql) + ActiveRecord::Type.register(:xml, OID::Xml, adapter: :postgresql) end end end diff --git a/activerecord/lib/active_record/connection_adapters/schema_cache.rb b/activerecord/lib/active_record/connection_adapters/schema_cache.rb index 37ff4e4613..981d5d7a3c 100644 --- a/activerecord/lib/active_record/connection_adapters/schema_cache.rb +++ b/activerecord/lib/active_record/connection_adapters/schema_cache.rb @@ -13,6 +13,14 @@ module ActiveRecord @tables = {} end + def initialize_dup(other) + super + @columns = @columns.dup + @columns_hash = @columns_hash.dup + @primary_keys = @primary_keys.dup + @tables = @tables.dup + end + def primary_keys(table_name) @primary_keys[table_name] ||= table_exists?(table_name) ? connection.primary_key(table_name) : nil end diff --git a/activerecord/lib/active_record/connection_adapters/sql_type_metadata.rb b/activerecord/lib/active_record/connection_adapters/sql_type_metadata.rb new file mode 100644 index 0000000000..ccb7e154ee --- /dev/null +++ b/activerecord/lib/active_record/connection_adapters/sql_type_metadata.rb @@ -0,0 +1,32 @@ +module ActiveRecord + # :stopdoc: + module ConnectionAdapters + class SqlTypeMetadata + attr_reader :sql_type, :type, :limit, :precision, :scale + + def initialize(sql_type: nil, type: nil, limit: nil, precision: nil, scale: nil) + @sql_type = sql_type + @type = type + @limit = limit + @precision = precision + @scale = scale + end + + def ==(other) + other.is_a?(SqlTypeMetadata) && + attributes_for_hash == other.attributes_for_hash + end + alias eql? == + + def hash + attributes_for_hash.hash + end + + protected + + def attributes_for_hash + [self.class, sql_type, type, limit, precision, scale] + end + end + end +end diff --git a/activerecord/lib/active_record/connection_adapters/sqlite3/schema_creation.rb b/activerecord/lib/active_record/connection_adapters/sqlite3/schema_creation.rb new file mode 100644 index 0000000000..fe1dcbd710 --- /dev/null +++ b/activerecord/lib/active_record/connection_adapters/sqlite3/schema_creation.rb @@ -0,0 +1,15 @@ +module ActiveRecord + module ConnectionAdapters + module SQLite3 + class SchemaCreation < AbstractAdapter::SchemaCreation + private + def add_column_options!(sql, options) + if options[:collation] + sql << " COLLATE \"#{options[:collation]}\"" + end + super + end + end + end + end +end diff --git a/activerecord/lib/active_record/connection_adapters/sqlite3_adapter.rb b/activerecord/lib/active_record/connection_adapters/sqlite3_adapter.rb index 03dfd29a0a..87129c42cf 100644 --- a/activerecord/lib/active_record/connection_adapters/sqlite3_adapter.rb +++ b/activerecord/lib/active_record/connection_adapters/sqlite3_adapter.rb @@ -1,6 +1,6 @@ require 'active_record/connection_adapters/abstract_adapter' require 'active_record/connection_adapters/statement_pool' -require 'arel/visitors/bind_visitor' +require 'active_record/connection_adapters/sqlite3/schema_creation' gem 'sqlite3', '~> 1.3.6' require 'sqlite3' @@ -41,25 +41,6 @@ module ActiveRecord end module ConnectionAdapters #:nodoc: - class SQLite3Binary < Type::Binary # :nodoc: - def cast_value(value) - if value.encoding != Encoding::ASCII_8BIT - value = value.force_encoding(Encoding::ASCII_8BIT) - end - value - end - end - - class SQLite3String < Type::String # :nodoc: - def type_cast_for_database(value) - if value.is_a?(::String) && value.encoding == Encoding::ASCII_8BIT - value.encode(Encoding::UTF_8) - else - super - end - end - end - # The SQLite3 adapter works SQLite 3.6.16 or newer # with the sqlite3-ruby drivers (available as gem from https://rubygems.org/gems/sqlite3). # @@ -97,40 +78,17 @@ module ActiveRecord end class StatementPool < ConnectionAdapters::StatementPool - def initialize(connection, max) - super - @cache = Hash.new { |h,pid| h[pid] = {} } - end - - def each(&block); cache.each(&block); end - def key?(key); cache.key?(key); end - def [](key); cache[key]; end - def length; cache.length; end - - def []=(sql, key) - while @max <= cache.size - dealloc(cache.shift.last[:stmt]) - end - cache[sql] = key - end - - def clear - cache.each_value do |hash| - dealloc hash[:stmt] - end - cache.clear - end - private - def cache - @cache[$$] - end def dealloc(stmt) - stmt.close unless stmt.closed? + stmt[:stmt].close unless stmt[:stmt].closed? end end + def schema_creation # :nodoc: + SQLite3::SchemaCreation.new self + end + def initialize(connection, logger, connection_options, config) super(connection, logger) @@ -140,6 +98,7 @@ module ActiveRecord @config = config @visitor = Arel::Visitors::SQLite.new self + @quoted_column_names = {} if self.class.type_cast_config_to_boolean(config.fetch(:prepared_statements) { true }) @prepared_statements = true @@ -239,6 +198,12 @@ module ActiveRecord case value when BigDecimal value.to_f + when String + if value.encoding == Encoding::ASCII_8BIT + super(value.encode(Encoding::UTF_8)) + else + super + end else super end @@ -253,7 +218,7 @@ module ActiveRecord end def quote_column_name(name) #:nodoc: - %Q("#{name.to_s.gsub('"', '""')}") + @quoted_column_names[name] ||= %Q("#{name.to_s.gsub('"', '""')}") end #-- @@ -280,11 +245,9 @@ module ActiveRecord end def exec_query(sql, name = nil, binds = []) - type_casted_binds = binds.map { |col, val| - [col, type_cast(val, col)] - } + type_casted_binds = binds.map { |attr| type_cast(attr.value_for_database) } - log(sql, name, type_casted_binds) do + log(sql, name, binds) do # Don't cache statements if they are not prepared if without_prepared_statement?(binds) stmt = @connection.prepare(sql) @@ -302,7 +265,7 @@ module ActiveRecord stmt = cache[:stmt] cols = cache[:cols] ||= stmt.columns stmt.reset! - stmt.bind_params type_casted_binds.map { |_, val| val } + stmt.bind_params type_casted_binds end ActiveRecord::Result.new(cols, stmt.to_a) @@ -386,9 +349,10 @@ module ActiveRecord field["dflt_value"] = $1.gsub('""', '"') end + collation = field['collation'] sql_type = field['type'] - cast_type = lookup_cast_type(sql_type) - new_column(field['name'], field['dflt_value'], cast_type, sql_type, field['notnull'].to_i == 0) + type_metadata = fetch_type_metadata(sql_type) + new_column(field['name'], field['dflt_value'], type_metadata, field['notnull'].to_i == 0, nil, collation) end end @@ -464,7 +428,7 @@ module ActiveRecord end end - def change_column_null(table_name, column_name, null, default = nil) + def change_column_null(table_name, column_name, null, default = nil) #:nodoc: unless null || default.nil? exec_query("UPDATE #{quote_table_name(table_name)} SET #{quote_column_name(column_name)}=#{quote(default)} WHERE #{quote_column_name(column_name)} IS NULL") end @@ -483,6 +447,7 @@ module ActiveRecord self.null = options[:null] if options.include?(:null) self.precision = options[:precision] if options.include?(:precision) self.scale = options[:scale] if options.include?(:scale) + self.collation = options[:collation] if options.include?(:collation) end end end @@ -495,16 +460,10 @@ module ActiveRecord protected - def initialize_type_map(m) - super - m.register_type(/binary/i, SQLite3Binary.new) - register_class_with_limit m, %r(char)i, SQLite3String - end - def table_structure(table_name) - structure = exec_query("PRAGMA table_info(#{quote_table_name(table_name)})", 'SCHEMA').to_hash + structure = exec_query("PRAGMA table_info(#{quote_table_name(table_name)})", 'SCHEMA') raise(ActiveRecord::StatementInvalid, "Could not find table '#{table_name}'") if structure.empty? - structure + table_structure_with_collation(table_name, structure) end def alter_table(table_name, options = {}) #:nodoc: @@ -539,7 +498,7 @@ module ActiveRecord @definition.column(column_name, column.type, :limit => column.limit, :default => column.default, :precision => column.precision, :scale => column.scale, - :null => column.null) + :null => column.null, collation: column.collation) end yield @definition if block_given? end @@ -601,6 +560,46 @@ module ActiveRecord super end end + + private + COLLATE_REGEX = /.*\"(\w+)\".*collate\s+\"(\w+)\".*/i.freeze + + def table_structure_with_collation(table_name, basic_structure) + collation_hash = {} + sql = "SELECT sql FROM + (SELECT * FROM sqlite_master UNION ALL + SELECT * FROM sqlite_temp_master) + WHERE type='table' and name='#{ table_name }' \;" + + # Result will have following sample string + # CREATE TABLE "users" ("id" INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, + # "password_digest" varchar COLLATE "NOCASE"); + result = exec_query(sql, 'SCHEMA').first + + if result + # Splitting with left parantheses and picking up last will return all + # columns separated with comma(,). + columns_string = result["sql"].split('(').last + + columns_string.split(',').each do |column_string| + # This regex will match the column name and collation type and will save + # the value in $1 and $2 respectively. + collation_hash[$1] = $2 if (COLLATE_REGEX =~ column_string) + end + + basic_structure.map! do |column| + column_name = column['name'] + + if collation_hash.has_key? column_name + column['collation'] = collation_hash[column_name] + end + + column + end + else + basic_structure.to_hash + end + end end end end diff --git a/activerecord/lib/active_record/connection_adapters/statement_pool.rb b/activerecord/lib/active_record/connection_adapters/statement_pool.rb index c6b1bc8b5b..82e9ef3d3d 100644 --- a/activerecord/lib/active_record/connection_adapters/statement_pool.rb +++ b/activerecord/lib/active_record/connection_adapters/statement_pool.rb @@ -4,35 +4,53 @@ module ActiveRecord include Enumerable def initialize(connection, max = 1000) + @cache = Hash.new { |h,pid| h[pid] = {} } @connection = connection @max = max end - def each - raise NotImplementedError + def each(&block) + cache.each(&block) end def key?(key) - raise NotImplementedError + cache.key?(key) end def [](key) - raise NotImplementedError + cache[key] end def length - raise NotImplementedError + cache.length end - def []=(sql, key) - raise NotImplementedError + def []=(sql, stmt) + while @max <= cache.size + dealloc(cache.shift.last) + end + cache[sql] = stmt end def clear - raise NotImplementedError + cache.each_value do |stmt| + dealloc stmt + end + cache.clear end def delete(key) + dealloc cache[key] + cache.delete(key) + end + + private + + def cache + @cache[Process.pid] + end + + def dealloc(stmt) raise NotImplementedError end end |