diff options
20 files changed, 150 insertions, 105 deletions
diff --git a/activerecord/CHANGELOG.md b/activerecord/CHANGELOG.md index 1470c6dec1..c1b803c7f8 100644 --- a/activerecord/CHANGELOG.md +++ b/activerecord/CHANGELOG.md @@ -1,3 +1,8 @@ +* `find_in_batches` now accepts an `:end_at` parameter that complements the `:start` + parameter to specify where to stop batch processing. + + *Vipul A M* + * Fix rounding problem for PostgreSQL timestamp column. If timestamp column have the precision, it need to format according to @@ -20,7 +25,7 @@ *Rafael Mendonça França* -* Use `SCHEMA` instead of `DB_STRUCTURE` for specifiying structure file. +* Use `SCHEMA` instead of `DB_STRUCTURE` for specifying structure file. This makes the db:structure tasks consistent with test:load_structure. @@ -104,7 +109,7 @@ *Sean Griffin* * Values which would error while being sent to the database (such as an - ASCII-8BIT string with invalid UTF-8 bytes on Sqlite3), no longer error on + ASCII-8BIT string with invalid UTF-8 bytes on SQLite3), no longer error on assignment. They will still error when sent to the database, but you are given the ability to re-assign it to a valid value. diff --git a/activerecord/lib/active_record/associations.rb b/activerecord/lib/active_record/associations.rb index 81a42e22f3..5a3b4f0c40 100644 --- a/activerecord/lib/active_record/associations.rb +++ b/activerecord/lib/active_record/associations.rb @@ -1014,7 +1014,7 @@ module ActiveRecord # record(s) being removed so that callbacks are run. However <tt>delete</tt> and <tt>delete_all</tt> will either # do the deletion according to the strategy specified by the <tt>:dependent</tt> option, or # if no <tt>:dependent</tt> option is given, then it will follow the default strategy. - # The default strategy is <tt>:nullify</tt> (set the foreign keys to <tt>nil</tt>), except for + # The default strategy is to do nothing (leave the foreign keys with the parent ids set), except for # +has_many+ <tt>:through</tt>, where the default strategy is <tt>delete_all</tt> (delete # the join records, without running their callbacks). # diff --git a/activerecord/lib/active_record/attribute_set/builder.rb b/activerecord/lib/active_record/attribute_set/builder.rb index 0f3c285a80..e85777c335 100644 --- a/activerecord/lib/active_record/attribute_set/builder.rb +++ b/activerecord/lib/active_record/attribute_set/builder.rb @@ -20,7 +20,7 @@ module ActiveRecord end class LazyAttributeHash # :nodoc: - delegate :select, :transform_values, :each_key, to: :materialize + delegate :transform_values, :each_key, to: :materialize def initialize(types, values, additional_types) @types = types @@ -50,6 +50,16 @@ module ActiveRecord super end + def select + keys = types.keys | values.keys | delegate_hash.keys + keys.each_with_object({}) do |key, hash| + attribute = self[key] + if yield(key, attribute) + hash[key] = attribute + end + end + end + protected attr_reader :types, :values, :additional_types, :delegate_hash diff --git a/activerecord/lib/active_record/connection_adapters/abstract/quoting.rb b/activerecord/lib/active_record/connection_adapters/abstract/quoting.rb index 55d3360070..29d85d82cb 100644 --- a/activerecord/lib/active_record/connection_adapters/abstract/quoting.rb +++ b/activerecord/lib/active_record/connection_adapters/abstract/quoting.rb @@ -127,7 +127,12 @@ module ActiveRecord end end - value.to_s(:db) + result = value.to_s(:db) + if value.respond_to?(:usec) && value.usec > 0 + "#{result}.#{sprintf("%06d", value.usec)}" + else + result + end end def prepare_binds_for_database(binds) # :nodoc: diff --git a/activerecord/lib/active_record/connection_adapters/abstract/schema_dumper.rb b/activerecord/lib/active_record/connection_adapters/abstract/schema_dumper.rb index 932aaf7aa7..4f6a1ec67b 100644 --- a/activerecord/lib/active_record/connection_adapters/abstract/schema_dumper.rb +++ b/activerecord/lib/active_record/connection_adapters/abstract/schema_dumper.rb @@ -29,7 +29,7 @@ module ActiveRecord limit = column.limit || native_database_types[column.type][:limit] spec[:limit] = limit.inspect if limit - spec[:precision] = column.precision.inspect if column.precision + spec[:precision] = column.precision.inspect if column.precision && column.precision != 0 spec[:scale] = column.scale.inspect if column.scale default = schema_default(column) if column.has_default? diff --git a/activerecord/lib/active_record/connection_adapters/abstract_adapter.rb b/activerecord/lib/active_record/connection_adapters/abstract_adapter.rb index 9392bcb473..ee11c0efa4 100644 --- a/activerecord/lib/active_record/connection_adapters/abstract_adapter.rb +++ b/activerecord/lib/active_record/connection_adapters/abstract_adapter.rb @@ -263,18 +263,6 @@ module ActiveRecord {} end - # QUOTING ================================================== - - # Quote date/time values for use in SQL input. Includes microseconds - # if the value is a Time responding to usec. - def quoted_date(value) #:nodoc: - if value.acts_like?(:time) && value.respond_to?(:usec) - "#{super}.#{sprintf("%06d", value.usec)}" - else - super - end - end - # Returns a bind substitution value given a bind +column+ # NOTE: The column param is currently being used by the sqlserver-adapter def substitute_at(column, _unused = 0) @@ -409,15 +397,15 @@ module ActiveRecord protected def initialize_type_map(m) # :nodoc: - register_class_with_limit m, %r(boolean)i, Type::Boolean - register_class_with_limit m, %r(char)i, Type::String - register_class_with_limit m, %r(binary)i, Type::Binary - register_class_with_limit m, %r(text)i, Type::Text - register_class_with_limit m, %r(date)i, Type::Date - register_class_with_limit m, %r(time)i, Type::Time - register_class_with_limit m, %r(datetime)i, Type::DateTime - register_class_with_limit m, %r(float)i, Type::Float - register_class_with_limit m, %r(int)i, Type::Integer + register_class_with_limit m, %r(boolean)i, Type::Boolean + register_class_with_limit m, %r(char)i, Type::String + register_class_with_limit m, %r(binary)i, Type::Binary + register_class_with_limit m, %r(text)i, Type::Text + register_class_with_precision m, %r(date)i, Type::Date + register_class_with_precision m, %r(time)i, Type::Time + register_class_with_precision m, %r(datetime)i, Type::DateTime + register_class_with_limit m, %r(float)i, Type::Float + register_class_with_limit m, %r(int)i, Type::Integer m.alias_type %r(blob)i, 'binary' m.alias_type %r(clob)i, 'text' @@ -451,6 +439,13 @@ module ActiveRecord end end + def register_class_with_precision(mapping, key, klass) # :nodoc: + mapping.register_type(key) do |*args| + precision = extract_precision(args.last) + klass.new(precision: precision) + end + end + def extract_scale(sql_type) # :nodoc: case sql_type when /\((\d+)\)/ then 0 diff --git a/activerecord/lib/active_record/connection_adapters/abstract_mysql_adapter.rb b/activerecord/lib/active_record/connection_adapters/abstract_mysql_adapter.rb index c29692d6ca..054c59ef5c 100644 --- a/activerecord/lib/active_record/connection_adapters/abstract_mysql_adapter.rb +++ b/activerecord/lib/active_record/connection_adapters/abstract_mysql_adapter.rb @@ -712,11 +712,6 @@ module ActiveRecord m.alias_type %r(year)i, 'integer' m.alias_type %r(bit)i, 'binary' - m.register_type(%r(datetime)i) do |sql_type| - precision = extract_precision(sql_type) - MysqlDateTime.new(precision: precision) - end - m.register_type(%r(enum)i) do |sql_type| limit = sql_type[/^enum\((.+)\)/i, 1] .split(',').map{|enum| enum.strip.length - 2}.max @@ -734,6 +729,14 @@ module ActiveRecord end end + def extract_precision(sql_type) + if /datetime/ === sql_type + super || 0 + else + super + end + end + def fetch_type_metadata(sql_type, collation = "", extra = "") MysqlTypeMetadata.new(super(sql_type), collation: collation, extra: extra, strict: strict_mode?) end @@ -916,14 +919,6 @@ module ActiveRecord TableDefinition.new(native_database_types, name, temporary, options, as) end - class MysqlDateTime < Type::DateTime # :nodoc: - private - - def has_precision? - precision || 0 - end - end - class MysqlString < Type::String # :nodoc: def type_cast_for_database(value) case value @@ -945,7 +940,7 @@ module ActiveRecord end def type_classes_with_standard_constructor - super.merge(string: MysqlString, date_time: MysqlDateTime) + super.merge(string: MysqlString) end end end diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/oid/date_time.rb b/activerecord/lib/active_record/connection_adapters/postgresql/oid/date_time.rb index 2fe61eeb77..b9e7894e5c 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql/oid/date_time.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql/oid/date_time.rb @@ -5,15 +5,6 @@ module ActiveRecord class DateTime < Type::DateTime # :nodoc: include Infinity - def type_cast_for_database(value) - if has_precision? && value.acts_like?(:time) && value.year <= 0 - bce_year = format("%04d", -value.year + 1) - super.sub(/^-?\d+/, bce_year) + " BC" - else - super - end - end - def cast_value(value) if value.is_a?(::String) case value diff --git a/activerecord/lib/active_record/relation/batches.rb b/activerecord/lib/active_record/relation/batches.rb index 9d690af11d..a543341149 100644 --- a/activerecord/lib/active_record/relation/batches.rb +++ b/activerecord/lib/active_record/relation/batches.rb @@ -27,11 +27,12 @@ module ActiveRecord # # ==== Options # * <tt>:batch_size</tt> - Specifies the size of the batch. Default to 1000. - # * <tt>:start</tt> - Specifies the primary key value to start from. + # * <tt>:start</tt> - Specifies the primary key value to start from, inclusive of the value. + # * <tt>:end_at</tt> - Specifies the primary key value to end at, inclusive of the value. # This is especially useful if you want multiple workers dealing with # the same processing queue. You can make worker 1 handle all the records # between id 0 and 10,000 and worker 2 handle from 10,000 and beyond - # (by setting the +:start+ option on that worker). + # (by setting the +:start+ and +:end_at+ option on each worker). # # # Let's process for a batch of 2000 records, skipping the first 2000 rows # Person.find_each(start: 2000, batch_size: 2000) do |person| @@ -45,14 +46,15 @@ module ActiveRecord # # NOTE: You can't set the limit either, that's used to control # the batch sizes. - def find_each(start: nil, batch_size: 1000) + def find_each(start: nil, end_at: nil, batch_size: 1000) if block_given? - find_in_batches(start: start, batch_size: batch_size) do |records| + find_in_batches(start: start, end_at: end_at, batch_size: batch_size) do |records| records.each { |record| yield record } end else - enum_for(:find_each, start: start, batch_size: batch_size) do - start ? where(table[primary_key].gteq(start)).size : size + enum_for(:find_each, start: start, end_at: end_at, batch_size: batch_size) do + relation = self + apply_limits(relation, start, end_at).size end end end @@ -77,11 +79,12 @@ module ActiveRecord # # ==== Options # * <tt>:batch_size</tt> - Specifies the size of the batch. Default to 1000. - # * <tt>:start</tt> - Specifies the primary key value to start from. + # * <tt>:start</tt> - Specifies the primary key value to start from, inclusive of the value. + # * <tt>:end_at</tt> - Specifies the primary key value to end at, inclusive of the value. # This is especially useful if you want multiple workers dealing with # the same processing queue. You can make worker 1 handle all the records # between id 0 and 10,000 and worker 2 handle from 10,000 and beyond - # (by setting the +:start+ option on that worker). + # (by setting the +:start+ and +:end_at+ option on each worker). # # # Let's process the next 2000 records # Person.find_in_batches(start: 2000, batch_size: 2000) do |group| @@ -95,12 +98,12 @@ module ActiveRecord # # NOTE: You can't set the limit either, that's used to control # the batch sizes. - def find_in_batches(start: nil, batch_size: 1000) + def find_in_batches(start: nil, end_at: nil, batch_size: 1000) relation = self unless block_given? - return to_enum(:find_in_batches, start: start, batch_size: batch_size) do - total = start ? where(table[primary_key].gteq(start)).size : size + return to_enum(:find_in_batches, start: start, end_at: end_at, batch_size: batch_size) do + total = apply_limits(relation, start, end_at).size (total - 1).div(batch_size) + 1 end end @@ -110,7 +113,8 @@ module ActiveRecord end relation = relation.reorder(batch_order).limit(batch_size) - records = start ? relation.where(table[primary_key].gteq(start)).to_a : relation.to_a + relation = apply_limits(relation, start, end_at) + records = relation.to_a while records.any? records_size = records.size @@ -127,6 +131,12 @@ module ActiveRecord private + def apply_limits(relation, start, end_at) + relation = relation.where(table[primary_key].gteq(start)) if start + relation = relation.where(table[primary_key].lteq(end_at)) if end_at + relation + end + def batch_order "#{quoted_table_name}.#{quoted_primary_key} ASC" end diff --git a/activerecord/lib/active_record/type/date_time.rb b/activerecord/lib/active_record/type/date_time.rb index e8614b16e0..a25f2521bb 100644 --- a/activerecord/lib/active_record/type/date_time.rb +++ b/activerecord/lib/active_record/type/date_time.rb @@ -11,28 +11,25 @@ module ActiveRecord end def type_cast_for_database(value) - return super unless value.acts_like?(:time) - - zone_conversion_method = ActiveRecord::Base.default_timezone == :utc ? :getutc : :getlocal - - if value.respond_to?(zone_conversion_method) - value = value.send(zone_conversion_method) + if precision && value.respond_to?(:usec) + number_of_insignificant_digits = 6 - precision + round_power = 10 ** number_of_insignificant_digits + value = value.change(usec: value.usec / round_power * round_power) end - return value unless has_precision? + if value.acts_like?(:time) + zone_conversion_method = ActiveRecord::Base.default_timezone == :utc ? :getutc : :getlocal - result = value.to_s(:db) - if value.respond_to?(:usec) && (1..6).cover?(precision) - "#{result}.#{sprintf("%0#{precision}d", value.usec / 10 ** (6 - precision))}" - else - result + if value.respond_to?(zone_conversion_method) + value = value.send(zone_conversion_method) + end end + + value end private - alias has_precision? precision - def cast_value(string) return string unless string.is_a?(::String) return if string.empty? diff --git a/activerecord/test/cases/adapters/mysql2/connection_test.rb b/activerecord/test/cases/adapters/mysql2/connection_test.rb index d261e2db55..ff8ce93248 100644 --- a/activerecord/test/cases/adapters/mysql2/connection_test.rb +++ b/activerecord/test/cases/adapters/mysql2/connection_test.rb @@ -122,11 +122,4 @@ class MysqlConnectionTest < ActiveRecord::TestCase ensure @connection.execute "DROP TABLE `bar_baz`" end - - if mysql_56? - def test_quote_time_usec - assert_equal "'1970-01-01 00:00:00.000000'", @connection.quote(Time.at(0)) - assert_equal "'1970-01-01 00:00:00.000000'", @connection.quote(Time.at(0).to_datetime) - end - end end diff --git a/activerecord/test/cases/adapters/mysql2/datetime_test.rb b/activerecord/test/cases/adapters/mysql2/datetime_test.rb index ae00f4e131..7a37247d6a 100644 --- a/activerecord/test/cases/adapters/mysql2/datetime_test.rb +++ b/activerecord/test/cases/adapters/mysql2/datetime_test.rb @@ -4,15 +4,12 @@ if mysql_56? class DateTimeTest < ActiveRecord::TestCase self.use_transactional_fixtures = false - class Foo < ActiveRecord::Base; end - - def test_default_datetime_precision - ActiveRecord::Base.connection.create_table(:foos, force: true) - ActiveRecord::Base.connection.add_column :foos, :created_at, :datetime - ActiveRecord::Base.connection.add_column :foos, :updated_at, :datetime - assert_nil activerecord_column_option('foos', 'created_at', 'precision') + teardown do + ActiveRecord::Base.connection.drop_table(:foos, if_exists: true) end + class Foo < ActiveRecord::Base; end + def test_datetime_data_type_with_precision ActiveRecord::Base.connection.create_table(:foos, force: true) ActiveRecord::Base.connection.add_column :foos, :created_at, :datetime, precision: 1 diff --git a/activerecord/test/cases/adapters/postgresql/quoting_test.rb b/activerecord/test/cases/adapters/postgresql/quoting_test.rb index 894cf1ffa2..60baacf67a 100644 --- a/activerecord/test/cases/adapters/postgresql/quoting_test.rb +++ b/activerecord/test/cases/adapters/postgresql/quoting_test.rb @@ -27,11 +27,6 @@ module ActiveRecord assert_equal "'Infinity'", @conn.quote(infinity) end - def test_quote_time_usec - assert_equal "'1970-01-01 00:00:00.000000'", @conn.quote(Time.at(0)) - assert_equal "'1970-01-01 00:00:00.000000'", @conn.quote(Time.at(0).to_datetime) - end - def test_quote_range range = "1,2]'; SELECT * FROM users; --".."a" type = OID::Range.new(Type::Integer.new, :int8range) diff --git a/activerecord/test/cases/adapters/postgresql/timestamp_test.rb b/activerecord/test/cases/adapters/postgresql/timestamp_test.rb index 8246b14b93..9e631fb4ca 100644 --- a/activerecord/test/cases/adapters/postgresql/timestamp_test.rb +++ b/activerecord/test/cases/adapters/postgresql/timestamp_test.rb @@ -145,10 +145,13 @@ class TimestampTest < ActiveRecord::TestCase date = ::Time.utc(2014, 8, 17, 12, 30, 0, 999999) Foo.create!(created_at: date, updated_at: date) assert foo = Foo.find_by(created_at: date) + assert_equal 1, Foo.where(updated_at: date).count assert_equal date.to_s, foo.created_at.to_s assert_equal date.to_s, foo.updated_at.to_s assert_equal 000000, foo.created_at.usec assert_equal 999900, foo.updated_at.usec + ensure + ActiveRecord::Base.connection.drop_table(:foos, if_exists: true) end private diff --git a/activerecord/test/cases/attribute_set_test.rb b/activerecord/test/cases/attribute_set_test.rb index 8025c7c4d2..112cd2fb14 100644 --- a/activerecord/test/cases/attribute_set_test.rb +++ b/activerecord/test/cases/attribute_set_test.rb @@ -65,6 +65,16 @@ module ActiveRecord assert_equal({ foo: 1, bar: 2.2 }, attributes.to_h) end + test "to_hash maintains order" do + builder = AttributeSet::Builder.new(foo: Type::Integer.new, bar: Type::Float.new) + attributes = builder.build_from_database(foo: '2.2', bar: '3.3') + + attributes[:bar] + hash = attributes.to_h + + assert_equal [[:foo, 2], [:bar, 3.3]], hash.to_a + end + test "values_before_type_cast" do builder = AttributeSet::Builder.new(foo: Type::Integer.new, bar: Type::Integer.new) attributes = builder.build_from_database(foo: '1.1', bar: '2.2') diff --git a/activerecord/test/cases/batches_test.rb b/activerecord/test/cases/batches_test.rb index c12fa03015..c05382598b 100644 --- a/activerecord/test/cases/batches_test.rb +++ b/activerecord/test/cases/batches_test.rb @@ -106,6 +106,15 @@ class EachTest < ActiveRecord::TestCase end end + def test_find_in_batches_should_end_at_the_end_option + assert_queries(6) do + Post.find_in_batches(batch_size: 1, end_at: 5) do |batch| + assert_kind_of Array, batch + assert_kind_of Post, batch.first + end + end + end + def test_find_in_batches_shouldnt_execute_query_unless_needed assert_queries(2) do Post.find_in_batches(:batch_size => @total) {|batch| assert_kind_of Array, batch } diff --git a/activerecord/test/cases/quoting_test.rb b/activerecord/test/cases/quoting_test.rb index ad09340518..6d91f96bf6 100644 --- a/activerecord/test/cases/quoting_test.rb +++ b/activerecord/test/cases/quoting_test.rb @@ -46,28 +46,28 @@ module ActiveRecord def test_quoted_time_utc with_timezone_config default: :utc do - t = Time.now + t = Time.now.change(usec: 0) assert_equal t.getutc.to_s(:db), @quoter.quoted_date(t) end end def test_quoted_time_local with_timezone_config default: :local do - t = Time.now + t = Time.now.change(usec: 0) assert_equal t.getlocal.to_s(:db), @quoter.quoted_date(t) end end def test_quoted_time_crazy with_timezone_config default: :asdfasdf do - t = Time.now + t = Time.now.change(usec: 0) assert_equal t.getlocal.to_s(:db), @quoter.quoted_date(t) end end def test_quoted_datetime_utc with_timezone_config default: :utc do - t = DateTime.now + t = Time.now.change(usec: 0).to_datetime assert_equal t.getutc.to_s(:db), @quoter.quoted_date(t) end end @@ -76,7 +76,7 @@ module ActiveRecord # DateTime doesn't define getlocal, so make sure it does nothing def test_quoted_datetime_local with_timezone_config default: :local do - t = DateTime.now + t = Time.now.change(usec: 0).to_datetime assert_equal t.to_s(:db), @quoter.quoted_date(t) end end diff --git a/guides/source/active_record_querying.md b/guides/source/active_record_querying.md index 373a98bb85..c5ca848753 100644 --- a/guides/source/active_record_querying.md +++ b/guides/source/active_record_querying.md @@ -343,6 +343,19 @@ end Another example would be if you wanted multiple workers handling the same processing queue. You could have each worker handle 10000 records by setting the appropriate `:start` option on each worker. +**`:end_at`** + +Similar to the `:start` option, `:end_at` allows you to configure the last ID of the sequence whenever the highest ID is not the one you need. +This would be useful, for example, if you wanted to run a batch process, using a subset of records based on `:start` and `:end_at` + +For example, to send newsletters only to users with the primary key starting from 2000 upto 10000 and to retrieve them in batches of 1000: + +```ruby +User.find_each(start: 2000, end_at: 10000, batch_size: 5000) do |user| + NewsMailer.weekly(user).deliver_now +end +``` + #### `find_in_batches` The `find_in_batches` method is similar to `find_each`, since both retrieve batches of records. The difference is that `find_in_batches` yields _batches_ to the block as an array of models, instead of individually. The following example will yield to the supplied block an array of up to 1000 invoices at a time, with the final block containing any remaining invoices: @@ -356,7 +369,7 @@ end ##### Options for `find_in_batches` -The `find_in_batches` method accepts the same `:batch_size` and `:start` options as `find_each`. +The `find_in_batches` method accepts the same `:batch_size`, `:start` and `:end_at` options as `find_each`. Conditions ---------- diff --git a/guides/source/i18n.md b/guides/source/i18n.md index fbee267975..8f24c53edb 100644 --- a/guides/source/i18n.md +++ b/guides/source/i18n.md @@ -588,6 +588,25 @@ you can look up the `books.index.title` value **inside** `app/views/books/index. NOTE: Automatic translation scoping by partial is only available from the `translate` view helper method. +"Lazy" lookup can also be used in _controllers_: + +```yaml +en: + books: + create: + success: Book created! +``` +which is especially useful for setting flash messages: + +```ruby +class BooksController < ApplicationController + def create + # ... + redirect_to books_url, notice: t('.success') + end +end +``` + ### Interpolation In many cases you want to abstract your translations so that **variables can be interpolated into the translation**. For this reason the I18n API provides an interpolation feature. diff --git a/railties/test/generators/plugin_generator_test.rb b/railties/test/generators/plugin_generator_test.rb index 9c49766a2f..a0f244da28 100644 --- a/railties/test/generators/plugin_generator_test.rb +++ b/railties/test/generators/plugin_generator_test.rb @@ -192,13 +192,11 @@ class PluginGeneratorTest < Rails::Generators::TestCase def test_generation_runs_bundle_install_with_full_and_mountable result = run_generator [destination_root, "--mountable", "--full", "--dev"] + assert_match(/run bundle install/, result) + assert $?.success?, "Command failed: #{result}" assert_file "#{destination_root}/Gemfile.lock" do |contents| assert_match(/bukkits/, contents) end - assert_match(/run bundle install/, result) - assert_match(/Using bukkits \(?0\.0\.1\)?/, result) - assert_match(/Your bundle is complete/, result) - assert_equal 1, result.scan("Your bundle is complete").size end def test_skipping_javascripts_without_mountable_option |