From 213796fb4936dce1da2f0c097a054e1af5c25c2c Mon Sep 17 00:00:00 2001 From: Sean Griffin Date: Mon, 24 Jul 2017 08:19:35 -0400 Subject: Refactor Active Record to let Arel manage bind params A common source of bugs and code bloat within Active Record has been the need for us to maintain the list of bind values separately from the AST they're associated with. This makes any sort of AST manipulation incredibly difficult, as any time we want to potentially insert or remove an AST node, we need to traverse the entire tree to find where the associated bind parameters are. With this change, the bind parameters now live on the AST directly. Active Record does not need to know or care about them until the final AST traversal for SQL construction. Rather than returning just the SQL, the Arel collector will now return both the SQL and the bind parameters. At this point the connection adapter will have all the values that it had before. A bit of this code is janky and something I'd like to refactor later. In particular, I don't like how we're handling associations in the predicate builder, the special casing of `StatementCache::Substitute` in `QueryAttribute`, or generally how we're handling bind value replacement in the statement cache when prepared statements are disabled. This also mostly reverts #26378, as it moved all the code into a location that I wanted to delete. /cc @metaskills @yahonda, this change will affect the adapters Fixes #29766. Fixes #29804. Fixes #26541. Close #28539. Close #24769. Close #26468. Close #26202. There are probably other issues/PRs that can be closed because of this commit, but that's all I could find on the first few pages. --- .../lib/active_record/relation/calculations.rb | 6 +- .../lib/active_record/relation/finder_methods.rb | 6 +- .../lib/active_record/relation/from_clause.rb | 8 -- .../active_record/relation/predicate_builder.rb | 84 +++----------------- .../relation/predicate_builder/array_handler.rb | 8 +- .../relation/predicate_builder/base_handler.rb | 2 - .../predicate_builder/basic_object_handler.rb | 11 ++- .../relation/predicate_builder/range_handler.rb | 26 ++++-- .../lib/active_record/relation/query_attribute.rb | 5 ++ .../lib/active_record/relation/query_methods.rb | 44 ++++------- .../lib/active_record/relation/where_clause.rb | 92 ++++++++-------------- .../active_record/relation/where_clause_factory.rb | 48 +---------- 12 files changed, 109 insertions(+), 231 deletions(-) (limited to 'activerecord/lib/active_record/relation') diff --git a/activerecord/lib/active_record/relation/calculations.rb b/activerecord/lib/active_record/relation/calculations.rb index c105abe774..66a9a3c9c9 100644 --- a/activerecord/lib/active_record/relation/calculations.rb +++ b/activerecord/lib/active_record/relation/calculations.rb @@ -186,7 +186,7 @@ module ActiveRecord relation.select_values = column_names.map { |cn| @klass.has_attribute?(cn) || @klass.attribute_alias?(cn) ? arel_attribute(cn) : cn } - result = skip_query_cache_if_necessary { klass.connection.select_all(relation.arel, nil, bound_attributes) } + result = skip_query_cache_if_necessary { klass.connection.select_all(relation.arel, nil) } result.cast_values(klass.attribute_types) end end @@ -262,7 +262,7 @@ module ActiveRecord query_builder = relation.arel end - result = skip_query_cache_if_necessary { @klass.connection.select_all(query_builder, nil, bound_attributes) } + result = skip_query_cache_if_necessary { @klass.connection.select_all(query_builder, nil) } row = result.first value = row && row.values.first type = result.column_types.fetch(column_alias) do @@ -313,7 +313,7 @@ module ActiveRecord relation.group_values = group_fields relation.select_values = select_values - calculated_data = skip_query_cache_if_necessary { @klass.connection.select_all(relation.arel, nil, relation.bound_attributes) } + calculated_data = skip_query_cache_if_necessary { @klass.connection.select_all(relation.arel, nil) } if association key_ids = calculated_data.collect { |row| row[group_aliases.first] } diff --git a/activerecord/lib/active_record/relation/finder_methods.rb b/activerecord/lib/active_record/relation/finder_methods.rb index 36d35b98fd..626c50470e 100644 --- a/activerecord/lib/active_record/relation/finder_methods.rb +++ b/activerecord/lib/active_record/relation/finder_methods.rb @@ -317,7 +317,7 @@ module ActiveRecord relation = construct_relation_for_exists(relation, conditions) - skip_query_cache_if_necessary { connection.select_value(relation.arel, "#{name} Exists", relation.bound_attributes) } ? true : false + skip_query_cache_if_necessary { connection.select_value(relation.arel, "#{name} Exists") } ? true : false rescue ::RangeError false end @@ -378,7 +378,7 @@ module ActiveRecord if ActiveRecord::NullRelation === relation [] else - rows = skip_query_cache_if_necessary { connection.select_all(relation.arel, "SQL", relation.bound_attributes) } + rows = skip_query_cache_if_necessary { connection.select_all(relation.arel, "SQL") } join_dependency.instantiate(rows, aliases) end end @@ -426,7 +426,7 @@ module ActiveRecord relation = relation.except(:select).select(values).distinct! - id_rows = skip_query_cache_if_necessary { @klass.connection.select_all(relation.arel, "SQL", relation.bound_attributes) } + id_rows = skip_query_cache_if_necessary { @klass.connection.select_all(relation.arel, "SQL") } id_rows.map { |row| row[primary_key] } end diff --git a/activerecord/lib/active_record/relation/from_clause.rb b/activerecord/lib/active_record/relation/from_clause.rb index 03f1202470..c53a682aee 100644 --- a/activerecord/lib/active_record/relation/from_clause.rb +++ b/activerecord/lib/active_record/relation/from_clause.rb @@ -10,14 +10,6 @@ module ActiveRecord @name = name end - def binds - if value.is_a?(Relation) - value.bound_attributes - else - [] - end - end - def merge(other) self end diff --git a/activerecord/lib/active_record/relation/predicate_builder.rb b/activerecord/lib/active_record/relation/predicate_builder.rb index 3b8f8da634..ee7fef4711 100644 --- a/activerecord/lib/active_record/relation/predicate_builder.rb +++ b/activerecord/lib/active_record/relation/predicate_builder.rb @@ -8,10 +8,9 @@ module ActiveRecord @table = table @handlers = [] - register_handler(BasicObject, BasicObjectHandler.new) + register_handler(BasicObject, BasicObjectHandler.new(self)) register_handler(Base, BaseHandler.new(self)) - register_handler(Range, RangeHandler.new) - register_handler(RangeHandler::RangeWithBinds, RangeHandler.new) + register_handler(Range, RangeHandler.new(self)) register_handler(Relation, RelationHandler.new) register_handler(Array, ArrayHandler.new(self)) end @@ -21,11 +20,6 @@ module ActiveRecord expand_from_hash(attributes) end - def create_binds(attributes) - attributes = convert_dot_notation_to_hash(attributes) - create_binds_for_hash(attributes) - end - def self.references(attributes) attributes.map do |key, value| if value.is_a?(Hash) @@ -56,8 +50,11 @@ module ActiveRecord handler_for(value).call(attribute, value) end - # TODO Change this to private once we've dropped Ruby 2.2 support. - # Workaround for Ruby 2.2 "private attribute?" warning. + def build_bind_attribute(column_name, value) + attr = Relation::QueryAttribute.new(column_name.to_s, value, table.type(column_name)) + Arel::Nodes::BindParam.new(attr) + end + protected attr_reader :table @@ -68,29 +65,13 @@ module ActiveRecord attributes.flat_map do |key, value| if value.is_a?(Hash) && !table.has_column?(key) associated_predicate_builder(key).expand_from_hash(value) - else - build(table.arel_attribute(key), value) - end - end - end - - def create_binds_for_hash(attributes) - result = attributes.dup - binds = [] - - attributes.each do |column_name, value| - case - when value.is_a?(Hash) && !table.has_column?(column_name) - attrs, bvs = associated_predicate_builder(column_name).create_binds_for_hash(value) - result[column_name] = attrs - binds += bvs - when table.associated_with?(column_name) + elsif table.associated_with?(key) # Find the foreign key when using queries such as: # Post.where(author: author) # # For polymorphic relationships, find the foreign key and type: # PriceEstimate.where(estimate_of: treasure) - associated_table = table.associated_table(column_name) + associated_table = table.associated_table(key) if associated_table.polymorphic_association? case value.is_a?(Array) ? value.first : value when Base, Relation @@ -100,40 +81,14 @@ module ActiveRecord end klass ||= AssociationQueryValue - result[column_name] = klass.new(associated_table, value).queries.map do |query| - attrs, bvs = create_binds_for_hash(query) - binds.concat(bvs) - attrs - end - when value.is_a?(Range) && !table.type(column_name).respond_to?(:subtype) - first = value.begin - last = value.end - unless first.respond_to?(:infinite?) && first.infinite? - binds << build_bind_attribute(column_name, first) - first = Arel::Nodes::BindParam.new + queries = klass.new(associated_table, value).queries.map do |query| + expand_from_hash(query).reduce(&:and) end - unless last.respond_to?(:infinite?) && last.infinite? - binds << build_bind_attribute(column_name, last) - last = Arel::Nodes::BindParam.new - end - - result[column_name] = RangeHandler::RangeWithBinds.new(first, last, value.exclude_end?) - when value.is_a?(Relation) - binds.concat(value.bound_attributes) + queries.reduce(&:or) else - if can_be_bound?(column_name, value) - bind_attribute = build_bind_attribute(column_name, value) - if value.is_a?(StatementCache::Substitute) || !bind_attribute.value_for_database.nil? - result[column_name] = Arel::Nodes::BindParam.new - binds << bind_attribute - else - result[column_name] = nil - end - end + build(table.arel_attribute(key), value) end end - - [result, binds] end private @@ -161,19 +116,6 @@ module ActiveRecord def handler_for(object) @handlers.detect { |klass, _| klass === object }.last end - - def can_be_bound?(column_name, value) - case value - when Array, Range - table.type(column_name).respond_to?(:subtype) - else - !value.nil? && handler_for(value).is_a?(BasicObjectHandler) - end - end - - def build_bind_attribute(column_name, value) - Relation::QueryAttribute.new(column_name.to_s, value, table.type(column_name)) - end end end diff --git a/activerecord/lib/active_record/relation/predicate_builder/array_handler.rb b/activerecord/lib/active_record/relation/predicate_builder/array_handler.rb index 85ec7d1428..ad617365fc 100644 --- a/activerecord/lib/active_record/relation/predicate_builder/array_handler.rb +++ b/activerecord/lib/active_record/relation/predicate_builder/array_handler.rb @@ -19,7 +19,11 @@ module ActiveRecord case values.length when 0 then NullPredicate when 1 then predicate_builder.build(attribute, values.first) - else attribute.in(values) + else + bind_values = values.map do |v| + predicate_builder.build_bind_attribute(attribute.name, v) + end + attribute.in(bind_values) end unless nils.empty? @@ -31,8 +35,6 @@ module ActiveRecord array_predicates.inject(&:or) end - # TODO Change this to private once we've dropped Ruby 2.2 support. - # Workaround for Ruby 2.2 "private attribute?" warning. protected attr_reader :predicate_builder diff --git a/activerecord/lib/active_record/relation/predicate_builder/base_handler.rb b/activerecord/lib/active_record/relation/predicate_builder/base_handler.rb index 632c68a103..112821135f 100644 --- a/activerecord/lib/active_record/relation/predicate_builder/base_handler.rb +++ b/activerecord/lib/active_record/relation/predicate_builder/base_handler.rb @@ -11,8 +11,6 @@ module ActiveRecord predicate_builder.build(attribute, value.id) end - # TODO Change this to private once we've dropped Ruby 2.2 support. - # Workaround for Ruby 2.2 "private attribute?" warning. protected attr_reader :predicate_builder diff --git a/activerecord/lib/active_record/relation/predicate_builder/basic_object_handler.rb b/activerecord/lib/active_record/relation/predicate_builder/basic_object_handler.rb index a17cf30ffd..34db266f05 100644 --- a/activerecord/lib/active_record/relation/predicate_builder/basic_object_handler.rb +++ b/activerecord/lib/active_record/relation/predicate_builder/basic_object_handler.rb @@ -3,9 +3,18 @@ module ActiveRecord class PredicateBuilder class BasicObjectHandler # :nodoc: + def initialize(predicate_builder) + @predicate_builder = predicate_builder + end + def call(attribute, value) - attribute.eq(value) + bind = predicate_builder.build_bind_attribute(attribute.name, value) + attribute.eq(bind) end + + protected + + attr_reader :predicate_builder end end end diff --git a/activerecord/lib/active_record/relation/predicate_builder/range_handler.rb b/activerecord/lib/active_record/relation/predicate_builder/range_handler.rb index 072e238306..6d16579708 100644 --- a/activerecord/lib/active_record/relation/predicate_builder/range_handler.rb +++ b/activerecord/lib/active_record/relation/predicate_builder/range_handler.rb @@ -3,25 +3,39 @@ module ActiveRecord class PredicateBuilder class RangeHandler # :nodoc: - RangeWithBinds = Struct.new(:begin, :end, :exclude_end?) + class RangeWithBinds < Struct.new(:begin, :end) + def exclude_end? + false + end + end + + def initialize(predicate_builder) + @predicate_builder = predicate_builder + end def call(attribute, value) + begin_bind = predicate_builder.build_bind_attribute(attribute.name, value.begin) + end_bind = predicate_builder.build_bind_attribute(attribute.name, value.end) if value.begin.respond_to?(:infinite?) && value.begin.infinite? if value.end.respond_to?(:infinite?) && value.end.infinite? attribute.not_in([]) elsif value.exclude_end? - attribute.lt(value.end) + attribute.lt(end_bind) else - attribute.lteq(value.end) + attribute.lteq(end_bind) end elsif value.end.respond_to?(:infinite?) && value.end.infinite? - attribute.gteq(value.begin) + attribute.gteq(begin_bind) elsif value.exclude_end? - attribute.gteq(value.begin).and(attribute.lt(value.end)) + attribute.gteq(begin_bind).and(attribute.lt(end_bind)) else - attribute.between(value) + attribute.between(RangeWithBinds.new(begin_bind, end_bind)) end end + + protected + + attr_reader :predicate_builder end end end diff --git a/activerecord/lib/active_record/relation/query_attribute.rb b/activerecord/lib/active_record/relation/query_attribute.rb index e6883acd90..5a9a7fd432 100644 --- a/activerecord/lib/active_record/relation/query_attribute.rb +++ b/activerecord/lib/active_record/relation/query_attribute.rb @@ -16,6 +16,11 @@ module ActiveRecord def with_cast_value(value) QueryAttribute.new(name, value, type) end + + def nil? + !value_before_type_cast.is_a?(StatementCache::Substitute) && + (value_before_type_cast.nil? || value_for_database.nil?) + end end end end diff --git a/activerecord/lib/active_record/relation/query_methods.rb b/activerecord/lib/active_record/relation/query_methods.rb index e2af62873c..812c8e7e3f 100644 --- a/activerecord/lib/active_record/relation/query_methods.rb +++ b/activerecord/lib/active_record/relation/query_methods.rb @@ -76,31 +76,6 @@ module ActiveRecord CODE end - def bound_attributes - if limit_value - limit_bind = Attribute.with_cast_value( - "LIMIT".freeze, - connection.sanitize_limit(limit_value), - Type.default_value, - ) - end - if offset_value - offset_bind = Attribute.with_cast_value( - "OFFSET".freeze, - offset_value.to_i, - Type.default_value, - ) - end - connection.combine_bind_parameters( - from_clause: from_clause.binds, - join_clause: arel.bind_values, - where_clause: where_clause.binds, - having_clause: having_clause.binds, - limit: limit_bind, - offset: offset_bind, - ) - end - alias extensions extending_values # Specify relationships to be included in the result set. For @@ -952,8 +927,22 @@ module ActiveRecord arel.where(where_clause.ast) unless where_clause.empty? arel.having(having_clause.ast) unless having_clause.empty? - arel.take(Arel::Nodes::BindParam.new) if limit_value - arel.skip(Arel::Nodes::BindParam.new) if offset_value + if limit_value + limit_attribute = Attribute.with_cast_value( + "LIMIT".freeze, + connection.sanitize_limit(limit_value), + Type.default_value, + ) + arel.take(Arel::Nodes::BindParam.new(limit_attribute)) + end + if offset_value + offset_attribute = Attribute.with_cast_value( + "OFFSET".freeze, + offset_value.to_i, + Type.default_value, + ) + arel.skip(Arel::Nodes::BindParam.new(offset_attribute)) + end arel.group(*arel_columns(group_values.uniq.reject(&:blank?))) unless group_values.empty? build_order(arel) @@ -1029,7 +1018,6 @@ module ActiveRecord join_infos.each do |info| info.joins.each { |join| manager.from(join) } - manager.bind_values.concat info.binds end manager.join_sources.concat(join_list) diff --git a/activerecord/lib/active_record/relation/where_clause.rb b/activerecord/lib/active_record/relation/where_clause.rb index 0b4c7b7dfa..ef2bca9155 100644 --- a/activerecord/lib/active_record/relation/where_clause.rb +++ b/activerecord/lib/active_record/relation/where_clause.rb @@ -3,31 +3,26 @@ module ActiveRecord class Relation class WhereClause # :nodoc: - attr_reader :binds - delegate :any?, :empty?, to: :predicates - def initialize(predicates, binds) + def initialize(predicates) @predicates = predicates - @binds = binds end def +(other) WhereClause.new( predicates + other.predicates, - binds + other.binds, ) end def merge(other) WhereClause.new( predicates_unreferenced_by(other) + other.predicates, - non_conflicting_binds(other) + other.binds, ) end def except(*columns) - WhereClause.new(*except_predicates_and_binds(columns)) + WhereClause.new(except_predicates(columns)) end def or(other) @@ -38,7 +33,6 @@ module ActiveRecord else WhereClause.new( [ast.or(other.ast)], - binds + other.binds ) end end @@ -51,17 +45,10 @@ module ActiveRecord end end - binds = self.binds.map { |attr| [attr.name, attr.value] }.to_h - equalities.map { |node| name = node.left.name.to_s - [name, binds.fetch(name) { - case node.right - when Array then node.right.map(&:val) - when Arel::Nodes::Casted, Arel::Nodes::Quoted - node.right.val - end - }] + value = extract_node_value(node.right) + [name, value] }.to_h end @@ -71,20 +58,17 @@ module ActiveRecord def ==(other) other.is_a?(WhereClause) && - predicates == other.predicates && - binds == other.binds + predicates == other.predicates end def invert - WhereClause.new(inverted_predicates, binds) + WhereClause.new(inverted_predicates) end def self.empty - @empty ||= new([], []) + @empty ||= new([]) end - # TODO Change this to private once we've dropped Ruby 2.2 support. - # Workaround for Ruby 2.2 "private attribute?" warning. protected attr_reader :predicates @@ -108,12 +92,6 @@ module ActiveRecord node.respond_to?(:operator) && node.operator == :== end - def non_conflicting_binds(other) - conflicts = referenced_columns & other.referenced_columns - conflicts.map! { |node| node.name.to_s } - binds.reject { |attr| conflicts.include?(attr.name) } - end - def inverted_predicates predicates.map { |node| invert_predicate(node) } end @@ -133,44 +111,22 @@ module ActiveRecord end end - def except_predicates_and_binds(columns) - except_binds = [] - binds_index = 0 - - predicates = self.predicates.reject do |node| - binds_contains = node.grep(Arel::Nodes::BindParam).size if node.is_a?(Arel::Nodes::Node) - - except = \ - case node - when Arel::Nodes::Between, Arel::Nodes::In, Arel::Nodes::NotIn, Arel::Nodes::Equality, Arel::Nodes::NotEqual, Arel::Nodes::LessThan, Arel::Nodes::LessThanOrEqual, Arel::Nodes::GreaterThan, Arel::Nodes::GreaterThanOrEqual - subrelation = (node.left.kind_of?(Arel::Attributes::Attribute) ? node.left : node.right) - columns.include?(subrelation.name.to_s) - end - - if except && binds_contains > 0 - (binds_index...(binds_index + binds_contains)).each do |i| - except_binds[i] = true - end + def except_predicates(columns) + self.predicates.reject do |node| + case node + when Arel::Nodes::Between, Arel::Nodes::In, Arel::Nodes::NotIn, Arel::Nodes::Equality, Arel::Nodes::NotEqual, Arel::Nodes::LessThan, Arel::Nodes::LessThanOrEqual, Arel::Nodes::GreaterThan, Arel::Nodes::GreaterThanOrEqual + subrelation = (node.left.kind_of?(Arel::Attributes::Attribute) ? node.left : node.right) + columns.include?(subrelation.name.to_s) end - - binds_index += binds_contains if binds_contains - - except end - - binds = self.binds.reject.with_index do |_, i| - except_binds[i] - end - - [predicates, binds] end def predicates_with_wrapped_sql_literals non_empty_predicates.map do |node| - if Arel::Nodes::Equality === node - node - else + case node + when Arel::Nodes::SqlLiteral, ::String wrap_sql_literal(node) + else node end end end @@ -186,6 +142,22 @@ module ActiveRecord end Arel::Nodes::Grouping.new(node) end + + def extract_node_value(node) + case node + when Array + node.map { |v| extract_node_value(v) } + when Arel::Nodes::Casted, Arel::Nodes::Quoted + node.val + when Arel::Nodes::BindParam + value = node.value + if value.respond_to?(:value_before_type_cast) + value.value_before_type_cast + else + value + end + end + end end end end diff --git a/activerecord/lib/active_record/relation/where_clause_factory.rb b/activerecord/lib/active_record/relation/where_clause_factory.rb index 4a0868314d..1374785354 100644 --- a/activerecord/lib/active_record/relation/where_clause_factory.rb +++ b/activerecord/lib/active_record/relation/where_clause_factory.rb @@ -17,63 +17,19 @@ module ActiveRecord attributes = klass.send(:expand_hash_conditions_for_aggregates, attributes) attributes.stringify_keys! - if perform_case_sensitive?(options = other.last) - parts, binds = build_for_case_sensitive(attributes, options) - else - attributes, binds = predicate_builder.create_binds(attributes) - parts = predicate_builder.build_from_hash(attributes) - end + parts = predicate_builder.build_from_hash(attributes) when Arel::Nodes::Node parts = [opts] else raise ArgumentError, "Unsupported argument type: #{opts} (#{opts.class})" end - WhereClause.new(parts, binds || []) + WhereClause.new(parts) end - # TODO Change this to private once we've dropped Ruby 2.2 support. - # Workaround for Ruby 2.2 "private attribute?" warning. protected attr_reader :klass, :predicate_builder - - private - - def perform_case_sensitive?(options) - options && options.key?(:case_sensitive) - end - - def build_for_case_sensitive(attributes, options) - parts, binds = [], [] - table = klass.arel_table - - attributes.each do |attribute, value| - if reflection = klass._reflect_on_association(attribute) - attribute = reflection.foreign_key.to_s - value = value[reflection.klass.primary_key] unless value.nil? - end - - if value.nil? - parts << table[attribute].eq(value) - else - column = klass.column_for_attribute(attribute) - - binds << predicate_builder.send(:build_bind_attribute, attribute, value) - value = Arel::Nodes::BindParam.new - - predicate = if options[:case_sensitive] - klass.connection.case_sensitive_comparison(table, attribute, column, value) - else - klass.connection.case_insensitive_comparison(table, attribute, column, value) - end - - parts << predicate - end - end - - [parts, binds] - end end end end -- cgit v1.2.3