From 445c12f7dfa345b86e05dc610d665f9afde14c26 Mon Sep 17 00:00:00 2001 From: Sean Griffin Date: Sun, 29 Mar 2015 13:51:56 -0600 Subject: Reduce memory usage when loading types in PG We were never clearing the `PG::Result` object used to query the types when the connection is first established. This would lead to a potentially large amount of memory being retained for the life of the connection. Investigating this issue also revealed several low hanging fruit on the performance of these methods, and the number of allocations has been reduced by ~90%. Fixes #19578 --- activerecord/CHANGELOG.md | 6 ++++++ .../postgresql/oid/type_map_initializer.rb | 20 ++++++++++++++++---- .../connection_adapters/postgresql_adapter.rb | 14 ++++++++++---- .../lib/active_record/type/hash_lookup_type_map.rb | 10 ++++++++-- 4 files changed, 40 insertions(+), 10 deletions(-) diff --git a/activerecord/CHANGELOG.md b/activerecord/CHANGELOG.md index be3a461eda..8793d6de70 100644 --- a/activerecord/CHANGELOG.md +++ b/activerecord/CHANGELOG.md @@ -1,3 +1,9 @@ +* Reduce memory usage from loading types on pg. + + Fixes #19578. + + *Sean Griffin* + * Add `config.active_record.warn_on_records_fetched_greater_than` option When set to an integer, a warning will be logged whenever a result set diff --git a/activerecord/lib/active_record/connection_adapters/postgresql/oid/type_map_initializer.rb b/activerecord/lib/active_record/connection_adapters/postgresql/oid/type_map_initializer.rb index 9d421d6975..191c828e60 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql/oid/type_map_initializer.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql/oid/type_map_initializer.rb @@ -15,10 +15,10 @@ module ActiveRecord def run(records) nodes = records.reject { |row| @store.key? row['oid'].to_i } mapped, nodes = nodes.partition { |row| @store.key? row['typname'] } - ranges, nodes = nodes.partition { |row| row['typtype'] == 'r' } - enums, nodes = nodes.partition { |row| row['typtype'] == 'e' } - domains, nodes = nodes.partition { |row| row['typtype'] == 'd' } - arrays, nodes = nodes.partition { |row| row['typinput'] == 'array_in' } + ranges, nodes = nodes.partition { |row| row['typtype'] == 'r'.freeze } + enums, nodes = nodes.partition { |row| row['typtype'] == 'e'.freeze } + domains, nodes = nodes.partition { |row| row['typtype'] == 'd'.freeze } + arrays, nodes = nodes.partition { |row| row['typinput'] == 'array_in'.freeze } composites, nodes = nodes.partition { |row| row['typelem'].to_i != 0 } mapped.each { |row| register_mapped_type(row) } @@ -29,6 +29,18 @@ module ActiveRecord composites.each { |row| register_composite_type(row) } end + def query_conditions_for_initial_load(type_map) + known_type_names = type_map.keys.map { |n| "'#{n}'" } + known_type_types = %w('r' 'e' 'd') + <<-SQL % [known_type_names.join(", "), known_type_types.join(", ")] + WHERE + t.typname IN (%s) + OR t.typtype IN (%s) + OR t.typinput::varchar = 'array_in' + OR t.typelem != 0 + SQL + end + private def register_mapped_type(row) alias_type row['oid'], row['typname'] diff --git a/activerecord/lib/active_record/connection_adapters/postgresql_adapter.rb b/activerecord/lib/active_record/connection_adapters/postgresql_adapter.rb index e616cee5eb..00fc69c878 100644 --- a/activerecord/lib/active_record/connection_adapters/postgresql_adapter.rb +++ b/activerecord/lib/active_record/connection_adapters/postgresql_adapter.rb @@ -594,6 +594,8 @@ module ActiveRecord end def load_additional_types(type_map, oids = nil) # :nodoc: + initializer = OID::TypeMapInitializer.new(type_map) + if supports_ranges? query = <<-SQL SELECT t.oid, t.typname, t.typelem, t.typdelim, t.typinput, r.rngsubtype, t.typtype, t.typbasetype @@ -609,11 +611,13 @@ module ActiveRecord if oids query += "WHERE t.oid::integer IN (%s)" % oids.join(", ") + else + query += initializer.query_conditions_for_initial_load(type_map) end - initializer = OID::TypeMapInitializer.new(type_map) - records = execute(query, 'SCHEMA') - initializer.run(records) + execute_and_clear(query, 'SCHEMA', []) do |records| + initializer.run(records) + end end FEATURE_NOT_SUPPORTED = "0A000" #:nodoc: @@ -823,9 +827,11 @@ module ActiveRecord 'float8' => PG::TextDecoder::Float, 'bool' => PG::TextDecoder::Boolean, } - query = <<-SQL + known_coder_types = coders_by_name.keys.map { |n| quote(n) } + query = <<-SQL % known_coder_types.join(", ") SELECT t.oid, t.typname FROM pg_type as t + WHERE t.typname IN (%s) SQL coders = execute_and_clear(query, "SCHEMA", []) do |result| result diff --git a/activerecord/lib/active_record/type/hash_lookup_type_map.rb b/activerecord/lib/active_record/type/hash_lookup_type_map.rb index 82d9327fc0..3b01e3f8ca 100644 --- a/activerecord/lib/active_record/type/hash_lookup_type_map.rb +++ b/activerecord/lib/active_record/type/hash_lookup_type_map.rb @@ -1,12 +1,18 @@ module ActiveRecord module Type class HashLookupTypeMap < TypeMap # :nodoc: - delegate :key?, to: :@mapping - def alias_type(type, alias_type) register_type(type) { |_, *args| lookup(alias_type, *args) } end + def key?(key) + @mapping.key?(key) + end + + def keys + @mapping.keys + end + private def perform_fetch(type, *args, &block) -- cgit v1.2.3